From e62b255b136627a1c6713b9c36342bc41bd70517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 9 Dec 2025 00:14:55 +0100 Subject: [PATCH 01/95] init support for dml and ddl statements --- .../elastic/sql/bridge/package.scala | 22 +- .../elastic/sql/SQLCriteriaSpec.scala | 2 +- .../elastic/sql/SQLQuerySpec.scala | 114 ++--- build.sbt | 2 +- .../elastic/client/AggregateApi.scala | 6 +- .../softnetwork/elastic/client/AliasApi.scala | 2 +- .../client/ElasticClientDelegator.scala | 48 +- .../elastic/client/ScrollApi.scala | 16 +- .../elastic/client/SearchApi.scala | 65 +-- .../elastic/client/SettingsApi.scala | 4 +- .../client/metrics/MetricsElasticClient.scala | 18 +- .../elastic/client/SettingsApiSpec.scala | 2 +- documentation/sql/README.md | 4 +- documentation/sql/ddl_statements.md | 0 documentation/sql/dml_statements.md | 0 ...request_structure.md => dql_statements.md} | 0 documentation/sql/keywords.md | 7 + .../elastic/sql/bridge/package.scala | 22 +- .../elastic/sql/SQLCriteriaSpec.scala | 2 +- .../elastic/sql/SQLQuerySpec.scala | 122 ++--- .../elastic/client/jest/JestClientApi.scala | 8 +- .../elastic/client/jest/JestMappingApi.scala | 2 +- .../elastic/client/jest/JestScrollApi.scala | 2 +- .../elastic/client/jest/JestSearchApi.scala | 6 +- .../client/rest/RestHighLevelClientApi.scala | 4 +- .../client/rest/RestHighLevelClientApi.scala | 4 +- .../elastic/client/java/JavaClientApi.scala | 8 +- .../elastic/client/java/JavaClientApi.scala | 8 +- .../sql/macros/SQLQueryValidatorSpec.scala | 32 +- .../client/macros/TestElasticClientApi.scala | 4 +- .../elastic/sql/macros/SQLQueryMacros.scala | 6 +- .../sql/macros/SQLQueryValidator.scala | 20 +- .../persistence/query/ElasticProvider.scala | 4 +- .../elastic/sql/SQLImplicits.scala | 17 +- .../sql/function/aggregate/package.scala | 29 +- .../elastic/sql/function/geo/package.scala | 4 +- .../elastic/sql/function/package.scala | 4 +- .../app/softnetwork/elastic/sql/package.scala | 15 +- .../elastic/sql/parser/DmlParser.scala | 5 + .../elastic/sql/parser/Parser.scala | 212 +++++++- .../elastic/sql/parser/type/package.scala | 13 +- .../softnetwork/elastic/sql/query/From.scala | 11 +- .../elastic/sql/query/GroupBy.scala | 4 +- .../elastic/sql/query/Having.scala | 2 +- .../elastic/sql/query/OrderBy.scala | 5 +- .../sql/query/SQLMultiSearchRequest.scala | 38 -- .../elastic/sql/query/SQLQuery.scala | 34 -- .../elastic/sql/query/SQLSearchRequest.scala | 202 -------- .../elastic/sql/query/Select.scala | 8 +- .../softnetwork/elastic/sql/query/Where.scala | 32 +- .../elastic/sql/query/package.scala | 427 ++++++++++++++++ .../elastic/sql/schema/package.scala | 127 +++++ .../time/DateTimeFunctionSuite.scala} | 10 +- .../ParserSpec.scala} | 476 ++++++++++++++---- .../elastic/client/ElasticClientSpec.scala | 4 +- .../elastic/client/MockElasticClientApi.scala | 4 +- 56 files changed, 1518 insertions(+), 731 deletions(-) create mode 100644 documentation/sql/ddl_statements.md create mode 100644 documentation/sql/dml_statements.md rename documentation/sql/{request_structure.md => dql_statements.md} (100%) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala delete mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala rename sql/src/test/scala/app/softnetwork/elastic/sql/{SQLDateTimeFunctionSuite.scala => function/time/DateTimeFunctionSuite.scala} (95%) rename sql/src/test/scala/app/softnetwork/elastic/sql/{SQLParserSpec.scala => parser/ParserSpec.scala} (74%) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 69be62e9..2cc10739 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -52,7 +52,7 @@ import scala.language.implicitConversions package object bridge { implicit def requestToNestedFilterAggregation( - request: SQLSearchRequest, + request: SingleSearch, innerHitsName: String ): Option[FilterAggregation] = { val having: Option[Query] = @@ -129,7 +129,7 @@ package object bridge { } implicit def requestToFilterAggregation( - request: SQLSearchRequest + request: SingleSearch ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -146,7 +146,7 @@ package object bridge { } implicit def requestToRootAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -198,7 +198,7 @@ package object bridge { } implicit def requestToScopedAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path @@ -330,7 +330,7 @@ package object bridge { scopedAggregations } - implicit def requestToNestedWithoutCriteriaQuery(request: SQLSearchRequest): Option[Query] = + implicit def requestToNestedWithoutCriteriaQuery(request: SingleSearch): Option[Query] = NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match { case Nil => None case nestedTrees => @@ -410,7 +410,7 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -425,7 +425,7 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + implicit def requestToSearchRequest(request: SingleSearch): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -556,7 +556,7 @@ package object bridge { } implicit def requestToMultiSearchRequest( - request: SQLMultiSearchRequest + request: MultiSearch ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) @@ -996,12 +996,12 @@ package object bridge { } implicit def sqlQueryToAggregations( - query: SQLQuery + query: SelectStatement ): Seq[ElasticAggregation] = { import query._ - request + statement .map { - case Left(l) => + case l: SingleSearch => val filteredAgg: Option[FilterAggregation] = requestToFilterAggregation(l) l.aggregates .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts, l.sqlAggregations)) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 8da795a9..9f3bdc46 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { - import Queries._ + import parser.Queries._ import scala.language.implicitConversions diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 245bd887..fb8a7d9e 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.Queries._ +import app.softnetwork.elastic.sql.parser.Queries._ import app.softnetwork.elastic.sql.query._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -12,9 +12,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions - implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { - sqlQuery.request match { - case Some(Left(value)) => + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) case None => throw new IllegalArgumentException( @@ -23,9 +23,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } } - "SQLQuery" should "perform native count" in { + "SelectStatement" should "perform native count" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(t.id) c2 from Table t where t.nom = 'Nom'") + SelectStatement("select count(t.id) c2 from Table t where t.nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -61,7 +61,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform count distinct" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") + SelectStatement("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -97,7 +97,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as email from index i join unnest(i.emails) as inner_emails where i.nom = 'Nom'" ) results.size shouldBe 1 @@ -142,7 +142,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with nested criteria" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" ) results.size shouldBe 1 @@ -205,7 +205,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with filter" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\"" ) results.size shouldBe 1 @@ -279,7 +279,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with \"and not\" operator" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" ) results.size shouldBe 1 @@ -353,7 +353,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with date filtering" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" ) results.size shouldBe 1 @@ -421,7 +421,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested select" in { val select: ElasticSearchRequest = - SQLQuery(""" + SelectStatement(""" |SELECT |profileId, |profile_ccm.email as email, @@ -499,7 +499,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "exclude fields from select" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( except ) select.query shouldBe @@ -517,7 +517,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform query with group by and having" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHaving) + SelectStatement(groupByWithHaving) val query = select.query println(query) query shouldBe @@ -577,7 +577,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform complex query" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( s"""SELECT | inner_products.category as cat, | min(inner_products.price) as min_price, @@ -839,7 +839,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "add script fields" in { val select: ElasticSearchRequest = - SQLQuery(fieldsWithInterval) + SelectStatement(fieldsWithInterval) val query = select.query println(query) query shouldBe @@ -879,7 +879,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateTimeAndInterval) + SelectStatement(filterWithDateTimeAndInterval) val query = select.query println(query) query shouldBe @@ -914,7 +914,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateAndInterval) + SelectStatement(filterWithDateAndInterval) val query = select.query println(query) query shouldBe @@ -949,7 +949,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithTimeAndInterval) + SelectStatement(filterWithTimeAndInterval) val query = select.query println(query) query shouldBe @@ -997,7 +997,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle having with date functions" in { val select: ElasticSearchRequest = - SQLQuery("""SELECT userId, MAX(createdAt) as lastSeen + SelectStatement("""SELECT userId, MAX(createdAt) as lastSeen |FROM table |GROUP BY userId |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) @@ -1045,7 +1045,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle group by with having and date time functions" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHavingAndDateTimeFunctions) + SelectStatement(groupByWithHavingAndDateTimeFunctions) val query = select.query println(query) query shouldBe @@ -1110,7 +1110,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle group by index" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( groupByWithHavingAndDateTimeFunctions.replace("GROUP BY Country, City", "GROUP BY 3, 2") ) val query = select.query @@ -1177,7 +1177,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_parse function" in { val select: ElasticSearchRequest = - SQLQuery(dateParse) + SelectStatement(dateParse) val query = select.query println(query) query shouldBe @@ -1246,7 +1246,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateFormat) + SelectStatement(dateFormat) val query = select.query println(query) query shouldBe @@ -1345,7 +1345,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_parse function" in { // #25 val select: ElasticSearchRequest = - SQLQuery(dateTimeParse) + SelectStatement(dateTimeParse) val query = select.query println(query) query shouldBe @@ -1415,7 +1415,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeFormat) + SelectStatement(dateTimeFormat) val query = select.query println(query) query shouldBe @@ -1469,7 +1469,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_diff function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateDiff) + SelectStatement(dateDiff) val query = select.query println(query) query shouldBe @@ -1511,7 +1511,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with date_diff function" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithDateDiff) + SelectStatement(aggregationWithDateDiff) val query = select.query println(query) query shouldBe @@ -1564,7 +1564,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateAdd) + SelectStatement(dateAdd) val query = select.query println(query) query shouldBe @@ -1615,7 +1615,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_sub function as script field" in { // 30 val select: ElasticSearchRequest = - SQLQuery(dateSub) + SelectStatement(dateSub) val query = select.query println(query) query shouldBe @@ -1666,7 +1666,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeAdd) + SelectStatement(dateTimeAdd) val query = select.query println(query) query shouldBe @@ -1717,7 +1717,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_sub function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeSub) + SelectStatement(dateTimeSub) val query = select.query println(query) query shouldBe @@ -1768,7 +1768,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnull) + SelectStatement(isnull) val query = select.query println(query) query shouldBe @@ -1806,7 +1806,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnotnull) + SelectStatement(isnotnull) val query = select.query println(query) query shouldBe @@ -1848,7 +1848,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null criteria as must_not exists" in { val select: ElasticSearchRequest = - SQLQuery(isNullCriteria) + SelectStatement(isNullCriteria) val query = select.query println(query) query shouldBe @@ -1880,7 +1880,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull criteria as exists" in { val select: ElasticSearchRequest = - SQLQuery(isNotNullCriteria) + SelectStatement(isNotNullCriteria) val query = select.query println(query) query shouldBe @@ -1906,7 +1906,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle coalesce function as script field" in { val select: ElasticSearchRequest = - SQLQuery(coalesce) + SelectStatement(coalesce) val query = select.query println(query) query shouldBe @@ -1957,7 +1957,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nullif function as script field" in { val select: ElasticSearchRequest = - SQLQuery(nullif) + SelectStatement(nullif) val query = select.query println(query) query shouldBe @@ -2016,7 +2016,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(conversion) + SelectStatement(conversion) val query = select.query println(query) query shouldBe @@ -2101,7 +2101,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle case function as script field" in { // 40 val select: ElasticSearchRequest = - SQLQuery(caseWhen) + SelectStatement(caseWhen) val query = select.query println(query) query shouldBe @@ -2154,7 +2154,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle case with expression function as script field" in { val select: ElasticSearchRequest = - SQLQuery(caseWhenExpr) + SelectStatement(caseWhenExpr) val query = select.query println(query) query shouldBe @@ -2209,7 +2209,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle extract function as script field" in { val select: ElasticSearchRequest = - SQLQuery(extract) + SelectStatement(extract) val query = select.query println(query) query shouldBe @@ -2333,7 +2333,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle arithmetic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(arithmetic.replace("as group1", "")) + SelectStatement(arithmetic.replace("as group1", "")) val query = select.query println(query) query shouldBe @@ -2422,7 +2422,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle mathematic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(mathematical) + SelectStatement(mathematical) val query = select.query println(query) query shouldBe @@ -2592,7 +2592,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle string function as script field and condition" in { // 45 val select: ElasticSearchRequest = - SQLQuery(string) + SelectStatement(string) val query = select.query println(query) query shouldBe @@ -2748,7 +2748,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle top hits aggregation" in { val select: ElasticSearchRequest = - SQLQuery(topHits) + SelectStatement(topHits) val query = select.query println(query) query shouldBe @@ -2865,7 +2865,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle last day function" in { val select: ElasticSearchRequest = - SQLQuery(lastDay) + SelectStatement(lastDay) val query = select.query println(query) query shouldBe @@ -2932,7 +2932,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle all extractors" in { val select: ElasticSearchRequest = - SQLQuery(extractors) + SelectStatement(extractors) val query = select.query println(query) query shouldBe @@ -3069,7 +3069,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = - SQLQuery(geoDistance) + SelectStatement(geoDistance) val query = select.query println(query) query shouldBe @@ -3191,7 +3191,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle between with temporal" in { // 50 val select: ElasticSearchRequest = - SQLQuery(betweenTemporal) + SelectStatement(betweenTemporal) val query = select.query println(query) query shouldBe @@ -3276,7 +3276,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested of nested" in { val select: ElasticSearchRequest = - SQLQuery(nestedOfNested) + SelectStatement(nestedOfNested) val query = select.query println(query) query shouldBe @@ -3384,7 +3384,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle predicate with distinct nested" in { val select: ElasticSearchRequest = - SQLQuery(predicateWithDistinctNested) + SelectStatement(predicateWithDistinctNested) val query = select.query println(query) query shouldBe @@ -3494,7 +3494,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested without criteria" in { val select: ElasticSearchRequest = - SQLQuery(nestedWithoutCriteria) + SelectStatement(nestedWithoutCriteria) val query = select.query println(query) query shouldBe @@ -3598,7 +3598,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "determine the aggregation context" in { val select: ElasticSearchRequest = - SQLQuery(determinationOfTheAggregationContext) + SelectStatement(determinationOfTheAggregationContext) val query = select.query println(query) query shouldBe @@ -3632,7 +3632,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with nested of nested context" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithNestedOfNestedContext) + SelectStatement(aggregationWithNestedOfNestedContext) val query = select.query println(query) query shouldBe @@ -3668,7 +3668,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle where filters according to scope" in { val select: ElasticSearchRequest = - SQLQuery(whereFiltersAccordingToScope) + SelectStatement(whereFiltersAccordingToScope) val query = select.query println(query) query shouldBe diff --git a/build.sbt b/build.sbt index 9ff003d8..7fe63779 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.14.1" +ThisBuild / version := "0.15.0" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala index caa3325c..3bb04c7d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import java.time.temporal.Temporal import scala.annotation.tailrec @@ -37,7 +37,7 @@ trait AggregateApi[T <: AggregateResult] { * @return * a sequence of aggregated results */ - def aggregate(sqlQuery: SQLQuery)(implicit + def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[_root_.scala.collection.Seq[T]]] } @@ -57,7 +57,7 @@ trait SingleValueAggregateApi * a sequence of aggregated results */ override def aggregate( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit ec: ExecutionContext ): Future[ElasticResult[_root_.scala.collection.Seq[SingleValueAggregateResult]]] = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala index ccc76be6..18e76d7c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala @@ -292,7 +292,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => // ✅ Extracting aliases from JSON ElasticResult.fromTry( Try { - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject } ) match { case ElasticFailure(error) => diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 55a1f588..fa3e2da0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -22,7 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -146,26 +146,26 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { index: String, settings: String ): ElasticResult[Boolean] = - delegate.createIndex(index, settings) + delegate.executeCreateIndex(index, settings) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - delegate.deleteIndex(index) + delegate.executeDeleteIndex(index) override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - delegate.closeIndex(index) + delegate.executeCloseIndex(index) override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = - delegate.openIndex(index) + delegate.executeOpenIndex(index) override private[client] def executeReindex( sourceIndex: String, targetIndex: String, refresh: Boolean ): ElasticResult[(Boolean, Option[Long])] = - delegate.reindex(sourceIndex, targetIndex, refresh) + delegate.executeReindex(sourceIndex, targetIndex, refresh) override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - delegate.indexExists(index) + delegate.executeIndexExists(index) // ==================== AliasApi ==================== @@ -289,16 +289,16 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { index: String, alias: String ): ElasticResult[Boolean] = - delegate.addAlias(index, alias) + delegate.executeAddAlias(index, alias) override private[client] def executeRemoveAlias( index: String, alias: String ): ElasticResult[Boolean] = - delegate.removeAlias(index, alias) + delegate.executeRemoveAlias(index, alias) override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = - delegate.aliasExists(alias) + delegate.executeAliasExists(alias) override private[client] def executeGetAliases(index: String): ElasticResult[String] = delegate.executeGetAliases(index) @@ -308,7 +308,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { newIndex: String, alias: String ): ElasticResult[Boolean] = - delegate.swapAlias(oldIndex, newIndex, alias) + delegate.executeSwapAlias(oldIndex, newIndex, alias) // ==================== SettingsApi ==================== @@ -362,10 +362,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { index: String, settings: String ): ElasticResult[Boolean] = - delegate.updateSettings(index, settings) + delegate.executeUpdateSettings(index, settings) override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { - delegate.loadSettings(index) + delegate.executeLoadSettings(index) } // ==================== MappingApi ==================== @@ -440,10 +440,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { index: String, mapping: String ): ElasticResult[Boolean] = - delegate.setMapping(index, mapping) + delegate.executeSetMapping(index, mapping) override private[client] def executeGetMapping(index: String): ElasticResult[String] = { - delegate.getMapping(index) + delegate.executeGetMapping(index) } // ==================== RefreshApi ==================== @@ -882,7 +882,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a sequence of aggregated results */ - override def aggregate(sqlQuery: SQLQuery)(implicit + override def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[collection.Seq[SingleValueAggregateResult]]] = delegate.aggregate(sqlQuery) @@ -896,7 +896,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * the Elasticsearch response */ - override def search(sql: SQLQuery): ElasticResult[ElasticResponse] = delegate.search(sql) + override def search(sql: SelectStatement): ElasticResult[ElasticResponse] = delegate.search(sql) /** Search for documents / aggregations matching the Elasticsearch query. * @@ -941,7 +941,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a Future containing the Elasticsearch response */ - override def searchAsync(sqlQuery: SQLQuery)(implicit + override def searchAsync(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = delegate.searchAsync(sqlQuery) @@ -991,7 +991,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * the entities matching the query */ override def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = delegate.searchAsUnchecked(sqlQuery) @@ -1047,7 +1047,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a Future containing the entities */ - override def searchAsyncAsUnchecked[U](sqlQuery: SQLQuery)(implicit + override def searchAsyncAsUnchecked[U](sqlQuery: SelectStatement)(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats @@ -1103,7 +1103,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.multiSearchAsyncAs(elasticQueries, fieldAliases, aggregations) override def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats @@ -1123,7 +1123,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.multisearchWithInnerHits[U, I](elasticQueries, innerField) override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SQLSearchRequest + sqlSearch: SingleSearch ): String = delegate.sqlSearchRequestToJsonQuery(sqlSearch) @@ -1151,7 +1151,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SQLQuery, config: ScrollConfig)(implicit + override def scroll(sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(sql, config) @@ -1175,7 +1175,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * - Source of tuples (T, ScrollMetrics) */ - override def scrollAsUnchecked[T](sql: SQLQuery, config: ScrollConfig)(implicit + override def scrollAsUnchecked[T](sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem, m: Manifest[T], formats: Formats diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 6d738565..17415e87 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -29,7 +29,7 @@ import app.softnetwork.elastic.client.scroll.{ UseSearchAfter } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} import org.json4s.{Formats, JNothing} import org.json4s.jackson.JsonMethods.parse @@ -117,11 +117,11 @@ trait ScrollApi extends ElasticClientHelpers { /** Create a scrolling source with automatic strategy selection */ def scroll( - sql: SQLQuery, + sql: SelectStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { - sql.request match { - case Some(Left(single)) => + sql.statement match { + case Some(single: SingleSearch) => if (single.windowFunctions.nonEmpty) return scrollWithWindowEnrichment(sql, single, config) @@ -136,7 +136,7 @@ trait ScrollApi extends ElasticClientHelpers { single.sorts.nonEmpty ) - case Some(Right(_)) => + case Some(_) => Source.failed( new UnsupportedOperationException("Scrolling is not supported for multi-search queries") ) @@ -224,7 +224,7 @@ trait ScrollApi extends ElasticClientHelpers { * - Source of tuples (T, ScrollMetrics) */ def scrollAsUnchecked[T]( - sql: SQLQuery, + sql: SelectStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem, @@ -376,8 +376,8 @@ trait ScrollApi extends ElasticClientHelpers { /** Scroll with window function enrichment */ private def scrollWithWindowEnrichment( - sql: SQLQuery, - request: SQLSearchRequest, + sql: SelectStatement, + request: SingleSearch, config: ScrollConfig )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 947ae75f..315f8cd9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -23,7 +23,12 @@ import app.softnetwork.elastic.client.result.{ ElasticSuccess } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{ + MultiSearch, + SQLAggregation, + SelectStatement, + SingleSearch +} import com.google.gson.{Gson, JsonElement, JsonObject, JsonParser} import org.json4s.Formats @@ -60,9 +65,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * the Elasticsearch response */ - def search(sql: SQLQuery): ElasticResult[ElasticResponse] = { - sql.request match { - case Some(Left(single)) => + def search(sql: SelectStatement): ElasticResult[ElasticResponse] = { + sql.statement match { + case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*), @@ -73,7 +78,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { else singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(Right(multiple)) => + case Some(multiple: MultiSearch) => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -295,19 +300,19 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * a Future containing the Elasticsearch response */ def searchAsync( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { - sqlQuery.request match { - case Some(Left(single)) => + sqlQuery.statement match { + case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*) ) singleSearchAsync(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(Right(multiple)) => + case Some(multiple: MultiSearch) => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -529,7 +534,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * the entities matching the query */ def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = { for { response <- search(sqlQuery) @@ -626,7 +631,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * a Future containing the entities */ def searchAsyncAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], ec: ExecutionContext, @@ -739,20 +744,20 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * tuples (main entity, inner hits) */ def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats ): ElasticResult[Seq[(U, Seq[I])]] = { - sql.request match { - case Some(Left(single)) => + sql.statement match { + case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*) ) singleSearchWithInnerHits[U, I](elasticQuery, innerField) - case Some(Right(multiple)) => + case Some(multiple: MultiSearch) => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -981,7 +986,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String + private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String private def parseInnerHits[M: Manifest: ClassTag, I: Manifest: ClassTag]( searchResult: JsonObject, @@ -1095,8 +1100,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * functions) 3. Enrich results with window values */ private def searchWithWindowEnrichment( - sql: SQLQuery, - request: SQLSearchRequest + sql: SelectStatement, + request: SingleSearch ): ElasticResult[ElasticResponse] = { logger.info(s"🪟 Detected ${request.windowFunctions.size} window functions") @@ -1122,7 +1127,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * window values */ protected def executeWindowAggregations( - request: SQLSearchRequest + request: SingleSearch ): ElasticResult[WindowCache] = { // Build aggregation request @@ -1157,8 +1162,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Build aggregation request for window functions */ private def buildWindowAggregationRequest( - request: SQLSearchRequest - ): SQLSearchRequest = { + request: SingleSearch + ): SingleSearch = { // Create modified request with: // - Only window buckets in GROUP BY @@ -1180,7 +1185,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { */ private def parseWindowAggregationsToCache( response: ElasticResponse, - request: SQLSearchRequest + request: SingleSearch ): ElasticResult[WindowCache] = { logger.info( @@ -1209,8 +1214,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Execute base query without window functions */ private def executeBaseQuery( - sql: SQLQuery, - request: SQLSearchRequest + sql: SelectStatement, + request: SingleSearch ): ElasticResult[ElasticResponse] = { val baseQuery = createBaseQuery(sql, request) @@ -1231,9 +1236,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Create base query by removing window functions from SELECT */ protected def createBaseQuery( - sql: SQLQuery, - request: SQLSearchRequest - ): SQLSearchRequest = { + sql: SelectStatement, + request: SingleSearch + ): SingleSearch = { // Remove window function fields from SELECT val baseFields = request.select.fields.filterNot(_.identifier.hasWindow) @@ -1253,7 +1258,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { */ private def extractPartitionKey( row: Map[String, Any], - request: SQLSearchRequest + request: SingleSearch ): PartitionKey = { // Get all partition fields from window functions @@ -1294,7 +1299,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { private def enrichResponseWithWindowValues( response: ElasticResponse, cache: WindowCache, - request: SQLSearchRequest + request: SingleSearch ): ElasticResult[ElasticResponse] = { val baseRows = response.results @@ -1311,7 +1316,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { protected def enrichDocumentWithWindowValues( doc: Map[String, Any], cache: WindowCache, - request: SQLSearchRequest + request: SingleSearch ): Map[String, Any] = { if (request.windowFunctions.isEmpty) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala index ddefc61b..cd76abeb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala @@ -129,7 +129,7 @@ trait SettingsApi { _: IndicesApi => def getRefreshInterval(index: String): ElasticResult[String] = { loadSettings(index).flatMap { settingsJson => ElasticResult.attempt( - new JsonParser().parse(settingsJson).getAsJsonObject + JsonParser.parseString(settingsJson).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON settings for index '$index': ${error.message}") @@ -192,7 +192,7 @@ trait SettingsApi { _: IndicesApi => executeLoadSettings(index).flatMap { jsonString => // ✅ Extracting settings from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON settings for index '$index': ${error.message}") diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 8a16dabd..78ff461b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -30,7 +30,7 @@ import app.softnetwork.elastic.client.{ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement} import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -619,7 +619,7 @@ class MetricsElasticClient( * @return * a sequence of aggregated results */ - override def aggregate(sqlQuery: SQLQuery)(implicit + override def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[collection.Seq[SingleValueAggregateResult]]] = measureAsync("aggregate") { @@ -635,7 +635,7 @@ class MetricsElasticClient( * @return * the Elasticsearch response */ - override def search(sql: SQLQuery): ElasticResult[ElasticResponse] = + override def search(sql: SelectStatement): ElasticResult[ElasticResponse] = measureResult("search") { delegate.search(sql) } @@ -648,7 +648,7 @@ class MetricsElasticClient( * a Future containing the Elasticsearch response */ override def searchAsync( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit ec: ExecutionContext): Future[ElasticResult[ElasticResponse]] = measureAsync("searchAsync") { delegate.searchAsync(sqlQuery) @@ -664,7 +664,7 @@ class MetricsElasticClient( * the entities matching the query */ override def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = measureResult("searchAs") { delegate.searchAsUnchecked[U](sqlQuery) @@ -682,7 +682,7 @@ class MetricsElasticClient( * @return * a Future containing the entities */ - override def searchAsyncAsUnchecked[U](sqlQuery: SQLQuery)(implicit + override def searchAsyncAsUnchecked[U](sqlQuery: SelectStatement)(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats @@ -848,7 +848,7 @@ class MetricsElasticClient( } override def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats @@ -881,7 +881,7 @@ class MetricsElasticClient( /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SQLQuery, config: ScrollConfig)(implicit + override def scroll(sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { // Note: For streams, we measure at the beginning but not every element @@ -922,7 +922,7 @@ class MetricsElasticClient( * @return * - Source of tuples (T, ScrollMetrics) */ - override def scrollAsUnchecked[T](sql: SQLQuery, config: ScrollConfig)(implicit + override def scrollAsUnchecked[T](sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem, m: Manifest[T], formats: Formats diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index e6a86dba..298fa1b1 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -489,7 +489,7 @@ class SettingsApiSpec // Then result.isSuccess shouldBe true - val parsedResult = new JsonParser().parse(result.get).getAsJsonObject + val parsedResult = JsonParser.parseString(result.get).getAsJsonObject parsedResult.has("number_of_shards") shouldBe true parsedResult.get("number_of_shards").getAsString shouldBe "3" } diff --git a/documentation/sql/README.md b/documentation/sql/README.md index d383815f..324a61e1 100644 --- a/documentation/sql/README.md +++ b/documentation/sql/README.md @@ -2,7 +2,7 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: -- [Query Structure](request_structure.md) +- [Query Structure](dql_statements.md) - [Query Validation](validation.md) - [Operators](operators.md) - [Operator Precedence](operator_precedence.md) @@ -14,3 +14,5 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: - [Conditional Functions](functions_conditional.md) - [Geo Functions](functions_geo.md) - [Keywords](keywords.md) +- [DML Statements](dml_statements.md) +- [DDL Statements](ddl_statements.md) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md new file mode 100644 index 00000000..e69de29b diff --git a/documentation/sql/request_structure.md b/documentation/sql/dql_statements.md similarity index 100% rename from documentation/sql/request_structure.md rename to documentation/sql/dql_statements.md diff --git a/documentation/sql/keywords.md b/documentation/sql/keywords.md index 986848de..5cb98962 100644 --- a/documentation/sql/keywords.md +++ b/documentation/sql/keywords.md @@ -6,6 +6,13 @@ A list of reserved words recognized by the parser for this engine. ## Main clauses SELECT +INSERT +UPDATE +DELETE +CREATE +ALTER +DROP +TRUNCATE FROM JOIN UNNEST diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 7f7843b4..87a77d4f 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -48,7 +48,7 @@ import scala.language.implicitConversions package object bridge { implicit def requestToNestedFilterAggregation( - request: SQLSearchRequest, + request: SingleSearch, innerHitsName: String ): Option[FilterAggregation] = { val having: Option[Query] = @@ -125,7 +125,7 @@ package object bridge { } implicit def requestToFilterAggregation( - request: SQLSearchRequest + request: SingleSearch ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -142,7 +142,7 @@ package object bridge { } implicit def requestToRootAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -192,7 +192,7 @@ package object bridge { } implicit def requestToScopedAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path @@ -324,7 +324,7 @@ package object bridge { scopedAggregations } - implicit def requestToNestedWithoutCriteriaQuery(request: SQLSearchRequest): Option[Query] = + implicit def requestToNestedWithoutCriteriaQuery(request: SingleSearch): Option[Query] = NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match { case Nil => None case nestedTrees => @@ -404,7 +404,7 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -419,7 +419,7 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + implicit def requestToSearchRequest(request: SingleSearch): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -550,7 +550,7 @@ package object bridge { } implicit def requestToMultiSearchRequest( - request: SQLMultiSearchRequest + request: MultiSearch ): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) @@ -991,12 +991,12 @@ package object bridge { @deprecated implicit def sqlQueryToAggregations( - query: SQLQuery + query: SelectStatement ): Seq[ElasticAggregation] = { import query._ - request + statement .map { - case Left(l) => + case l: SingleSearch => val filteredAgg: Option[FilterAggregation] = requestToFilterAggregation(l) l.aggregates .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts, l.sqlAggregations)) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 88221237..dc7cd685 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { - import Queries._ + import parser.Queries._ import scala.language.implicitConversions diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index be4c7494..1d13b242 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1,8 +1,8 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.Queries._ -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.parser.Queries._ +import app.softnetwork.elastic.sql.query.{SelectStatement, SingleSearch} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -12,9 +12,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions - implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { - sqlQuery.request match { - case Some(Left(value)) => + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) case None => throw new IllegalArgumentException( @@ -23,9 +23,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } } - "SQLQuery" should "perform native count" in { + "SelectStatement" should "perform native count" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(t.id) c2 from Table t where t.nom = 'Nom'") + SelectStatement("select count(t.id) c2 from Table t where t.nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -61,7 +61,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform count distinct" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") + SelectStatement("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -97,7 +97,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as email from index i join unnest(i.emails) as inner_emails where i.nom = 'Nom'" ) results.size shouldBe 1 @@ -142,7 +142,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with nested criteria" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" ) results.size shouldBe 1 @@ -205,7 +205,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with filter" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\"" ) results.size shouldBe 1 @@ -279,7 +279,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with \"and not\" operator" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" ) results.size shouldBe 1 @@ -353,7 +353,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with date filtering" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" ) results.size shouldBe 1 @@ -421,7 +421,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested select" in { val select: ElasticSearchRequest = - SQLQuery(""" + SelectStatement(""" |SELECT |profileId, |profile_ccm.email as email, @@ -499,7 +499,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "exclude fields from select" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( except ) select.query shouldBe @@ -517,7 +517,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform query with group by and having" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHaving) + SelectStatement(groupByWithHaving) val query = select.query println(query) query shouldBe @@ -577,7 +577,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform complex query" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( s"""SELECT | inner_products.category as cat, | min(inner_products.price) as min_price, @@ -839,7 +839,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "add script fields" in { val select: ElasticSearchRequest = - SQLQuery(fieldsWithInterval) + SelectStatement(fieldsWithInterval) val query = select.query println(query) query shouldBe @@ -879,7 +879,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateTimeAndInterval) + SelectStatement(filterWithDateTimeAndInterval) val query = select.query println(query) query shouldBe @@ -914,7 +914,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateAndInterval) + SelectStatement(filterWithDateAndInterval) val query = select.query println(query) query shouldBe @@ -949,7 +949,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithTimeAndInterval) + SelectStatement(filterWithTimeAndInterval) val query = select.query println(query) query shouldBe @@ -997,7 +997,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle having with date functions" in { val select: ElasticSearchRequest = - SQLQuery("""SELECT userId, MAX(createdAt) as lastSeen + SelectStatement("""SELECT userId, MAX(createdAt) as lastSeen |FROM table |GROUP BY userId |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) @@ -1045,7 +1045,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle group by with having and date time functions" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHavingAndDateTimeFunctions) + SelectStatement(groupByWithHavingAndDateTimeFunctions) val query = select.query println(query) query shouldBe @@ -1110,7 +1110,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle group by index" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( groupByWithHavingAndDateTimeFunctions.replace("GROUP BY Country, City", "GROUP BY 3, 2") ) val query = select.query @@ -1177,7 +1177,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_parse function" in { val select: ElasticSearchRequest = - SQLQuery(dateParse) + SelectStatement(dateParse) val query = select.query println(query) query shouldBe @@ -1246,7 +1246,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateFormat) + SelectStatement(dateFormat) val query = select.query println(query) query shouldBe @@ -1345,7 +1345,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_parse function" in { // #25 val select: ElasticSearchRequest = - SQLQuery(dateTimeParse) + SelectStatement(dateTimeParse) val query = select.query println(query) query shouldBe @@ -1415,7 +1415,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeFormat) + SelectStatement(dateTimeFormat) val query = select.query println(query) query shouldBe @@ -1469,7 +1469,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_diff function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateDiff) + SelectStatement(dateDiff) val query = select.query println(query) query shouldBe @@ -1511,7 +1511,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with date_diff function" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithDateDiff) + SelectStatement(aggregationWithDateDiff) val query = select.query println(query) query shouldBe @@ -1564,7 +1564,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateAdd) + SelectStatement(dateAdd) val query = select.query println(query) query shouldBe @@ -1615,7 +1615,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_sub function as script field" in { // 30 val select: ElasticSearchRequest = - SQLQuery(dateSub) + SelectStatement(dateSub) val query = select.query println(query) query shouldBe @@ -1666,7 +1666,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeAdd) + SelectStatement(dateTimeAdd) val query = select.query println(query) query shouldBe @@ -1717,7 +1717,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_sub function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeSub) + SelectStatement(dateTimeSub) val query = select.query println(query) query shouldBe @@ -1768,7 +1768,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnull) + SelectStatement(isnull) val query = select.query println(query) query shouldBe @@ -1806,7 +1806,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnotnull) + SelectStatement(isnotnull) val query = select.query println(query) query shouldBe @@ -1848,7 +1848,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null criteria as must_not exists" in { val select: ElasticSearchRequest = - SQLQuery(isNullCriteria) + SelectStatement(isNullCriteria) val query = select.query println(query) query shouldBe @@ -1880,7 +1880,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull criteria as exists" in { val select: ElasticSearchRequest = - SQLQuery(isNotNullCriteria) + SelectStatement(isNotNullCriteria) val query = select.query println(query) query shouldBe @@ -1906,7 +1906,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle coalesce function as script field" in { val select: ElasticSearchRequest = - SQLQuery(coalesce) + SelectStatement(coalesce) val query = select.query println(query) query shouldBe @@ -1957,7 +1957,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nullif function as script field" in { val select: ElasticSearchRequest = - SQLQuery(nullif) + SelectStatement(nullif) val query = select.query println(query) query shouldBe @@ -2016,7 +2016,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(conversion) + SelectStatement(conversion) val query = select.query println(query) query shouldBe @@ -2101,7 +2101,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle case function as script field" in { // 40 val select: ElasticSearchRequest = - SQLQuery(caseWhen) + SelectStatement(caseWhen) val query = select.query println(query) query shouldBe @@ -2154,7 +2154,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle case with expression function as script field" in { val select: ElasticSearchRequest = - SQLQuery(caseWhenExpr) + SelectStatement(caseWhenExpr) val query = select.query println(query) query shouldBe @@ -2209,7 +2209,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle extract function as script field" in { val select: ElasticSearchRequest = - SQLQuery(extract) + SelectStatement(extract) val query = select.query println(query) query shouldBe @@ -2333,7 +2333,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle arithmetic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(arithmetic.replace("as group1", "")) + SelectStatement(arithmetic.replace("as group1", "")) val query = select.query println(query) query shouldBe @@ -2422,7 +2422,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle mathematic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(mathematical) + SelectStatement(mathematical) val query = select.query println(query) query shouldBe @@ -2592,7 +2592,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle string function as script field and condition" in { // 45 val select: ElasticSearchRequest = - SQLQuery(string) + SelectStatement(string) val query = select.query println(query) query shouldBe @@ -2748,7 +2748,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle top hits aggregation" in { val select: ElasticSearchRequest = - SQLQuery(topHits) + SelectStatement(topHits) val query = select.query println(query) query shouldBe @@ -2865,7 +2865,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle last day function" in { val select: ElasticSearchRequest = - SQLQuery(lastDay) + SelectStatement(lastDay) val query = select.query println(query) query shouldBe @@ -2932,7 +2932,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle all extractors" in { val select: ElasticSearchRequest = - SQLQuery(extractors) + SelectStatement(extractors) val query = select.query println(query) query shouldBe @@ -3069,7 +3069,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = - SQLQuery(geoDistance) + SelectStatement(geoDistance) val query = select.query println(query) query shouldBe @@ -3191,7 +3191,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle between with temporal" in { // 50 val select: ElasticSearchRequest = - SQLQuery(betweenTemporal) + SelectStatement(betweenTemporal) val query = select.query println(query) query shouldBe @@ -3276,7 +3276,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested of nested" in { val select: ElasticSearchRequest = - SQLQuery(nestedOfNested) + SelectStatement(nestedOfNested) val query = select.query println(query) query shouldBe @@ -3384,7 +3384,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle predicate with distinct nested" in { val select: ElasticSearchRequest = - SQLQuery(predicateWithDistinctNested) + SelectStatement(predicateWithDistinctNested) val query = select.query println(query) query shouldBe @@ -3494,7 +3494,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested without criteria" in { val select: ElasticSearchRequest = - SQLQuery(nestedWithoutCriteria) + SelectStatement(nestedWithoutCriteria) val query = select.query println(query) query shouldBe @@ -3598,7 +3598,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "determine the aggregation context" in { val select: ElasticSearchRequest = - SQLQuery(determinationOfTheAggregationContext) + SelectStatement(determinationOfTheAggregationContext) val query = select.query println(query) query shouldBe @@ -3632,7 +3632,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with nested of nested context" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithNestedOfNestedContext) + SelectStatement(aggregationWithNestedOfNestedContext) val query = select.query println(query) query shouldBe @@ -3668,7 +3668,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle where filters according to scope" in { val select: ElasticSearchRequest = - SQLQuery(whereFiltersAccordingToScope) + SelectStatement(whereFiltersAccordingToScope) val query = select.query println(query) query shouldBe @@ -3753,7 +3753,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { "\n", " " ) - val select: ElasticSearchRequest = SQLQuery(query) + val select: ElasticSearchRequest = SelectStatement(query) println(select.query)*/ val query = """ SELECT @@ -3778,8 +3778,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | GROUP BY product_id, product_name, DATE_TRUNC( sale_date, MONTH ) | ORDER BY product_id, DATE_TRUNC( sale_date, MONTH ) |""".stripMargin - SQLQuery(query).request.flatMap(_.left.toOption) match { - case Some(request) => + SelectStatement(query).statement match { + case Some(request: SingleSearch) => val aggRequest = request .copy( diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index f6cd2e76..8d34727b 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client._ -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.{SelectStatement, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import io.searchbox.action.BulkableAction import io.searchbox.core._ @@ -56,10 +56,10 @@ object JestClientApi extends SerializationApi { search.build() } - implicit class SearchSQLQuery(sqlQuery: SQLQuery) { + implicit class SearchSQLQuery(sqlQuery: SelectStatement) { def jestSearch: Option[Search] = { - sqlQuery.request match { - case Some(Left(value)) => + sqlQuery.statement match { + case Some(value: SingleSearch) => val request: ElasticSearchRequest = value Some(request) case _ => None diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index f9a3cec0..c85ebd11 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -71,7 +71,7 @@ trait JestMappingApi extends MappingApi with JestClientHelpers { getMapping(index).flatMap { jsonString => // ✅ Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala index af2cf47f..59096d33 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala @@ -145,7 +145,7 @@ trait JestScrollApi extends ScrollApi with JestClientHelpers { logger.debug(s"Fetching next search_after batch (after: ${values.mkString(", ")})") } - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Check if sorts already exist in the query if (!hasSorts && !queryJson.has("sort")) { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 36e0881a..cf92984a 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.{ElasticQueries, ElasticQuery, SearchApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.bridge.ElasticSearchRequest -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.MultiSearch import scala.concurrent.{ExecutionContext, Future} @@ -29,7 +29,9 @@ import scala.language.implicitConversions trait JestSearchApi extends SearchApi with JestClientHelpers { _: JestClientCompanion with SerializationApi => - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: SingleSearch + ): String = implicitly[ElasticSearchRequest](sqlSearch).query import JestClientApi._ diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 9758537f..b806adc4 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -22,7 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -737,7 +737,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 63356f2e..8597d9e7 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest @@ -741,7 +741,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index ec367082..cb325d0f 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,7 +24,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import co.elastic.clients.elasticsearch._types.{ @@ -401,7 +401,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { getMapping(index).flatMap { jsonString => // ✅ Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") @@ -757,7 +757,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1253,7 +1253,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ) // Parse query to add query clause (not indices, they're in PIT) - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Extract query clause if present if (queryJson.has("query")) { diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index a6fc8e2e..6f47d492 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,7 +24,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import co.elastic.clients.elasticsearch._types.{ @@ -396,7 +396,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { getMapping(index).flatMap { jsonString => // ✅ Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") @@ -750,7 +750,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1246,7 +1246,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ) // Parse query to add query clause (not indices, they're in PIT) - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Extract query clause if present if (queryJson.has("query")) { diff --git a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala index 4733479d..7e7bc00b 100644 --- a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala +++ b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala @@ -14,7 +14,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Numbers - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Numbers]( "SELECT tiny::TINYINT as tiny, small::SMALLINT as small, normal::INT as normal, big::BIGINT as big, huge::BIGINT as huge, decimal::DOUBLE as decimal, r::REAL as r FROM numbers" @@ -26,7 +26,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Strings - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Strings]( "SELECT vchar::VARCHAR, c::CHAR, text FROM strings" @@ -38,7 +38,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Temporal - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Temporal]( "SELECT d::DATE, t::TIME, dt::DATETIME, ts::TIMESTAMP FROM temporal" @@ -50,7 +50,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, name, price::DOUBLE, stock::INT, active::BOOLEAN, createdAt::DATETIME FROM products" @@ -62,7 +62,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT product_id AS id, product_name AS name, product_price::DOUBLE AS price, product_stock::INT AS stock, is_active::BOOLEAN AS active, created_at::TIMESTAMP AS createdAt FROM products" @@ -74,7 +74,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.ProductWithOptional - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[ProductWithOptional]( "SELECT id, name FROM products" @@ -87,7 +87,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.ProductWithDefaults - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[ProductWithDefaults]( "SELECT id, name FROM products" @@ -100,10 +100,10 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAsUnchecked[Product]( - SQLQuery("SELECT * FROM products") + SelectStatement("SELECT * FROM products") ) """) } @@ -139,7 +139,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, name FROM products" @@ -151,7 +151,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, invalid_name, price, stock, active, createdAt FROM products" @@ -162,7 +162,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { assertDoesNotCompile(""" import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement case class WrongTypes(id: Int, name: Int) @@ -176,7 +176,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, nam, price, stock, active, createdAt FROM products" @@ -188,7 +188,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement val dynamicField = "name" TestElasticClientApi.searchAs[Product]( @@ -201,7 +201,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT * FROM products" @@ -214,7 +214,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT * FROM products WHERE active = true" diff --git a/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala b/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala index ecfaed86..a31d591d 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client.macros import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import org.json4s.{DefaultFormats, Formats} import scala.language.experimental.macros @@ -34,7 +34,7 @@ trait TestElasticClientApi { /** Search without compile-time validation (runtime). */ def searchAsUnchecked[T]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[T], formats: Formats): Seq[T] = { // Dummy implementation for tests Seq.empty[T] diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala index 8371e771..d4ff1eba 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala @@ -44,7 +44,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.searchAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery) + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery) )($m, $formats) """ } @@ -71,7 +71,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.searchAsyncAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery) + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery) )($m, $ec, $formats) """ } @@ -99,7 +99,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.scrollAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery), + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery), $config )($system, $m, $formats) """ diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala index 0afd31b1..7d1e445e 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.sql.macros import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.{COUNT, WindowFunction} import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.{MultiSearch, SingleSearch} import scala.language.experimental.macros import scala.reflect.macros.blackbox @@ -156,7 +156,7 @@ trait SQLQueryValidator { | scrollAs[Product](\"\"\"SELECT id, name FROM products\"\"\".stripMargin) | |❌ For dynamic queries, use: - | scrollAsUnchecked[Product](SQLQuery(dynamicSql), ScrollConfig()) + | scrollAsUnchecked[Product](SelectStatement(dynamicSql), ScrollConfig()) | |""".stripMargin ) @@ -170,12 +170,12 @@ trait SQLQueryValidator { // ============================================================ // Helper: Parse SQL query into SQLSearchRequest // ============================================================ - private def parseSQLQuery(c: blackbox.Context)(sqlQuery: String): SQLSearchRequest = { + private def parseSQLQuery(c: blackbox.Context)(sqlQuery: String): SingleSearch = { Parser(sqlQuery) match { - case Right(Left(request)) => + case Right(request: SingleSearch) => request - case Right(Right(multi)) => + case Right(multi: MultiSearch) => multi.requests.headOption.getOrElse { c.abort(c.enclosingPosition, "❌ Empty multi-search query") } @@ -193,7 +193,7 @@ trait SQLQueryValidator { // Reject SELECT * (incompatible with compile-time validation) // ============================================================ private def rejectSelectStar[T: c.WeakTypeTag](c: blackbox.Context)( - parsedQuery: SQLSearchRequest, + parsedQuery: SingleSearch, sqlQuery: String ): Unit = { import c.universe._ @@ -237,7 +237,7 @@ trait SQLQueryValidator { | SELECT $fieldNames FROM ... | | 2. Use the *Unchecked() variant for dynamic queries: - | searchAsUnchecked[${tpe.typeSymbol.name}](SQLQuery("SELECT * FROM ...")) + | searchAsUnchecked[${tpe.typeSymbol.name}](SelectStatement("SELECT * FROM ...")) | |Best Practice: | Always explicitly select only the fields you need. @@ -330,7 +330,7 @@ trait SQLQueryValidator { // ============================================================ // Helper: Extract selected fields from parsed SQL query // ============================================================ - private def extractQueryFields(parsedQuery: SQLSearchRequest): Set[String] = { + private def extractQueryFields(parsedQuery: SingleSearch): Set[String] = { parsedQuery.select.fields.map { field => field.fieldAlias.map(_.alias).getOrElse(field.identifier.name) }.toSet @@ -339,7 +339,7 @@ trait SQLQueryValidator { // ============================================================ // Helper: Extract UNNEST collections from the query // ============================================================ - private def extractUnnestedCollections(parsedQuery: SQLSearchRequest): Set[String] = { + private def extractUnnestedCollections(parsedQuery: SingleSearch): Set[String] = { // Check if the query has nested elements (UNNEST) parsedQuery.select.fields.flatMap { field => field.identifier.nestedElement.map { nested => @@ -630,7 +630,7 @@ trait SQLQueryValidator { // Helper: Validate Type compatibility // ============================================================ private def validateTypes(c: blackbox.Context)( - parsedQuery: SQLSearchRequest, + parsedQuery: SingleSearch, requiredFields: Map[String, c.universe.Type] ): Unit = { diff --git a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala index 533cfb84..a86a611e 100644 --- a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala +++ b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.persistence.query import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} import app.softnetwork.elastic.client.spi.ElasticClientFactory import app.softnetwork.elastic.client.{ElasticClientApi, ElasticClientDelegator} -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import mustache.Mustache import org.json4s.Formats import app.softnetwork.persistence._ @@ -188,7 +188,7 @@ trait ElasticProvider[T <: Timestamped] override def searchDocuments( query: String )(implicit m: Manifest[T], formats: Formats): List[T] = { - searchAsUnchecked[T](SQLQuery(query)) match { + searchAsUnchecked[T](SelectStatement(query)) match { case ElasticSuccess(results) => results.toList case ElasticFailure(elasticError) => logger.error(s"searchDocuments failed -> ${elasticError.message}") diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index 924270c2..726ff774 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{Criteria, SQLMultiSearchRequest, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{Criteria, MultiSearch, SingleSearch, Statement} import scala.util.matching.Regex @@ -27,16 +27,21 @@ object SQLImplicits { import scala.language.implicitConversions implicit def queryToSQLCriteria(query: String): Option[Criteria] = { - val maybeQuery: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = query + val maybeQuery: Option[Statement] = query maybeQuery match { - case Some(Left(l)) => l.where.flatMap(_.criteria) - case _ => None + case Some(statement) => + statement match { + case single: SingleSearch => + single.where.flatMap(_.criteria) + case _ => None + } + case _ => None } } - implicit def queryToSQLQuery( + implicit def queryToStatement( query: String - ): Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + ): Option[Statement] = { Parser(query) match { case Left(_) => None case Right(r) => Some(r) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 1e853ae5..89e2caa3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -16,14 +16,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.query.{ - Bucket, - BucketPath, - Field, - Limit, - OrderBy, - SQLSearchRequest -} +import app.softnetwork.elastic.sql.query.{Bucket, BucketPath, Field, Limit, OrderBy, SingleSearch} import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} package object aggregate { @@ -94,7 +87,7 @@ package object aggregate { override lazy val bucketPath: String = aggregations.map(_.bucketPath).distinct.sortBy(_.length).reverse.headOption.getOrElse("") - override def update(request: SQLSearchRequest): BucketScriptAggregation = { + override def update(request: SingleSearch): BucketScriptAggregation = { val identifiers = FunctionUtils.aggregateIdentifiers(identifier) val params = identifiers.flatMap { case identifier: Identifier => @@ -148,7 +141,7 @@ package object aggregate { def withFields(fields: Seq[Field]): WindowFunction - def update(request: SQLSearchRequest): WindowFunction = { + def update(request: SingleSearch): WindowFunction = { val updated = this .withPartitionBy(partitionBy = partitionBy.map(_.update(request))) updated.withFields( @@ -177,7 +170,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[FirstValue] .copy( @@ -197,7 +190,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[LastValue] .copy( @@ -217,7 +210,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[ArrayAgg] .copy( @@ -244,7 +237,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[CountAgg] .copy( @@ -268,7 +261,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[MinAgg] .copy( @@ -292,7 +285,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[MaxAgg] .copy( @@ -316,7 +309,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[AvgAgg] .copy( @@ -340,7 +333,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[SumAgg] .copy( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 6f450160..2e80bc1c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -29,7 +29,7 @@ import app.softnetwork.elastic.sql.{ Updateable } import app.softnetwork.elastic.sql.operator.Operator -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.SingleSearch package object geo { @@ -97,7 +97,7 @@ package object geo { with PainlessParams with Updateable { - override def update(request: SQLSearchRequest): Distance = this.copy( + override def update(request: SingleSearch): Distance = this.copy( from = from.fold(id => Left(id.update(request)), p => Right(p)), to = to.fold(id => Left(id.update(request)), p => Right(p)) ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 3667718b..4f6982fc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -20,7 +20,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.SingleSearch package object function { @@ -189,7 +189,7 @@ package object function { functions.indexOf(function) } - def updateFunctions(request: SQLSearchRequest): List[Function] = { + def updateFunctions(request: SingleSearch): List[Function] = { functions.map { case f: Updateable => f.update(request).asInstanceOf[Function] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index d58d26d7..70073314 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -248,7 +248,7 @@ package object sql { } trait Updateable extends Token { - def update(request: SQLSearchRequest): Updateable + def update(request: SingleSearch): Updateable } abstract class Expr(override val sql: String) extends Token @@ -301,6 +301,13 @@ package object sql { override def baseType: SQLType = SQLTypes.Null } + case object ParamValue extends Value[String](null) with TokenRegex { + override def sql: String = "?" + override def painless(context: Option[PainlessContext]): String = "params.paramValue" + override def nullable: Boolean = true + override def baseType: SQLType = SQLTypes.Any + } + case class BooleanValue(override val value: Boolean) extends Value[Boolean](value) { override def sql: String = value.toString override def baseType: SQLType = SQLTypes.Boolean @@ -606,7 +613,7 @@ package object sql { trait Source extends Updateable { def name: String - def update(request: SQLSearchRequest): Source + def update(request: SingleSearch): Source } sealed trait Identifier @@ -620,7 +627,7 @@ package object sql { def withFunctions(functions: List[Function]): Identifier - def update(request: SQLSearchRequest): Identifier + def update(request: SingleSearch): Identifier def tableAlias: Option[String] def distinct: Boolean @@ -854,7 +861,7 @@ package object sql { id } - def update(request: SQLSearchRequest): Identifier = { + def update(request: SingleSearch): Identifier = { val bucketPath: String = request.groupBy match { case Some(gb) => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala new file mode 100644 index 00000000..18363953 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala @@ -0,0 +1,5 @@ +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.sql.query.{Delete, Insert, Update} + +trait DmlParser { self: Parser with WhereParser => } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index ec798f2a..a51821ba 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -29,9 +29,11 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.Column import scala.language.implicitConversions import scala.language.existentials +import scala.util.matching.Regex import scala.util.parsing.combinator.{PackratParsers, RegexParsers} import scala.util.parsing.input.CharSequenceReader @@ -49,25 +51,199 @@ object Parser with OrderByParser with LimitParser { - def request: PackratParser[SQLSearchRequest] = { + def single: PackratParser[SingleSearch] = { phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.?) ^^ { case s ~ f ~ w ~ g ~ h ~ o ~ l => - val request = SQLSearchRequest(s, f, w, g, h, o, l).update() - request.validate() match { - case Left(error) => throw ValidationError(error) - case _ => - } - request + SingleSearch(s, f, w, g, h, o, l).update() } } def union: PackratParser[UNION.type] = UNION.regex ^^ (_ => UNION) - def requests: PackratParser[List[SQLSearchRequest]] = rep1sep(request, union) ^^ (s => s) + def dqlStatement: PackratParser[DqlStatement] = rep1sep(single, union) ^^ { + case x :: Nil => x + case s => MultiSearch(s) + } + + def ident: Parser[String] = """[a-zA-Z_][a-zA-Z0-9_]*""".r + + def option: PackratParser[(String, Value[_])] = + ident ~ "=" ~ value ^^ { case key ~ _ ~ value => + (key, value) + } + + def options: PackratParser[Map[String, Value[_]]] = + "OPTIONS" ~ "(" ~ repsep(option, separator) ~ ")" ^^ { case _ ~ _ ~ opts ~ _ => + opts.toMap + } + + def multiFields: PackratParser[List[Column]] = + "FIELDS" ~ "(" ~ repsep(column, separator) ~ ")" ^^ { case _ ~ _ ~ cols ~ _ => + cols + } | success(Nil) + + def ifExists: PackratParser[Boolean] = + opt("IF" ~ "EXISTS") ^^ { + case Some(_) => true + case None => false + } + + def ifNotExists: PackratParser[Boolean] = + opt("IF" ~ "NOT" ~ "EXISTS") ^^ { + case Some(_) => true + case None => false + } + + def notNull: PackratParser[Boolean] = + opt("NOT" ~ "NULL") ^^ { + case Some(_) => true + case None => false + } + + def defaultVal: PackratParser[Option[Value[_]]] = + opt("DEFAULT" ~ value) ^^ { + case Some(_ ~ v) => Some(v) + case None => None + } + + def column: PackratParser[Column] = + ident ~ sql_type ~ (options | success( + Map.empty[String, Value[_]] + )) ~ notNull ~ defaultVal ~ multiFields ^^ { case name ~ dt ~ opts ~ nn ~ dv ~ mfs => + Column(name, dt, opts, mfs, nn, dv) + } + + def columns: PackratParser[List[Column]] = + "(" ~ repsep(column, separator) ~ ")" ^^ { case _ ~ cols ~ _ => cols } + + def createOrReplaceTable: PackratParser[CreateTable] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columns | ("AS" ~> dqlStatement)) ^^ { + case _ ~ name ~ lr => + lr match { + case cols: List[Column] => + CreateTable(name, Right(cols), ifNotExists = false, orReplace = true) + case sel: DqlStatement => + CreateTable(name, Left(sel), ifNotExists = false, orReplace = true) + } + } + + def createTable: PackratParser[CreateTable] = + ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columns | ("AS" ~> dqlStatement)) ^^ { + case _ ~ ine ~ name ~ lr => + lr match { + case cols: List[Column] => CreateTable(name, Right(cols), ine) + case sel: DqlStatement => CreateTable(name, Left(sel), ine) + } + } + + def dropTable: PackratParser[DropTable] = + ("DROP" ~ "TABLE") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropTable(name, ifExists = ie) + } + + def truncateTable: PackratParser[TruncateTable] = + ("TRUNCATE" ~ "TABLE") ~ ident ^^ { case _ ~ name => + TruncateTable(name) + } + + def addColumn: PackratParser[AddColumn] = + ("ADD" ~ "COLUMN") ~ ifNotExists ~ column ^^ { case _ ~ ine ~ col => + AddColumn(col, ifNotExists = ine) + } + + def dropColumn: PackratParser[DropColumn] = + ("DROP" ~ "COLUMN") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropColumn(name, ifExists = ie) + } + + def renameColumn: PackratParser[RenameColumn] = + ("RENAME" ~ "COLUMN") ~ ident ~ ("TO" ~> ident) ^^ { case _ ~ oldName ~ newName => + RenameColumn(oldName, newName) + } + + def alterColumnIfExists: PackratParser[Boolean] = + ("ALTER" ~ "COLUMN") ~ ifExists ^^ { case _ ~ ie => + ie + } + + def setColumnOptions: PackratParser[AlterColumnOptions] = + alterColumnIfExists ~ ident ~ "SET" ~ options ^^ { case ie ~ col ~ _ ~ opts => + AlterColumnOptions(col, opts, ifExists = ie) + } + + def setColumnType: PackratParser[AlterColumnType] = + alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ sql_type ^^ { + case ie ~ name ~ _ ~ newType => AlterColumnType(name, newType, ifExists = ie) + } + + def setColumnDefault: PackratParser[AlterColumnDefault] = + alterColumnIfExists ~ ident ~ ("SET" ~ "DEFAULT") ~ value ^^ { case ie ~ name ~ _ ~ dv => + AlterColumnDefault(name, dv, ifExists = ie) + } + + def dropColumnDefault: PackratParser[DropColumnDefault] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "DEFAULT") ^^ { case ie ~ name ~ _ => + DropColumnDefault(name, ifExists = ie) + } + + def setColumnNotNull: PackratParser[AlterColumnNotNull] = + alterColumnIfExists ~ ident ~ ("SET" ~ "NOT" ~ "NULL") ^^ { case ie ~ name ~ _ => + AlterColumnNotNull(name, ifExists = ie) + } + + def dropColumnNotNull: PackratParser[DropColumnNotNull] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "NOT" ~ "NULL") ^^ { case ie ~ name ~ _ => + DropColumnNotNull(name, ifExists = ie) + } + + def alterTableStatement: PackratParser[AlterTableStatement] = + addColumn | + dropColumn | + renameColumn | + setColumnOptions | + setColumnType | + setColumnDefault | + dropColumnDefault | + setColumnNotNull | + dropColumnNotNull + + def alterTable: PackratParser[AlterTable] = + ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ alterTableStatement ^^ { case _ ~ ie ~ table ~ stmt => + AlterTable(table, ie, stmt) + } + + def ddlStatement: PackratParser[DdlStatement] = + createTable | createOrReplaceTable | alterTable | dropTable | truncateTable + + /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ + def insert: PackratParser[Insert] = + ("INSERT" ~ "INTO") ~ ident ~ opt("(" ~> repsep(ident, separator) <~ ")") ~ + (("VALUES" ~ "(" ~> repsep(value, separator) <~ ")") ^^ { vs => Right(vs) } + | dqlStatement ^^ { q => Left(q) }) ^^ { case _ ~ table ~ colsOpt ~ vals => + Insert(table, colsOpt.getOrElse(Nil), vals) + } + + /** UPDATE table SET col1 = v1, col2 = v2 [WHERE ...] */ + def update: PackratParser[Update] = + ("UPDATE" ~> ident) ~ ("SET" ~> repsep(ident ~ "=" ~ value, separator)) ~ where.? ^^ { + case table ~ assigns ~ w => + val values = assigns.map { case col ~ _ ~ v => col -> v }.toMap + Update(table, values, w) + } + + /** DELETE FROM table [WHERE ...] */ + def delete: PackratParser[Delete] = + ("DELETE" ~ "FROM") ~> ident ~ where.? ^^ { case table ~ w => + Delete(Table(table), w) + } + + def dmlStatement: PackratParser[DmlStatement] = insert | update | delete + + def statement: PackratParser[Statement] = ddlStatement | dqlStatement | dmlStatement def apply( query: String - ): Either[ParserError, Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + ): Either[ParserError, Statement] = { val normalizedQuery = query .split("\n") @@ -75,14 +251,14 @@ object Parser .filterNot(w => w.isEmpty || w.startsWith("--")) .mkString(" ") val reader = new PackratReader(new CharSequenceReader(normalizedQuery)) - parse(requests, reader) match { + parse(statement, reader) match { case NoSuccess(msg, _) => Console.err.println(msg) Left(ParserError(msg)) case Success(result, _) => - result match { - case x :: Nil => Right(Left(x)) - case _ => Right(Right(SQLMultiSearchRequest(result))) + result.validate() match { + case Left(error) => Left(ParserError(error)) + case _ => Right(result) } } } @@ -134,6 +310,14 @@ trait Parser private val reservedKeywords = Seq( "select", + "insert", + "update", + "delete", + "create", + "alter", + "drop", + "truncate", + "column", "from", "join", "where", @@ -261,7 +445,7 @@ trait Parser private val identifierRegexStr = s"""(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[\\*a-zA-Z_\\-][a-zA-Z0-9_\\-.\\[\\]\\*]*""" - val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex + val identifierRegex: Regex = identifierRegexStr.r // scala.util.matching.Regex def identifier: PackratParser[Identifier] = (Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 228673ea..fac388a8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -21,6 +21,7 @@ import app.softnetwork.elastic.sql.{ DoubleValue, Identifier, LongValue, + ParamValue, PiValue, StringValue, Value @@ -48,8 +49,11 @@ package object `type` { def boolean: PackratParser[BooleanValue] = """(?i)(true|false)\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) + def param: PackratParser[ParamValue.type] = + "?" ^^ (_ => ParamValue) + def value: PackratParser[Value[_]] = - literal | pi | double | long | boolean + literal | pi | double | long | boolean | param def identifierWithValue: Parser[Identifier] = (value ^^ functionAsIdentifier) >> cast @@ -87,8 +91,13 @@ package object `type` { def float_type: PackratParser[SQLTypes.Real.type] = "(?i)float|real".r ^^ (_ => SQLTypes.Real) + def array_type: PackratParser[SQLTypes.Array] = + "(?i)array<".r ~> sql_type <~ ">" ^^ { elementType => + SQLTypes.Array(elementType) + } + def sql_type: PackratParser[SQLType] = - char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type + char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type | array_type } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 94edf995..a953e211 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -46,7 +46,7 @@ case object On extends Expr("ON") with TokenRegex case class On(criteria: Criteria) extends Updateable { override def sql: String = s" $On $criteria" - def update(request: SQLSearchRequest): On = this.copy(criteria = criteria.update(request)) + def update(request: SingleSearch): On = this.copy(criteria = criteria.update(request)) } case object Join extends Expr("JOIN") with TokenRegex @@ -59,7 +59,7 @@ sealed trait Join extends Updateable { override def sql: String = s" ${asString(joinType)} $Join $source${asString(on)}${asString(alias)}" - override def update(request: SQLSearchRequest): Join + override def update(request: SingleSearch): Join override def validate(): Either[String, Unit] = for { @@ -90,7 +90,7 @@ case class Unnest( ) extends Source with Join { override def sql: String = s"$Join $Unnest($identifier)${asString(alias)}" - def update(request: SQLSearchRequest): Unnest = { + def update(request: SingleSearch): Unnest = { val updated = this.copy( identifier = identifier.withNested(true).update(request), limit = limit.orElse(request.limit) @@ -141,7 +141,8 @@ case class Unnest( case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join] = Nil) extends Source { override def sql: String = s"$name${asString(tableAlias)} ${joins.map(_.sql).mkString(" ")}".trim - def update(request: SQLSearchRequest): Table = this.copy(joins = joins.map(_.update(request))) + def update(request: SingleSearch): Table = + this.copy(joins = joins.map(_.update(request))) override def validate(): Either[String, Unit] = for { @@ -179,7 +180,7 @@ case class From(tables: Seq[Table]) extends Updateable { (u.alias.map(_.alias).getOrElse(u.name), (u.name, u.limit)) ) .toMap - def update(request: SQLSearchRequest): From = + def update(request: SingleSearch): From = this.copy(tables = tables.map(_.update(request))) override def validate(): Either[String, Unit] = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 8eb01687..c02aace2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -32,7 +32,7 @@ case object GroupBy extends Expr("GROUP BY") with TokenRegex case class GroupBy(buckets: Seq[Bucket]) extends Updateable { override def sql: String = s" $GroupBy ${buckets.mkString(", ")}" - def update(request: SQLSearchRequest): GroupBy = + def update(request: SingleSearch): GroupBy = this.copy(buckets = buckets.map(_.update(request))) lazy val bucketNames: Map[String, Bucket] = buckets.map { b => b.identifier.identifierName -> b @@ -56,7 +56,7 @@ case class Bucket( ) extends Updateable with PainlessScript { override def sql: String = s"$identifier" - def update(request: SQLSearchRequest): Bucket = { + def update(request: SingleSearch): Bucket = { identifier.functions.headOption match { case Some(func: LongValue) => if (func.value <= 0) { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 07ad6625..243116f7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -25,7 +25,7 @@ case class Having(criteria: Option[Criteria]) extends Updateable { case Some(c) => s" $Having $c" case _ => "" } - def update(request: SQLSearchRequest): Having = + def update(request: SingleSearch): Having = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria.map(_.validate()).getOrElse(Right(())) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala index 952a08ad..3f3212dd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala @@ -36,7 +36,7 @@ case class FieldSort( lazy val direction: SortOrder = order.getOrElse(Asc) lazy val name: String = field.identifierName override def sql: String = s"$name $direction" - override def update(request: SQLSearchRequest): FieldSort = this.copy( + override def update(request: SingleSearch): FieldSort = this.copy( field = field.update(request) ) def isScriptSort: Boolean = functions.nonEmpty && !hasAggregation && field.fieldAlias.isEmpty @@ -59,5 +59,6 @@ case class OrderBy(sorts: Seq[FieldSort]) extends Updateable { } } yield () - def update(request: SQLSearchRequest): OrderBy = this.copy(sorts = sorts.map(_.update(request))) + def update(request: SingleSearch): OrderBy = + this.copy(sorts = sorts.map(_.update(request))) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala deleted file mode 100644 index 459472e1..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.sql.query - -import app.softnetwork.elastic.sql.Token - -case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends Token { - override def sql: String = s"${requests.map(_.sql).mkString(" UNION ALL ")}" - - def update(): SQLMultiSearchRequest = this.copy(requests = requests.map(_.update())) - - override def validate(): Either[String, Unit] = { - requests.map(_.validate()).filter(_.isLeft) match { - case Nil => Right(()) - case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) - } - } - - lazy val sqlAggregations: Map[String, SQLAggregation] = - requests.flatMap(_.sqlAggregations).distinct.toMap - - lazy val fieldAliases: Map[String, String] = - requests.flatMap(_.fieldAliases).distinct.toMap -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala deleted file mode 100644 index 29c1bb9c..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.sql.query - -import app.softnetwork.elastic.sql.SQL - -/** SQL Query wrapper - * @param query - * - the SQL query - * @param score - * - optional minimum score for the elasticsearch query - */ -case class SQLQuery(query: SQL, score: Option[Double] = None) { - import app.softnetwork.elastic.sql.SQLImplicits._ - lazy val request: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { - query - } - - def minScore(score: Double): SQLQuery = this.copy(score = Some(score)) -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala deleted file mode 100644 index 069aa5eb..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.sql.query - -import app.softnetwork.elastic.sql.function.aggregate.WindowFunction -import app.softnetwork.elastic.sql.{asString, Token} - -case class SQLSearchRequest( - select: Select = Select(), - from: From, - where: Option[Where], - groupBy: Option[GroupBy] = None, - having: Option[Having] = None, - orderBy: Option[OrderBy] = None, - limit: Option[Limit] = None, - score: Option[Double] = None -) extends Token { - override def sql: String = - s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" - - lazy val fieldAliases: Map[String, String] = select.fieldAliases - lazy val tableAliases: Map[String, String] = from.tableAliases - lazy val unnestAliases: Map[String, (String, Option[Limit])] = from.unnestAliases - lazy val bucketNames: Map[String, Bucket] = buckets.flatMap { b => - val name = b.identifier.identifierName - "\\d+".r.findFirstIn(name) match { - case Some(n) if name.trim.split(" ").length == 1 => - val identifier = select.fields(n.toInt - 1).identifier - val updated = b.copy(identifier = select.fields(n.toInt - 1).identifier) - Map( - n -> updated, // also map numeric bucket to field name - identifier.identifierName -> updated - ) - case _ => Map(name -> b) - } - }.toMap - - var unnests: scala.collection.mutable.Map[String, Unnest] = { - val map = from.unnests.map(u => u.alias.map(_.alias).getOrElse(u.name) -> u).toMap - scala.collection.mutable.Map(map.toSeq: _*) - } - - lazy val nestedFields: Map[String, Seq[Field]] = - select.fields - .filterNot(_.isAggregation) - .filter(_.nested) - .groupBy(_.identifier.innerHitsName.getOrElse("")) - lazy val nested: Seq[NestedElement] = - from.unnests.map(toNestedElement).groupBy(_.path).map(_._2.head).toList - private[this] lazy val nestedFieldsWithoutCriteria: Map[String, Seq[Field]] = { - val innerHitsWithCriteria = (where.map(_.nestedElements).getOrElse(Seq.empty) ++ - having.map(_.nestedElements).getOrElse(Seq.empty) ++ - groupBy.map(_.nestedElements).getOrElse(Seq.empty)) - .groupBy(_.path) - .map(_._2.head) - .toList - .map(_.innerHitsName) - val ret = nestedFields.filterNot { case (innerHitsName, _) => - innerHitsWithCriteria.contains(innerHitsName) - } - ret - } - // nested fields that are not part of where, having or group by clauses - lazy val nestedElementsWithoutCriteria: Seq[NestedElement] = - nested.filter(n => nestedFieldsWithoutCriteria.keys.toSeq.contains(n.innerHitsName)) - - def toNestedElement(u: Unnest): NestedElement = { - val updated = unnests.getOrElse(u.alias.map(_.alias).getOrElse(u.name), u) - val parent = updated.parent.map(toNestedElement) - NestedElement( - path = updated.path, - innerHitsName = updated.innerHitsName, - size = limit.map(_.limit), - children = Nil, - sources = nestedFields - .get(updated.innerHitsName) - .map(_.map(_.identifier.name.split('.').tail.mkString("."))) - .getOrElse(Nil), - parent = parent - ) - } - - lazy val sorts: Map[String, SortOrder] = - orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap - - def update(): SQLSearchRequest = { - (for { - from <- Option(this.copy(from = from.update(this))) - select <- Option( - from.copy( - select = select.update(from), - groupBy = groupBy.map(_.update(from)), - having = having.map(_.update(from)) - ) - ) - where <- Option(select.copy(where = where.map(_.update(select)))) - updated <- Option(where.copy(orderBy = orderBy.map(_.update(where)))) - } yield updated).getOrElse( - throw new IllegalStateException("Failed to update SQLSearchRequest") - ) - } - - lazy val scriptFields: Seq[Field] = { - if (aggregates.nonEmpty) - Seq.empty - else - select.fields.filter(_.isScriptField) - } - - lazy val fields: Seq[String] = { - if (groupBy.isEmpty && !windowFunctions.exists(_.isWindowing)) - select.fields - .filterNot(_.isScriptField) - .filterNot(_.nested) - .filterNot(_.isAggregation) - .map(_.sourceField) - .filterNot(f => excludes.contains(f)) - .distinct - else - Seq.empty - } - - lazy val windowFields: Seq[Field] = select.fields.filter(_.identifier.hasWindow) - - lazy val windowFunctions: Seq[WindowFunction] = windowFields.flatMap(_.identifier.windows) - - lazy val aggregates: Seq[Field] = - select.fields - .filter(f => f.isAggregation || f.isBucketScript) - .filterNot(_.identifier.hasWindow) ++ windowFields - - lazy val sqlAggregations: Map[String, SQLAggregation] = - aggregates.flatMap(f => SQLAggregation.fromField(f, this)).map(a => a.aggName -> a).toMap - - lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) - - lazy val sources: Seq[String] = from.tables.map(_.name) - - lazy val bucketTree: BucketTree = BucketTree.fromBuckets( - Seq(groupBy.map(_.buckets).getOrElse(Seq.empty)) ++ windowFunctions.map( - _.buckets - ) - ) - - lazy val buckets: Seq[Bucket] = bucketTree.allBuckets.flatten - - override def validate(): Either[String, Unit] = { - for { - _ <- from.validate() - _ <- select.validate() - _ <- where.map(_.validate()).getOrElse(Right(())) - _ <- groupBy.map(_.validate()).getOrElse(Right(())) - _ <- having.map(_.validate()).getOrElse(Right(())) - _ <- orderBy.map(_.validate()).getOrElse(Right(())) - _ <- limit.map(_.validate()).getOrElse(Right(())) - /*_ <- { - // validate that having clauses are only applied when group by is present - if (having.isDefined && groupBy.isEmpty) { - Left("HAVING clauses can only be applied when GROUP BY is present") - } else { - Right(()) - } - }*/ - _ <- { - // validate that non-aggregated fields are not present when group by is present - if (groupBy.isDefined) { - val nonAggregatedFields = - select.fields.filterNot(f => f.hasAggregation) - val invalidFields = nonAggregatedFields.filterNot(f => - buckets.exists(b => - b.name == f.fieldAlias.map(_.alias).getOrElse(f.sourceField.replace(".", "_")) - ) - ) - if (invalidFields.nonEmpty) { - Left( - s"Non-aggregated fields ${invalidFields.map(_.sql).mkString(", ")} cannot be selected when GROUP BY is present" - ) - } else { - Right(()) - } - } else { - Right(()) - } - } - } yield () - } - -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index fa3cbd26..14d03d57 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -70,7 +70,7 @@ case class Field( override def functions: List[Function] = identifier.functions - def update(request: SQLSearchRequest): Field = { + def update(request: SingleSearch): Field = { identifier.windows match { case Some(th) => val windowFunction = th.update(request) @@ -104,7 +104,7 @@ case object Except extends Expr("except") with TokenRegex case class Except(fields: Seq[Field]) extends Updateable { override def sql: String = s" $Except(${fields.mkString(",")})" - def update(request: SQLSearchRequest): Except = + def update(request: SingleSearch): Except = this.copy(fields = fields.map(_.update(request))) } @@ -117,7 +117,7 @@ case class Select( lazy val fieldAliases: Map[String, String] = fields.flatMap { field => field.fieldAlias.map(a => field.identifier.identifierName -> a.alias) }.toMap - def update(request: SQLSearchRequest): Select = + def update(request: SingleSearch): Select = this.copy(fields = fields.map(_.update(request)), except = except.map(_.update(request))) override def validate(): Either[String, Unit] = @@ -156,7 +156,7 @@ case class SQLAggregation( } object SQLAggregation { - def fromField(field: Field, request: SQLSearchRequest): Option[SQLAggregation] = { + def fromField(field: Field, request: SingleSearch): Option[SQLAggregation] = { import field._ val aggType = aggregateFunction match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index a983dfa6..2dcd82f9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -101,7 +101,7 @@ sealed trait Criteria extends Updateable with PainlessScript { def limit: Option[Limit] = None - def update(request: SQLSearchRequest): Criteria + def update(request: SingleSearch): Criteria def group: Boolean @@ -155,7 +155,7 @@ case class Predicate( else leftCriteria} $operator${not .map(_ => " NOT") .getOrElse("")} ${if (group) s"$rightCriteria)" else rightCriteria}" - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updatedPredicate = this.copy( leftCriteria = leftCriteria.update(request), rightCriteria = rightCriteria.update(request) @@ -527,7 +527,7 @@ case class GenericExpression( ) extends Expression { override def maybeValue: Option[Token] = Option(value) - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = value match { case id: Identifier => @@ -550,7 +550,7 @@ case class IsNullExpr(identifier: Identifier) extends Expression { override def maybeNot: Option[NOT.type] = None - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -568,7 +568,7 @@ case class IsNotNullExpr(identifier: Identifier) extends Expression { override def maybeNot: Option[NOT.type] = None - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -599,7 +599,7 @@ object ConditionalFunctionAsCriteria { case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { override val conditionalFunction: ConditionalFunction[SQLAny] = IsNull(identifier) override val operator: Operator = IS_NULL - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -629,7 +629,7 @@ case class IsNotNullCriteria(identifier: Identifier) identifier ) override val operator: Operator = IS_NOT_NULL - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -666,7 +666,7 @@ case class InExpr[R, +T <: Value[R]]( override def sql = s"$id $notAsString$operator $values" override def operator: Operator = IN - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -706,7 +706,7 @@ case class BetweenExpr( ) extends Expression { override def sql = s"$identifier $notAsString$operator $fromTo" override def operator: Operator = BETWEEN - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -757,7 +757,7 @@ case class DistanceCriteria( override def sql = s"$distance $operator $geoDistance" - override def update(request: SQLSearchRequest): DistanceCriteria = + override def update(request: SingleSearch): DistanceCriteria = this.copy(distance = distance.update(request)) override def maybeValue: Option[Token] = Some(geoDistance) @@ -775,7 +775,7 @@ case class MultiMatchCriteria( override def sql: String = s"$operator (${identifiers.mkString(",")}) $AGAINST ($value)" override def operator: Operator = MATCH - override def update(request: SQLSearchRequest): Criteria = + override def update(request: SingleSearch): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) override lazy val nested: Boolean = identifiers.forall(_.nested) @@ -816,7 +816,7 @@ case class MatchCriteria( override def sql: String = s"$operator($identifier,$value${options.map(o => s""","$o"""").getOrElse("")})" override def operator: Operator = MATCH - override def update(request: SQLSearchRequest): Criteria = + override def update(request: SingleSearch): Criteria = this.copy(identifier = identifier.update(request)) override def maybeValue: Option[Token] = Some(value) @@ -882,7 +882,7 @@ case class ElasticNested( def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticNested = + override def update(request: SingleSearch): ElasticNested = this.copy(criteria = criteria.update(request)) override def nested: Boolean = true @@ -902,7 +902,7 @@ case class ElasticChild( override val criteria: Criteria ) extends ElasticRelation(criteria, Child) { def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticChild = + override def update(request: SingleSearch): ElasticChild = this.copy(criteria = criteria.update(request)) } @@ -910,7 +910,7 @@ case class ElasticParent( override val criteria: Criteria ) extends ElasticRelation(criteria, Parent) { def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticParent = + override def update(request: SingleSearch): ElasticParent = this.copy(criteria = criteria.update(request)) } @@ -919,7 +919,7 @@ case class Where(criteria: Option[Criteria]) extends Updateable { case Some(c) => s" $Where $c" case _ => "" } - def update(request: SQLSearchRequest): Where = + def update(request: SingleSearch): Where = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala new file mode 100644 index 00000000..75ef6a51 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -0,0 +1,427 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.schema.Column +import app.softnetwork.elastic.sql.function.aggregate.WindowFunction + +package object query { + sealed trait Statement extends Token + + trait DqlStatement extends Statement + + /** Select Statement wrapper + * @param query + * - the SQL query + * @param score + * - optional minimum score for the elasticsearch query + */ + case class SelectStatement(query: SQL, score: Option[Double] = None) extends DqlStatement { + import app.softnetwork.elastic.sql.SQLImplicits._ + + lazy val statement: Option[DqlStatement] = { + queryToStatement(query) match { + case Some(s: DqlStatement) => Some(s) + case _ => None + } + } + + override def sql: SQL = + statement match { + case Some(value) => value.sql + case None => query + } + + def minScore(score: Double): SelectStatement = this.copy(score = Some(score)) + } + + case class SingleSearch( + select: Select = Select(), + from: From, + where: Option[Where], + groupBy: Option[GroupBy] = None, + having: Option[Having] = None, + orderBy: Option[OrderBy] = None, + limit: Option[Limit] = None, + score: Option[Double] = None + ) extends DqlStatement { + override def sql: String = + s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" + + lazy val fieldAliases: Map[String, String] = select.fieldAliases + lazy val tableAliases: Map[String, String] = from.tableAliases + lazy val unnestAliases: Map[String, (String, Option[Limit])] = from.unnestAliases + lazy val bucketNames: Map[String, Bucket] = buckets.flatMap { b => + val name = b.identifier.identifierName + "\\d+".r.findFirstIn(name) match { + case Some(n) if name.trim.split(" ").length == 1 => + val identifier = select.fields(n.toInt - 1).identifier + val updated = b.copy(identifier = select.fields(n.toInt - 1).identifier) + Map( + n -> updated, // also map numeric bucket to field name + identifier.identifierName -> updated + ) + case _ => Map(name -> b) + } + }.toMap + + var unnests: scala.collection.mutable.Map[String, Unnest] = { + val map = from.unnests.map(u => u.alias.map(_.alias).getOrElse(u.name) -> u).toMap + scala.collection.mutable.Map(map.toSeq: _*) + } + + lazy val nestedFields: Map[String, Seq[Field]] = + select.fields + .filterNot(_.isAggregation) + .filter(_.nested) + .groupBy(_.identifier.innerHitsName.getOrElse("")) + lazy val nested: Seq[NestedElement] = + from.unnests.map(toNestedElement).groupBy(_.path).map(_._2.head).toList + private[this] lazy val nestedFieldsWithoutCriteria: Map[String, Seq[Field]] = { + val innerHitsWithCriteria = (where.map(_.nestedElements).getOrElse(Seq.empty) ++ + having.map(_.nestedElements).getOrElse(Seq.empty) ++ + groupBy.map(_.nestedElements).getOrElse(Seq.empty)) + .groupBy(_.path) + .map(_._2.head) + .toList + .map(_.innerHitsName) + val ret = nestedFields.filterNot { case (innerHitsName, _) => + innerHitsWithCriteria.contains(innerHitsName) + } + ret + } + // nested fields that are not part of where, having or group by clauses + lazy val nestedElementsWithoutCriteria: Seq[NestedElement] = + nested.filter(n => nestedFieldsWithoutCriteria.keys.toSeq.contains(n.innerHitsName)) + + def toNestedElement(u: Unnest): NestedElement = { + val updated = unnests.getOrElse(u.alias.map(_.alias).getOrElse(u.name), u) + val parent = updated.parent.map(toNestedElement) + NestedElement( + path = updated.path, + innerHitsName = updated.innerHitsName, + size = limit.map(_.limit), + children = Nil, + sources = nestedFields + .get(updated.innerHitsName) + .map(_.map(_.identifier.name.split('.').tail.mkString("."))) + .getOrElse(Nil), + parent = parent + ) + } + + lazy val sorts: Map[String, SortOrder] = + orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap + + def update(): SingleSearch = { + (for { + from <- Option(this.copy(from = from.update(this))) + select <- Option( + from.copy( + select = select.update(from), + groupBy = groupBy.map(_.update(from)), + having = having.map(_.update(from)) + ) + ) + where <- Option(select.copy(where = where.map(_.update(select)))) + updated <- Option(where.copy(orderBy = orderBy.map(_.update(where)))) + } yield updated).getOrElse( + throw new IllegalStateException("Failed to update SQLSearchRequest") + ) + } + + lazy val scriptFields: Seq[Field] = { + if (aggregates.nonEmpty) + Seq.empty + else + select.fields.filter(_.isScriptField) + } + + lazy val fields: Seq[String] = { + if (groupBy.isEmpty && !windowFunctions.exists(_.isWindowing)) + select.fields + .filterNot(_.isScriptField) + .filterNot(_.nested) + .filterNot(_.isAggregation) + .map(_.sourceField) + .filterNot(f => excludes.contains(f)) + .distinct + else + Seq.empty + } + + lazy val windowFields: Seq[Field] = select.fields.filter(_.identifier.hasWindow) + + lazy val windowFunctions: Seq[WindowFunction] = windowFields.flatMap(_.identifier.windows) + + lazy val aggregates: Seq[Field] = + select.fields + .filter(f => f.isAggregation || f.isBucketScript) + .filterNot(_.identifier.hasWindow) ++ windowFields + + lazy val sqlAggregations: Map[String, SQLAggregation] = + aggregates.flatMap(f => SQLAggregation.fromField(f, this)).map(a => a.aggName -> a).toMap + + lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) + + lazy val sources: Seq[String] = from.tables.map(_.name) + + lazy val bucketTree: BucketTree = BucketTree.fromBuckets( + Seq(groupBy.map(_.buckets).getOrElse(Seq.empty)) ++ windowFunctions.map( + _.buckets + ) + ) + + lazy val buckets: Seq[Bucket] = bucketTree.allBuckets.flatten + + override def validate(): Either[String, Unit] = { + for { + _ <- from.validate() + _ <- select.validate() + _ <- where.map(_.validate()).getOrElse(Right(())) + _ <- groupBy.map(_.validate()).getOrElse(Right(())) + _ <- having.map(_.validate()).getOrElse(Right(())) + _ <- orderBy.map(_.validate()).getOrElse(Right(())) + _ <- limit.map(_.validate()).getOrElse(Right(())) + /*_ <- { + // validate that having clauses are only applied when group by is present + if (having.isDefined && groupBy.isEmpty) { + Left("HAVING clauses can only be applied when GROUP BY is present") + } else { + Right(()) + } + }*/ + _ <- { + // validate that non-aggregated fields are not present when group by is present + if (groupBy.isDefined) { + val nonAggregatedFields = + select.fields.filterNot(f => f.hasAggregation) + val invalidFields = nonAggregatedFields.filterNot(f => + buckets.exists(b => + b.name == f.fieldAlias.map(_.alias).getOrElse(f.sourceField.replace(".", "_")) + ) + ) + if (invalidFields.nonEmpty) { + Left( + s"Non-aggregated fields ${invalidFields.map(_.sql).mkString(", ")} cannot be selected when GROUP BY is present" + ) + } else { + Right(()) + } + } else { + Right(()) + } + } + } yield () + } + + } + + case class MultiSearch(requests: Seq[SingleSearch]) extends DqlStatement { + override def sql: String = s"${requests.map(_.sql).mkString(" UNION ALL ")}" + + def update(): MultiSearch = this.copy(requests = requests.map(_.update())) + + override def validate(): Either[String, Unit] = { + requests.map(_.validate()).filter(_.isLeft) match { + case Nil => Right(()) // TODO validate that all requests have the same fields + case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) + } + } + + lazy val sqlAggregations: Map[String, SQLAggregation] = + requests.flatMap(_.sqlAggregations).distinct.toMap + + lazy val fieldAliases: Map[String, String] = + requests.flatMap(_.fieldAliases).distinct.toMap + } + + sealed trait DmlStatement extends Statement + + case class Insert( + table: String, + cols: Seq[String], + values: Either[DqlStatement, Seq[Value[_]]] + ) extends DmlStatement { + override def sql: String = { + values match { + case Left(query) if cols.isEmpty => + s"INSERT INTO $table ${query.sql}" + case Left(query) => + s"INSERT INTO $table (${cols.mkString(",")}) ${query.sql}" + case Right(vs) => + val valuesSql = vs + .map { + case v if v.isInstanceOf[StringValue] => s"'${v.value}'" + case v => s"${v.value}" + } + .mkString(", ") + s"INSERT INTO $table ${cols.mkString(",")} VALUES ($valuesSql)" + } + } + + override def validate(): Either[String, Unit] = { + values match { + case Right(vs) if cols.size != vs.size => + Left(s"Number of columns (${cols.size}) does not match number of values (${vs.size})") + case _ => + Right(()) + } + } + } + + case class Update(table: String, values: Map[String, Value[_]], where: Option[Where]) + extends DmlStatement { + override def sql: String = s"UPDATE $table SET ${values + .map { case (k, v) => s"$k = ${v.value}" } + .mkString(", ")}${where.map(w => s" ${w.sql}").getOrElse("")}" + } + + case class Delete(table: Table, where: Option[Where]) extends DmlStatement { + override def sql: String = + s"DELETE FROM ${table.name}${where.map(w => s" ${w.sql}").getOrElse("")}" + } + + sealed trait DdlStatement extends Statement + + case class CreateTable( + table: String, + ddl: Either[DqlStatement, List[Column]], + ifNotExists: Boolean = false, + orReplace: Boolean = false + ) extends DdlStatement { + + override def sql: String = { + val ineClause = if (ifNotExists) " IF NOT EXISTS" else "" + val replaceClause = if (orReplace) " OR REPLACE" else "" + ddl match { + case Left(select) => + s"CREATE$replaceClause TABLE$ineClause $table AS ${select.sql}" + case Right(columns) => + val colsSql = columns.map(_.sql).mkString(", ") + s"CREATE$replaceClause TABLE$ineClause $table ($colsSql)" + } + } + + lazy val columns: Seq[Column] = { + ddl match { + case Left(select) => + select match { + case s: SingleSearch => + s.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + case m: MultiSearch => + m.requests.headOption + .map { req => + req.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + } + .getOrElse(Nil) + case _ => Nil + } + case Right(cols) => cols + } + } + } + + case class AlterTable(table: String, ifExists: Boolean, statement: AlterTableStatement) + extends DdlStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS " else "" + s"ALTER TABLE $table$ifExistsClause $statement" + } + } + + sealed trait AlterTableStatement extends Token + case class AddColumn(column: Column, ifNotExists: Boolean = false) extends AlterTableStatement { + override def sql: String = { + val ifNotExistsClause = if (ifNotExists) " IF NOT EXISTS" else "" + s"ADD COLUMN$ifNotExistsClause ${column.sql}" + } + } + case class DropColumn(columnName: String, ifExists: Boolean = false) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"DROP COLUMN$ifExistsClause $columnName" + } + } + case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { + override def sql: String = s"RENAME COLUMN $oldName TO $newName" + } + case class AlterColumnOptions( + columnName: String, + options: Map[String, Value[_]], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET OPTIONS (${options + .map { case (k, v) => s"$k = $v" } + .mkString(", ")})" + } + } + case class AlterColumnType(columnName: String, newType: SQLType, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET TYPE $newType" + } + } + case class AlterColumnDefault( + columnName: String, + defaultValue: Value[_], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET DEFAULT $defaultValue" + } + } + case class DropColumnDefault(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP DEFAULT" + } + } + case class AlterColumnNotNull(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET NOT NULL" + } + } + case class DropColumnNotNull(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP NOT NULL" + } + } + + case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) + extends DdlStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + val cascadeClause = if (cascade) " CASCADE" else "" + s"DROP TABLE $ifExistsClause$table$cascadeClause" + } + } + + case class TruncateTable(table: String) extends DdlStatement { + override def sql: String = s"TRUNCATE TABLE $table" + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala new file mode 100644 index 00000000..9a7b7ff2 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -0,0 +1,127 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.query._ + +package object schema { + case class Column( + name: String, + dataType: SQLType, + options: Map[String, Value[_]] = Map.empty, + multiFields: List[Column] = Nil, + notNull: Boolean = false, + defaultValue: Option[Value[_]] = None + ) extends Token { + def sql: String = { + val opts = if (options.nonEmpty) { + s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " + } else { + "" + } + val notNullOpt = if (notNull) " NOT NULL" else "" + val defaultOpt = defaultValue.map(v => s" DEFAULT $v").getOrElse("") + val fieldsOpt = if (multiFields.nonEmpty) { + s" FIELDS (${multiFields.mkString(", ")})" + } else { + "" + } + s"$name $dataType$opts$notNullOpt$defaultOpt$fieldsOpt" + } + } + + case class Table( + name: String, + columns: List[Column], + options: Map[String, Value[_]] = Map.empty + ) extends Token { + lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap + + def sql: String = { + val cols = columns.map(_.sql).mkString(", ") + val opts = if (options.nonEmpty) { + s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " + } else { + "" + } + s"CREATE OR REPLACE TABLE $name ($cols)$opts" + } + + def merge(statements: Seq[AlterTableStatement]): Table = { + statements.foldLeft(this) { (table, statement) => + statement match { + case AddColumn(column, ifNotExists) => + table.copy(columns = table.columns :+ column) + case DropColumn(columnName, ifExists) => + table.copy(columns = table.columns.filterNot(_.name == columnName)) + case RenameColumn(oldName, newName) => + table.copy( + columns = table.columns.map { col => + if (col.name == oldName) col.copy(name = newName) else col + } + ) + case AlterColumnType(columnName, newType, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(dataType = newType) + else col + } + ) + case AlterColumnDefault(columnName, newDefault, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) + else col + } + ) + case DropColumnDefault(columnName, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = None) + else col + } + ) + case AlterColumnNotNull(columnName, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = true) + else col + } + ) + case DropColumnNotNull(columnName, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = false) + else col + } + ) + case AlterColumnOptions(columnName, newOptions, ifExists) => + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options ++ newOptions) + else col + } + ) + case _ => table + } + } + } + } + +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala similarity index 95% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala rename to sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala index a9998da9..151f339b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala @@ -1,12 +1,12 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.function.time -import org.scalatest.funsuite.AnyFunSuite +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.function._ -import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.time._ -import app.softnetwork.elastic.sql.`type`.SQLType +import org.scalatest.funsuite.AnyFunSuite -class SQLDateTimeFunctionSuite extends AnyFunSuite { +class DateTimeFunctionSuite extends AnyFunSuite { // Base d'exemple val baseDate = "doc['createdAt'].value" diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala similarity index 74% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala rename to sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 186833e3..a74c8162 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,7 +1,7 @@ -package app.softnetwork.elastic.sql - -import app.softnetwork.elastic.sql.parser._ +package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -222,14 +222,14 @@ object Queries { /** Created by smanciot on 15/02/17. */ -class SQLParserSpec extends AnyFlatSpec with Matchers { +class ParserSpec extends AnyFlatSpec with Matchers { import Queries._ - "SQLParser" should "parse numerical EQ" in { + "Parser" should "parse numerical EQ" in { val result = Parser(numericalEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalEq) shouldBe true } @@ -237,7 +237,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical NE" in { val result = Parser(numericalNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalNe) shouldBe true } @@ -245,7 +245,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical LT" in { val result = Parser(numericalLt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalLt) shouldBe true } @@ -253,7 +253,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical LE" in { val result = Parser(numericalLe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalLe) shouldBe true } @@ -261,7 +261,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical GT" in { val result = Parser(numericalGt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalGt) shouldBe true } @@ -269,7 +269,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical GE" in { val result = Parser(numericalGe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalGe) shouldBe true } @@ -277,7 +277,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal EQ" in { val result = Parser(literalEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalEq) shouldBe true } @@ -285,7 +285,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LIKE" in { val result = Parser(literalLike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLike) shouldBe true } @@ -293,7 +293,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal RLIKE" in { val result = Parser(literalRlike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalRlike) shouldBe true } @@ -301,7 +301,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal NOT LIKE" in { val result = Parser(literalNotLike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalNotLike) shouldBe true } @@ -309,7 +309,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal NE" in { val result = Parser(literalNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalNe) shouldBe true } @@ -317,7 +317,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LT" in { val result = Parser(literalLt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLt) shouldBe true } @@ -325,7 +325,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LE" in { val result = Parser(literalLe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLe) shouldBe true } @@ -333,20 +333,20 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal GT" in { val result = Parser(literalGt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalGt) shouldBe true } it should "parse literal GE" in { val result = Parser(literalGe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") equalsIgnoreCase literalGe + result.toOption.map(_.sql).getOrElse("") equalsIgnoreCase literalGe } it should "parse boolean EQ" in { val result = Parser(boolEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(boolEq) shouldBe true } @@ -354,7 +354,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse boolean NE" in { val result = Parser(boolNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(boolNe) shouldBe true } @@ -362,7 +362,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse BETWEEN" in { val result = Parser(betweenExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(betweenExpression) shouldBe true } @@ -370,7 +370,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse AND predicate" in { val result = Parser(andPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(andPredicate) shouldBe true } @@ -378,7 +378,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse OR predicate" in { val result = Parser(orPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(orPredicate) shouldBe true } @@ -386,7 +386,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse left predicate with criteria" in { val result = Parser(leftPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(leftPredicate) shouldBe true } @@ -394,7 +394,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse right predicate with criteria" in { val result = Parser(rightPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(rightPredicate) shouldBe true } @@ -402,7 +402,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse multiple predicates" in { val result = Parser(predicates) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(predicates) shouldBe true } @@ -410,14 +410,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse nested predicate" in { val result = Parser(nestedPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedPredicate } it should "parse nested criteria" in { val result = Parser(nestedCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nestedCriteria) shouldBe true } @@ -425,7 +425,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse child predicate" in { val result = Parser(childPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(childPredicate) shouldBe true } @@ -433,7 +433,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse child criteria" in { val result = Parser(childCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(childCriteria) shouldBe true } @@ -441,7 +441,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse parent predicate" in { val result = Parser(parentPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(parentPredicate) shouldBe true } @@ -449,7 +449,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse parent criteria" in { val result = Parser(parentCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(parentCriteria) shouldBe true } @@ -457,7 +457,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN literal expression" in { val result = Parser(inLiteralExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inLiteralExpression) shouldBe true } @@ -465,7 +465,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN numerical expression with Int values" in { val result = Parser(inNumericalExpressionWithIntValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inNumericalExpressionWithIntValues) shouldBe true } @@ -473,7 +473,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN numerical expression with Double values" in { val result = Parser(inNumericalExpressionWithDoubleValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inNumericalExpressionWithDoubleValues) shouldBe true } @@ -481,7 +481,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN literal expression" in { val result = Parser(notInLiteralExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInLiteralExpression) shouldBe true } @@ -489,7 +489,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN numerical expression with Int values" in { val result = Parser(notInNumericalExpressionWithIntValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInNumericalExpressionWithIntValues) shouldBe true } @@ -497,7 +497,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN numerical expression with Double values" in { val result = Parser(notInNumericalExpressionWithDoubleValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInNumericalExpressionWithDoubleValues) shouldBe true } @@ -505,7 +505,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse nested with BETWEEN" in { val result = Parser(nestedWithBetween) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nestedWithBetween) shouldBe true } @@ -513,7 +513,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COUNT" in { val result = Parser(COUNT) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(COUNT) shouldBe true } @@ -521,7 +521,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse DISTINCT COUNT" in { val result = Parser(countDistinct) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(countDistinct) shouldBe true } @@ -529,7 +529,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COUNT with nested criteria" in { val result = Parser(countNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(countNested) shouldBe true } @@ -537,7 +537,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IS NULL" in { val result = Parser(isNull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNull) shouldBe true } @@ -545,7 +545,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IS NOT NULL" in { val result = Parser(isNotNull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNotNull) shouldBe true } @@ -553,14 +553,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse geo distance criteria" in { val result = Parser(geoDistanceCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe geoDistanceCriteria } it should "parse EXCEPT fields" in { val result = Parser(except) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(except) shouldBe true } @@ -568,7 +568,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse MATCH criteria" in { val result = Parser(matchCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(matchCriteria) shouldBe true } @@ -576,7 +576,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY" in { val result = Parser(groupBy) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupBy) shouldBe true } @@ -584,7 +584,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ORDER BY" in { val result = Parser(orderBy) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(orderBy) shouldBe true } @@ -592,14 +592,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse LIMIT" in { val result = Parser(limit) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe limit } it should "parse GROUP BY with ORDER BY and LIMIT" in { val result = Parser(groupByWithOrderByAndLimit) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupByWithOrderByAndLimit) shouldBe true } @@ -607,7 +607,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY with HAVING" in { val result = Parser(groupByWithHaving) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupByWithHaving) shouldBe true } @@ -615,7 +615,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date time fields" in { val result = Parser(dateTimeWithIntervalFields) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeWithIntervalFields) shouldBe true } @@ -623,7 +623,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse fields with INTERVAL" in { val result = Parser(fieldsWithInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(fieldsWithInterval) shouldBe true } @@ -631,7 +631,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with date time and INTERVAL" in { val result = Parser(filterWithDateTimeAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithDateTimeAndInterval) shouldBe true } @@ -639,7 +639,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with date and interval" in { val result = Parser(filterWithDateAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithDateAndInterval) shouldBe true } @@ -647,7 +647,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with time and interval" in { val result = Parser(filterWithTimeAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } @@ -655,14 +655,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY with HAVING and date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe groupByWithHavingAndDateTimeFunctions } it should "parse date_parse function" in { val result = Parser(dateParse) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateParse) shouldBe true } @@ -670,7 +670,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_parse_time function" in { val result = Parser(dateTimeParse) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeParse) shouldBe true } @@ -678,7 +678,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_diff function" in { val result = Parser(dateDiff) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateDiff) shouldBe true } @@ -686,7 +686,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_diff function with aggregation" in { val result = Parser(aggregationWithDateDiff) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(aggregationWithDateDiff) shouldBe true } @@ -694,7 +694,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse format_date function" in { val result = Parser(dateFormat) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateFormat) shouldBe true } @@ -702,7 +702,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse format_datetime function" in { val result = Parser(dateTimeFormat) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeFormat) shouldBe true } @@ -710,7 +710,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_add function" in { val result = Parser(dateAdd) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateAdd) shouldBe true } @@ -718,7 +718,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_sub function" in { val result = Parser(dateSub) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateSub) shouldBe true } @@ -726,7 +726,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse datetime_add function" in { val result = Parser(dateTimeAdd) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeAdd) shouldBe true } @@ -734,7 +734,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse datetime_sub function" in { val result = Parser(dateTimeSub) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeSub) shouldBe true } @@ -742,7 +742,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNULL function" in { val result = Parser(isnull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isnull) shouldBe true } @@ -750,7 +750,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNOTNULL function" in { val result = Parser(isnotnull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isnotnull) shouldBe true } @@ -758,7 +758,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNULL criteria" in { val result = Parser(isNullCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNullCriteria) shouldBe true } @@ -766,7 +766,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNOTNULL criteria" in { val result = Parser(isNotNullCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNotNullCriteria) shouldBe true } @@ -774,7 +774,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COALESCE function" in { val result = Parser(coalesce) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(coalesce) shouldBe true } @@ -782,7 +782,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NULLIF function" in { val result = Parser(nullif) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nullif) shouldBe true } @@ -790,14 +790,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse conversion function" in { val result = Parser(conversion) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe conversion } it should "parse all casts function" in { val result = Parser(allCasts) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(allCasts) shouldBe true } @@ -805,7 +805,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse CASE WHEN expression" in { val result = Parser(caseWhen) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(caseWhen) shouldBe true } @@ -813,7 +813,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse CASE WHEN with expression" in { val result = Parser(caseWhenExpr) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(caseWhenExpr) shouldBe true } @@ -821,14 +821,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse EXTRACT function" in { val result = Parser(extract) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe extract } it should "parse arithmetic expressions" in { val result = Parser(arithmetic) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(arithmetic) shouldBe true } @@ -836,91 +836,377 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse mathematical functions" in { val result = Parser(mathematical) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe mathematical } it should "parse string functions" in { val result = Parser(string) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe string } it should "parse top hits functions" in { val result = Parser(topHits) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe topHits } it should "parse last_day function" in { val result = Parser(lastDay) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe lastDay } it should "parse all date extractors" in { val result = Parser(extractors) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe extractors } it should "parse geo distance field" in { val result = Parser(geoDistance) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe geoDistance } it should "parse BETWEEN with temporal fields" in { val result = Parser(betweenTemporal) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe betweenTemporal } it should "parse nested of nested" in { val result = Parser(nestedOfNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedOfNested } it should "parse predicate with distinct nested" in { val result = Parser(predicateWithDistinctNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe predicateWithDistinctNested } it should "parse nested without criteria" in { val result = Parser(nestedWithoutCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedWithoutCriteria } it should "determine the aggregation context" in { val result = Parser(determinationOfTheAggregationContext) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe determinationOfTheAggregationContext } it should "parse aggregation with nested of nested context" in { val result = Parser(aggregationWithNestedOfNestedContext) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe aggregationWithNestedOfNestedContext } it should "parse where filters according to scope" in { val result = Parser(whereFiltersAccordingToScope) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe whereFiltersAccordingToScope } + + // --- DDL --- + + it should "parse CREATE TABLE if not exists" in { + val sql = "CREATE TABLE IF NOT EXISTS users (id INT NOT NULL, name VARCHAR DEFAULT 'anonymous')" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case CreateTable("users", Right(cols), true, false) => + cols.map(_.name) should contain allOf ("id", "name") + cols.find(_.name == "id").get.notNull shouldBe true + cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") + case _ => fail("Expected CreateTable") + } + } + + it should "parse CREATE OR REPLACE TABLE as select" in { + val sql = "CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case table: CreateTable => + println(table.sql) + table.table shouldBe "users" + table.ifNotExists shouldBe false + table.orReplace shouldBe true + table.columns.map(_.name) should contain allOf ("id", "name") + case _ => fail("Expected CreateTable") + } + } + + it should "parse DROP TABLE if exists" in { + val sql = "DROP TABLE IF EXISTS users" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case DropTable("users", ie, _) if ie => + case _ => fail("Expected DropTable") + } + } + + it should "parse TRUNCATE TABLE" in { + val sql = "TRUNCATE TABLE users" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case TruncateTable("users") => + case _ => fail("Expected TruncateTable") + } + } + + it should "parse ALTER TABLE add column if not exists" in { + val sql = + """ALTER TABLE users + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0 + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case AddColumn(c, ine) if ine => + c.name shouldBe "age" + c.dataType.typeId should include("INT") + c.defaultValue.map(_.value) shouldBe Some(0L) + case _ => fail("Expected AddColumn") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE rename column" in { + val sql = + """ALTER TABLE users + | RENAME COLUMN name TO full_name + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case RenameColumn(o, n) => + o shouldBe "name" + n shouldBe "full_name" + case _ => fail("Expected RenameColumn") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column options if exists" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN IF EXISTS status SET OPTIONS (description = 'a description') + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case AlterColumnOptions(c, d, ie) if ie => + c shouldBe "status" + d.get("description").map(_.value) shouldBe Some("a description") + case _ => fail("Expected AlterColumnDefault") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column default" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET DEFAULT 'active' + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case AlterColumnDefault(c, d, _) => + c shouldBe "status" + d.value shouldBe "active" + case other => fail(s"Expected AlterColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE drop column default" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status DROP DEFAULT + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case DropColumnDefault(c, _) => + c shouldBe "status" + case other => fail(s"Expected DropColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column not null" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET NOT NULL + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case AlterColumnNotNull(c, _) => + c shouldBe "status" + case other => fail(s"Expected AlterColumnNotNull, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE drop column not null" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status DROP NOT NULL + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case DropColumnNotNull(c, _) => + c shouldBe "status" + case other => fail(s"Expected DropColumnNotNull, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column type" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET DATA TYPE BIGINT + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmt) => + stmt match { + case AlterColumnType(c, d, _) => + c shouldBe "status" + d shouldBe SQLTypes.BigInt + case other => fail(s"Expected AlterColumnType, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE if exists" in { + val sql = + """ALTER TABLE IF EXISTS users + | ALTER COLUMN status SET DEFAULT 'active' + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", ifExists, stmt) if ifExists => + stmt match { + case AlterColumnDefault(c, d, _) => + c shouldBe "status" + d.value shouldBe "active" + case other => fail(s"Expected AlterColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + // --- DML --- + + it should "parse INSERT INTO ... VALUES" in { + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice')" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", cols, Right(values)) => + cols should contain inOrder ("id", "name") + values.map(_.value) should contain inOrder (1, "Alice") + case _ => fail("Expected Insert with values") + } + } + + it should "parse INSERT INTO ... SELECT" in { + val sql = "INSERT INTO users SELECT id, name FROM old_users" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", Nil, Left(sel: DqlStatement)) => + sel.sql should include("SELECT id, name FROM old_users") + case _ => fail("Expected Insert with select") + } + } + + it should "parse UPDATE" in { + val sql = "UPDATE users SET name = 'Bob', age = 42 WHERE id = 1" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Update("users", values, Some(where)) => + values("name").value shouldBe "Bob" + values("age").value shouldBe 42 + where.sql should include("id = 1") + case _ => fail("Expected Update") + } + } + + it should "parse DELETE" in { + val sql = "DELETE FROM users WHERE age > 30" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Delete(table, Some(where)) => + table.name shouldBe "users" + where.sql should include("age > 30") + case _ => fail("Expected Delete") + } + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 3e8d7ce1..287b20bf 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -25,7 +25,7 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.model.{Binary, Child, Parent, Sample} import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import app.softnetwork.persistence._ import app.softnetwork.persistence.person.model.Person import com.fasterxml.jackson.core.JsonParseException @@ -73,7 +73,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M import scala.language.implicitConversions - implicit def toSQLQuery(sqlQuery: String): SQLQuery = SQLQuery(sqlQuery) + implicit def toSQLQuery(sqlQuery: String): SelectStatement = SelectStatement(sqlQuery) implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = Source.fromIterator(() => list.iterator) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index a86957e2..a6298503 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -22,7 +22,7 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.serialization._ import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -288,7 +288,7 @@ trait MockElasticClientApi extends ElasticClientApi { // ==================== SearchApi ==================== override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SQLSearchRequest + sqlSearch: SingleSearch ): String = """{ | "query": { From a00f5f4183de20fd94ba753d8b4c2c2e2e3f573f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 10 Dec 2025 07:09:23 +0100 Subject: [PATCH 02/95] add support for ALTER COLUMN SET FIELDS, update documentation --- documentation/sql/ddl_statements.md | 157 ++++++++++++++++++ documentation/sql/dml_statements.md | 64 +++++++ .../elastic/sql/parser/Parser.scala | 22 ++- .../elastic/sql/query/package.scala | 21 ++- .../elastic/sql/parser/ParserSpec.scala | 90 +++++++--- 5 files changed, 322 insertions(+), 32 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index e69de29b..d2765cde 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -0,0 +1,157 @@ +# DDL Support +[Back to index](README.md) + +This document describes the SQL statements supported by the API, focusing on **Data Definition Language (DDL)**. Each section provides syntax, examples, and notes on behavior. + +--- + +## 📐 Data Definition Language (DDL) + +### CREATE TABLE +Create a new table with explicit column definitions or from a `SELECT` query. + +**Syntax:** +```sql +CREATE [OR REPLACE] TABLE [IF NOT EXISTS] table_name ( + column_name data_type [NOT NULL] [DEFAULT value] [OPTIONS (...)] [FIELDS (...)] +) +``` + +- `FIELDS (...)` can define **multi‑fields** (alternative analyzers for text) or **STRUCT** (nested objects). + +**Examples:** +```sql +CREATE TABLE IF NOT EXISTS users ( + id INT NOT NULL, + name VARCHAR DEFAULT 'anonymous' +); + +CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; +``` + +--- + +### ALTER TABLE +Modify an existing table. Multiple statements can be grouped inside parentheses. + +**Supported statements:** +- `ADD COLUMN [IF NOT EXISTS] column_definition` +- `DROP COLUMN [IF EXISTS] column_name` +- `RENAME COLUMN old_name TO new_name` +- `ALTER COLUMN [IF EXISTS] column_name SET OPTIONS (...)` +- `ALTER COLUMN [IF EXISTS] column_name SET DEFAULT value` +- `ALTER COLUMN [IF EXISTS] column_name DROP DEFAULT` +- `ALTER COLUMN [IF EXISTS] column_name SET NOT NULL` +- `ALTER COLUMN [IF EXISTS] column_name DROP NOT NULL` +- `ALTER COLUMN [IF EXISTS] column_name SET DATA TYPE new_type` +- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` + → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column. + +**Examples:** +```sql +ALTER TABLE users + ADD COLUMN IF NOT EXISTS age INT DEFAULT 0; + +ALTER TABLE users + RENAME COLUMN name TO full_name; + +ALTER TABLE users ( + ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + RENAME COLUMN name TO full_name, + ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + ALTER COLUMN IF EXISTS profile SET FIELDS ( + description VARCHAR DEFAULT 'N/A', + visibility BOOLEAN DEFAULT true + ) +); +``` + +--- + +### DROP TABLE +Remove an existing table. + +**Syntax:** +```sql +DROP TABLE [IF EXISTS] table_name [CASCADE] +``` + +**Example:** +```sql +DROP TABLE IF EXISTS users CASCADE; +``` + +--- + +### TRUNCATE TABLE +Delete all rows from a table without removing its definition. + +**Syntax:** +```sql +TRUNCATE TABLE table_name +``` + +**Example:** +```sql +TRUNCATE TABLE users; +``` + +--- + +## 🧩 Nested and Structured Data + +### FIELDS for Multi‑fields +`FIELDS (...)` can be used to define **multi‑fields** for text columns. This allows you to index the same column in multiple ways (e.g., with different analyzers). + +**Example:** +```sql +CREATE TABLE docs ( + content VARCHAR FIELDS ( + keyword VARCHAR OPTIONS (analyzer = 'keyword'), + english VARCHAR OPTIONS (analyzer = 'english') + ) +) +``` +- `content` is indexed as text. +- `content.keyword` is a keyword sub‑field. +- `content.english` is a text sub‑field with the English analyzer. + +--- + +### FIELDS for STRUCT +`FIELDS (...)` also enables the definition of **STRUCT** types, which represent nested objects with their own fields. This is how you model hierarchical data. + +**Example:** +```sql +CREATE TABLE users ( + id INT NOT NULL, + profile STRUCT FIELDS ( + first_name VARCHAR NOT NULL, + last_name VARCHAR NOT NULL, + address STRUCT FIELDS ( + street VARCHAR, + city VARCHAR, + zip VARCHAR + ) + ) +) +``` + +- `profile` is a `STRUCT` column containing multiple fields. +- `address` is a nested `STRUCT` inside `profile`. + +This maps naturally to: +- **Elasticsearch**: `object` or `nested` type. +- **Avro**: `record`. + +--- + +## 📝 Notes +- **Types**: Supported SQL types include `INT`, `BIGINT`, `VARCHAR`, `BOOLEAN`, `DATE`, `TIMESTAMP`, etc. +- **Constraints**: `NOT NULL` and `DEFAULT` are supported. Other relational constraints (e.g., `PRIMARY KEY`) are not enforced by Elasticsearch. +- **FIELDS**: Dual purpose — multi‑fields for text analysis and STRUCT for nested data modeling. +- **Options**: Column and table options can be specified via `OPTIONS (...)`. + +--- + +[Back to index](README.md) diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md index e69de29b..b2c6dc1f 100644 --- a/documentation/sql/dml_statements.md +++ b/documentation/sql/dml_statements.md @@ -0,0 +1,64 @@ +# DML Support +[Back to index](README.md) + +This document describes the SQL statements supported by the API, focusing on **Data Manipulation Language (DML)**. Each section provides syntax, examples, and notes on behavior. + +--- + +## 📊 Data Manipulation Language (DML) + +### INSERT +Insert new rows into a table, either with explicit values or from a `SELECT`. + +**Syntax:** +```sql +INSERT INTO table_name (col1, col2, ...) +VALUES (val1, val2, ...); + +INSERT INTO table_name +SELECT ... +``` + +**Examples:** +```sql +INSERT INTO users (id, name) VALUES (1, 'Alice'); + +INSERT INTO users SELECT id, name FROM old_users; +``` + +--- + +### UPDATE +Update existing rows in a table. + +**Syntax:** +```sql +UPDATE table_name +SET col1 = val1, col2 = val2, ... +[WHERE condition] +``` + +**Example:** +```sql +UPDATE users SET name = 'Bob', age = 42 WHERE id = 1; +``` + +--- + +### DELETE +Delete rows from a table. + +**Syntax:** +```sql +DELETE FROM table_name +[WHERE condition] +``` + +**Example:** +```sql +DELETE FROM users WHERE age > 30; +``` + +--- + +[Back to index](README.md) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index a51821ba..07ac268e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -171,6 +171,11 @@ object Parser AlterColumnOptions(col, opts, ifExists = ie) } + def setColumnFields: PackratParser[AlterColumnFields] = + alterColumnIfExists ~ ident ~ "SET" ~ multiFields ^^ { case ie ~ col ~ _ ~ fields => + AlterColumnFields(col, fields, ifExists = ie) + } + def setColumnType: PackratParser[AlterColumnType] = alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ sql_type ^^ { case ie ~ name ~ _ ~ newType => AlterColumnType(name, newType, ifExists = ie) @@ -205,11 +210,22 @@ object Parser setColumnDefault | dropColumnDefault | setColumnNotNull | - dropColumnNotNull + dropColumnNotNull | + setColumnFields def alterTable: PackratParser[AlterTable] = - ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ alterTableStatement ^^ { case _ ~ ie ~ table ~ stmt => - AlterTable(table, ie, stmt) + ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ start.? ~ repsep( + alterTableStatement, + separator + ) ~ end.? ^^ { case _ ~ ie ~ table ~ s ~ stmts ~ e => + if (s.isDefined && e.isEmpty) { + throw new Exception("Mismatched closing parentheses in ALTER TABLE statement") + } else if (s.isEmpty && e.isDefined) { + throw new Exception("Mismatched opening parentheses in ALTER TABLE statement") + } else if (s.isEmpty && e.isEmpty && stmts.size > 1) { + throw new Exception("Multiple ALTER TABLE statements require parentheses") + } else + AlterTable(table, ie, stmts) } def ddlStatement: PackratParser[DdlStatement] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 75ef6a51..6d846ee6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -337,11 +337,17 @@ package object query { } } - case class AlterTable(table: String, ifExists: Boolean, statement: AlterTableStatement) + case class AlterTable(table: String, ifExists: Boolean, statements: List[AlterTableStatement]) extends DdlStatement { override def sql: String = { val ifExistsClause = if (ifExists) " IF EXISTS " else "" - s"ALTER TABLE $table$ifExistsClause $statement" + val parenthesesNeeded = statements.size > 1 + val statementsSql = if (parenthesesNeeded) { + statements.map(_.sql).mkString("(\n\t", ",\n\t", "\n)") + } else { + statements.map(_.sql).mkString("") + } + s"ALTER TABLE $table$ifExistsClause $statementsSql" } } @@ -411,6 +417,17 @@ package object query { s"ALTER COLUMN$ifExistsClause $columnName DROP NOT NULL" } } + case class AlterColumnFields( + columnName: String, + fields: Seq[Column], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + val fieldsSql = fields.map(_.sql).mkString("(\n\t\t", ",\n\t\t", "\n\t)") + s"ALTER COLUMN$ifExistsClause $columnName SET FIELDS $fieldsSql" + } + } case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) extends DdlStatement { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index a74c8162..5e928d97 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -987,9 +987,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case AddColumn(c, ine) if ine => + case AlterTable("users", _, stmts) => + stmts match { + case List(AddColumn(c, ine)) if ine => c.name shouldBe "age" c.dataType.typeId should include("INT") c.defaultValue.map(_.value) shouldBe Some(0L) @@ -1008,9 +1008,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case RenameColumn(o, n) => + case AlterTable("users", _, stmts) => + stmts match { + case List(RenameColumn(o, n)) => o shouldBe "name" n shouldBe "full_name" case _ => fail("Expected RenameColumn") @@ -1028,9 +1028,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case AlterColumnOptions(c, d, ie) if ie => + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnOptions(c, d, ie)) if ie => c shouldBe "status" d.get("description").map(_.value) shouldBe Some("a description") case _ => fail("Expected AlterColumnDefault") @@ -1048,9 +1048,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case AlterColumnDefault(c, d, _) => + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnDefault(c, d, _)) => c shouldBe "status" d.value shouldBe "active" case other => fail(s"Expected AlterColumnDefault, got $other") @@ -1068,9 +1068,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case DropColumnDefault(c, _) => + case AlterTable("users", _, stmts) => + stmts match { + case List(DropColumnDefault(c, _)) => c shouldBe "status" case other => fail(s"Expected DropColumnDefault, got $other") } @@ -1087,9 +1087,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case AlterColumnNotNull(c, _) => + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnNotNull(c, _)) => c shouldBe "status" case other => fail(s"Expected AlterColumnNotNull, got $other") } @@ -1106,9 +1106,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case DropColumnNotNull(c, _) => + case AlterTable("users", _, stmts) => + stmts match { + case List(DropColumnNotNull(c, _)) => c shouldBe "status" case other => fail(s"Expected DropColumnNotNull, got $other") } @@ -1125,9 +1125,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", _, stmt) => - stmt match { - case AlterColumnType(c, d, _) => + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnType(c, d, _)) => c shouldBe "status" d shouldBe SQLTypes.BigInt case other => fail(s"Expected AlterColumnType, got $other") @@ -1145,9 +1145,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case AlterTable("users", ifExists, stmt) if ifExists => - stmt match { - case AlterColumnDefault(c, d, _) => + case AlterTable("users", ifExists, stmts) if ifExists => + stmts match { + case List(AlterColumnDefault(c, d, _)) => c shouldBe "status" d.value shouldBe "active" case other => fail(s"Expected AlterColumnDefault, got $other") @@ -1156,6 +1156,42 @@ class ParserSpec extends AnyFlatSpec with Matchers { } } + it should "parse ALTER TABLE with multiple statements" in { + val sql = + """ALTER TABLE users ( + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + | RENAME COLUMN name TO full_name, + | ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + | ALTER COLUMN IF EXISTS profile SET FIELDS ( + | description VARCHAR DEFAULT 'N/A', + | visibility BOOLEAN DEFAULT true + | ) + |)""".stripMargin + + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + println(stmt.sql) + stmt.sql.replaceAll("\t", " ") shouldBe sql + stmt match { + case AlterTable("users", _, stmts) => + stmts.length shouldBe 4 + stmts.collect { case AddColumn(c, true) => c.name } should contain("age") + stmts.collect { case RenameColumn(o, n) => (o, n) } should contain(("name", "full_name")) + stmts.collect { case AlterColumnDefault(c, d, true) => + List(c, d.value) + }.flatten should contain allOf ("status", "active") + stmts.collect { case AlterColumnFields("profile", fields, true) => + fields.map(f => (f.name, f.dataType.typeId, f.defaultValue.map(_.value))) + }.flatten should contain allOf (("description", "VARCHAR", Some("N/A")), ( + "visibility", + "BOOLEAN", + Some(true) + )) + case _ => fail("Expected AlterTable") + } + } + // --- DML --- it should "parse INSERT INTO ... VALUES" in { From 7c24b890df94ac55ca08139324e979093bbe7df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 10 Dec 2025 07:23:31 +0100 Subject: [PATCH 03/95] update documentation --- README.md | 77 ++++++++++++++++++++++++++++++++++--- documentation/sql/README.md | 4 +- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4eee333d..4b758012 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,9 @@ SoftClient4ES includes a powerful SQL parser that translates standard SQL `SELEC - ✅ Date / Time functions (`YEAR`, `QUARTER`, `MONTH`, `WEEK`, `DAY`, `HOUR`, `MINUTE`, `SECOND`, `MILLISECOND`, `MICROSECOND`, `NANOSECOND`, `EPOCHDAY`, `OFFSET_SECONDS`, `LAST_DAY`, `WEEKDAY`, `YEARDAY`, `INTERVAL`, `CURRENT_DATE`, `CURDATE`, `TODAY`, `NOW`, `CURRENT_TIME`, `CURTIME`, `CURRENT_DATETIME`, `CURRENT_TIMESTAMP`, `DATE_ADD`, `DATEADD`, `DATE_SUB`, `DATESUB`, `DATETIME_ADD`, `DATETIMEADD`, `DATETIME_SUB`, `DATETIMESUB`, `DATE_DIFF`, `DATEDIFF`, `DATE_FORMAT`, `DATE_PARSE`, `DATETIME_FORMAT`, `DATETIME_PARSE`, `DATE_TRUNC`, `EXTRACT`) - ✅ Geospatial functions (`POINT`, `ST_DISTANCE`) - ✅ Aggregate functions (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `DISTINCT`, `FIRST_VALUE`, `LAST_VALUE`, `ARRAY_AGG`) +- ✅ [Window functions](#32-window-functions-support) with `OVER` clause +- ✅ [DML Support](#34-dml-support) (`INSERT`, `UPDATE`, `DELETE`) +- ✅ [DDL Support](#35-ddl-support) (`CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`) **Example:** @@ -1188,6 +1191,70 @@ client.searchAsUnchecked[Product](SQLQuery(dynamicQuery)) client.scrollAsUnchecked[Product](dynamicQuery) ``` +### **3.4 DML Support** + +SoftClient4ES supports **SQL Data Manipulation Language (DML)** statements for interacting with Elasticsearch indices. + +#### **Supported DML Statements** +- ✅ `INSERT INTO … VALUES (…)` +- ✅ `INSERT INTO … SELECT …` +- ✅ `UPDATE … SET … [WHERE …]` +- ✅ `DELETE FROM … [WHERE …]` + +**Examples:** +```sql +INSERT INTO users (id, name) VALUES (1, 'Alice'); +INSERT INTO users SELECT id, name FROM old_users; + +UPDATE users SET name = 'Bob', age = 42 WHERE id = 1; + +DELETE FROM users WHERE age > 30; +``` + +--- + +### **3.5 DDL Support** + +SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to manage table schemas mapped to Elasticsearch indices. + +#### **Supported DDL Statements** +- ✅ `CREATE TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, and `FIELDS` (multi‑fields or STRUCT) +- ✅ `CREATE OR REPLACE TABLE … AS SELECT …` +- ✅ `ALTER TABLE …` with multiple sub‑statements: + - `ADD COLUMN [IF NOT EXISTS] …` + - `DROP COLUMN [IF EXISTS] …` + - `RENAME COLUMN … TO …` + - `ALTER COLUMN [IF EXISTS] … SET OPTIONS (…)` + - `ALTER COLUMN [IF EXISTS] … SET DEFAULT … / DROP DEFAULT` + - `ALTER COLUMN [IF EXISTS] … SET NOT NULL / DROP NOT NULL` + - `ALTER COLUMN [IF EXISTS] … SET DATA TYPE …` + - `ALTER COLUMN [IF EXISTS] … SET FIELDS (…)` (define nested STRUCT or multi‑fields) +- ✅ `DROP TABLE [IF EXISTS] … [CASCADE]` +- ✅ `TRUNCATE TABLE …` + +**Examples:** +```sql +CREATE TABLE IF NOT EXISTS users ( + id INT NOT NULL, + name VARCHAR DEFAULT 'anonymous' +); + +ALTER TABLE users ( + ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + RENAME COLUMN name TO full_name, + ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + ALTER COLUMN IF EXISTS profile SET FIELDS ( + description VARCHAR DEFAULT 'N/A', + visibility BOOLEAN DEFAULT true + ) +); + +DROP TABLE IF EXISTS users CASCADE; +TRUNCATE TABLE users; +``` + +--- + 📖 **[Full SQL Validation Documentation](documentation/sql/validation.md)** 📖 **[Full SQL Documentation](documentation/sql/README.md)** @@ -1517,18 +1584,18 @@ ThisBuild / resolvers ++= Seq( // For Elasticsearch 6 // Using Jest client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.15.0 // Or using Rest High Level client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.15.0 // For Elasticsearch 7 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.15.0 // For Elasticsearch 8 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.15.0 // For Elasticsearch 9 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.15.0 ``` ### **Quick Example** diff --git a/documentation/sql/README.md b/documentation/sql/README.md index 324a61e1..48546750 100644 --- a/documentation/sql/README.md +++ b/documentation/sql/README.md @@ -14,5 +14,5 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: - [Conditional Functions](functions_conditional.md) - [Geo Functions](functions_geo.md) - [Keywords](keywords.md) -- [DML Statements](dml_statements.md) -- [DDL Statements](ddl_statements.md) +- [DML Support](dml_statements.md) +- [DDL Support](ddl_statements.md) From da6c46c9cfc9bf2a7589a40de7fa1a70bc802f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 10 Dec 2025 13:48:06 +0100 Subject: [PATCH 04/95] add support for primary key and partition by within DDL, update dml and ddl documentation, add NopeClientApi, update reindex signature to include the optional pipeline to apply --- README.md | 39 ++- .../main/resources/softnetwork-elastic.conf | 3 + .../elastic/client/ElasticConfig.scala | 3 +- .../elastic/client/ElasticConfig.scala | 3 +- .../client/ElasticClientDelegator.scala | 10 +- .../elastic/client/IndicesApi.scala | 8 +- .../elastic/client/NopeClientApi.scala | 224 ++++++++++++++++++ .../client/metrics/MetricsElasticClient.scala | 5 +- .../elastic/client/AliasApiSpec.scala | 14 +- .../elastic/client/IndicesApiSpec.scala | 21 +- .../elastic/client/MappingApiSpec.scala | 3 +- .../elastic/client/SettingsApiSpec.scala | 21 +- documentation/sql/ddl_statements.md | 180 +++++++++++++- documentation/sql/dml_statements.md | 84 +++++++ .../elastic/client/jest/JestIndicesApi.scala | 3 +- .../client/rest/RestHighLevelClientApi.scala | 3 +- .../client/rest/RestHighLevelClientApi.scala | 3 +- .../elastic/client/java/JavaClientApi.scala | 4 +- .../elastic/client/java/JavaClientApi.scala | 3 +- .../elastic/sql/parser/Parser.scala | 59 ++++- .../elastic/sql/query/package.scala | 6 +- .../elastic/sql/schema/package.scala | 160 +++++++++---- .../elastic/sql/parser/ParserSpec.scala | 20 +- .../elastic/client/MockElasticClientApi.scala | 3 +- 24 files changed, 768 insertions(+), 114 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala diff --git a/README.md b/README.md index 4b758012..abef34b9 100644 --- a/README.md +++ b/README.md @@ -1191,6 +1191,8 @@ client.searchAsUnchecked[Product](SQLQuery(dynamicQuery)) client.scrollAsUnchecked[Product](dynamicQuery) ``` +--- + ### **3.4 DML Support** SoftClient4ES supports **SQL Data Manipulation Language (DML)** statements for interacting with Elasticsearch indices. @@ -1213,12 +1215,28 @@ DELETE FROM users WHERE age > 30; --- +#### 🔄 DML Execution Strategy + +The SQL DML statements (`INSERT`, `UPDATE`, `DELETE`) are automatically translated into Elasticsearch operations. +The execution path depends on the **number of impacted rows**: + +- **Single row impacted** → direct ES operation: + - `INSERT` → `_index` + - `UPDATE` → `_update` + - `DELETE` → `_delete` + +- **Multiple rows impacted** → bulk ingestion: + - All operations are batched and executed via the `_bulk` API. + - Bulk execution is implemented using **Akka Streams**, ensuring efficient back‑pressure handling, parallelism, and resilience for large datasets. + +--- + ### **3.5 DDL Support** SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to manage table schemas mapped to Elasticsearch indices. #### **Supported DDL Statements** -- ✅ `CREATE TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, and `FIELDS` (multi‑fields or STRUCT) +- ✅ `CREATE TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, `FIELDS` (multi‑fields or STRUCT) and `PARTITION BY …` - ✅ `CREATE OR REPLACE TABLE … AS SELECT …` - ✅ `ALTER TABLE …` with multiple sub‑statements: - `ADD COLUMN [IF NOT EXISTS] …` @@ -1236,8 +1254,9 @@ SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to ```sql CREATE TABLE IF NOT EXISTS users ( id INT NOT NULL, - name VARCHAR DEFAULT 'anonymous' -); + name VARCHAR DEFAULT 'anonymous', + birthdate DATE +) PARTITION BY birthdate (MONTH); ALTER TABLE users ( ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, @@ -1255,6 +1274,20 @@ TRUNCATE TABLE users; --- +#### 🔄 MappingApi Migration Workflow + +The `MappingApi` provides intelligent mapping management with **automatic migration, validation, and rollback capabilities**. This ensures that SQL commands such as `ALTER TABLE … ALTER COLUMN SET TYPE …` are safely translated into Elasticsearch operations. + +##### ✨ Features +- ✅ **Automatic Change Detection**: Compares existing mappings with new ones +- ✅ **Safe Migration Strategy**: Creates temporary indices, reindexes, and renames atomically +- ✅ **Automatic Rollback**: Reverts to original state if migration fails +- ✅ **Backup & Restore**: Preserves original mappings and settings +- ✅ **Progress Tracking**: Detailed logging of migration steps +- ✅ **Validation**: Strict JSON validation with error reporting + +--- + 📖 **[Full SQL Validation Documentation](documentation/sql/validation.md)** 📖 **[Full SQL Documentation](documentation/sql/README.md)** diff --git a/core/src/main/resources/softnetwork-elastic.conf b/core/src/main/resources/softnetwork-elastic.conf index 25f1644b..59b3da13 100644 --- a/core/src/main/resources/softnetwork-elastic.conf +++ b/core/src/main/resources/softnetwork-elastic.conf @@ -39,4 +39,7 @@ elastic { latency-threshold = 1000.0 # Alert if average latency > 1000ms } } + + # Custom settings + composite-key-separator = "|" } \ No newline at end of file diff --git a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala index a4b010e9..b49fede5 100644 --- a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala @@ -44,7 +44,8 @@ case class ElasticConfig( discovery: DiscoveryConfig, connectionTimeout: Duration, socketTimeout: Duration, - metrics: MetricsConfig + metrics: MetricsConfig, + compositeKeySeparator: String = "|" ) object ElasticConfig extends StrictLogging { diff --git a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala index ddc77f82..e0fcd4fd 100644 --- a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala @@ -44,7 +44,8 @@ case class ElasticConfig( discovery: DiscoveryConfig, connectionTimeout: Duration, socketTimeout: Duration, - metrics: MetricsConfig + metrics: MetricsConfig, + compositeKeySeparator: String ) object ElasticConfig extends StrictLogging { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index fa3e2da0..55caf5ed 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -128,9 +128,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = - delegate.reindex(sourceIndex, targetIndex, refresh) + delegate.reindex(sourceIndex, targetIndex, refresh, pipeline) /** Check if an index exists. * @@ -160,9 +161,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = - delegate.executeReindex(sourceIndex, targetIndex, refresh) + delegate.executeReindex(sourceIndex, targetIndex, refresh, pipeline) override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = delegate.executeIndexExists(index) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 6bb7e8f1..e3032dc9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -247,7 +247,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean = true + refresh: Boolean = true, + pipeline: Option[String] = None ): ElasticResult[(Boolean, Option[Long])] = { // Validation... validateIndexName(sourceIndex) match { @@ -322,7 +323,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => } // ✅ Performing the reindex with extracting the number of documents - executeReindex(sourceIndex, targetIndex, refresh) match { + executeReindex(sourceIndex, targetIndex, refresh, pipeline) match { case ElasticFailure(error) => logger.error(s"Reindex failed for index '$targetIndex': ${error.message}") ElasticFailure(error) @@ -407,7 +408,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] private[client] def executeIndexExists(index: String): ElasticResult[Boolean] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala new file mode 100644 index 00000000..279301aa --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -0,0 +1,224 @@ +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.query.SQLAggregation +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.client.scroll._ + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions + +trait NopeClientApi extends ElasticClientApi { + + override private[client] def executeAddAlias( + index: String, + alias: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeRemoveAlias( + index: String, + alias: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetAliases(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeSwapAlias( + oldIndex: String, + newIndex: String, + alias: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + /** Check if client is initialized and connected + */ + override def isInitialized: Boolean = true + + /** Test connection + * + * @return + * true if connection is successful + */ + override def testConnection(): Boolean = true + + override def close(): Unit = {} + + override private[client] def executeCount(query: ElasticQuery): ElasticResult[Option[Double]] = + ElasticResult.success(None) + + override private[client] def executeCountAsync( + query: ElasticQuery + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeDelete( + index: String, + id: String, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeFlush( + index: String, + force: Boolean, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeGet( + index: String, + id: String + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeGetAsync(index: String, id: String)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeIndex( + index: String, + id: String, + source: String, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeIndexAsync( + index: String, + id: String, + source: String, + wait: Boolean + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeCreateIndex( + index: String, + settings: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeReindex( + sourceIndex: String, + targetIndex: String, + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = ElasticResult.success((false, None)) + + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeSetMapping( + index: String, + mapping: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeGetMapping(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + /** Classic scroll (works for both hits and aggregations) + */ + override private[client] def scrollClassic( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + aggregations: Map[String, SQLAggregation], + config: ScrollConfig + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + /** Search After (only for hits, more efficient) + */ + override private[client] def searchAfter( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + config: ScrollConfig, + hasSorts: Boolean + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + override private[client] def pitSearchAfter( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + config: ScrollConfig, + hasSorts: Boolean + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + override private[client] def executeSingleSearch( + elasticQuery: ElasticQuery + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeMultiSearch( + elasticQueries: ElasticQueries + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeSingleSearchAsync(elasticQuery: ElasticQuery)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeMultiSearchAsync(elasticQueries: ElasticQueries)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + ): String = "{\"query\": {\"match_all\": {}}}" + + override private[client] def executeUpdateSettings( + index: String, + settings: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeUpdate( + index: String, + id: String, + source: String, + upsert: Boolean, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeUpdateAsync( + index: String, + id: String, + source: String, + upsert: Boolean, + wait: Boolean + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeVersion(): ElasticResult[String] = + ElasticResult.success("0.0.0") +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 78ff461b..c74d1fb4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -114,10 +114,11 @@ class MetricsElasticClient( override def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { measureResult("reindex", Some(s"$sourceIndex->$targetIndex")) { - delegate.reindex(sourceIndex, targetIndex, refresh) + delegate.reindex(sourceIndex, targetIndex, refresh, pipeline) } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index 5ea8bb8f..81bc6902 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -77,9 +77,10 @@ class AliasApiSpec override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean + sourceIndex: JSONQuery, + targetIndex: JSONQuery, + refresh: Boolean, + pipeline: Option[JSONQuery] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } @@ -1663,9 +1664,10 @@ class AliasApiSpec ??? override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean + sourceIndex: JSONQuery, + targetIndex: JSONQuery, + refresh: Boolean, + pipeline: Option[JSONQuery] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index cfd8cc50..4217295a 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -57,7 +57,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { executeReindexResult } @@ -688,7 +689,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } @@ -739,7 +741,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } @@ -942,7 +945,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -978,7 +982,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -1015,7 +1020,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } @@ -1050,7 +1056,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index e2ce89e7..e3b3100e 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -72,7 +72,8 @@ class MappingApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { executeReindexFunction(sourceIndex, targetIndex, refresh) } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index 298fa1b1..394256fa 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -61,7 +61,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? @@ -743,7 +744,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -795,7 +797,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -844,7 +847,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -1409,7 +1413,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -1450,7 +1455,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? @@ -1491,7 +1497,8 @@ class SettingsApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index d2765cde..f62ae342 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -13,8 +13,10 @@ Create a new table with explicit column definitions or from a `SELECT` query. **Syntax:** ```sql CREATE [OR REPLACE] TABLE [IF NOT EXISTS] table_name ( - column_name data_type [NOT NULL] [DEFAULT value] [OPTIONS (...)] [FIELDS (...)] -) + column_name data_type [NOT NULL] [DEFAULT value] [OPTIONS (...)] [FIELDS (...)], + [... more columns ...], + [PRIMARY KEY (column1, column2, ...)] +) [PARTITION BY column_name] ``` - `FIELDS (...)` can define **multi‑fields** (alternative analyzers for text) or **STRUCT** (nested objects). @@ -31,6 +33,15 @@ CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; --- +## 📝 Notes +- **Types**: Supported SQL types include `INT`, `BIGINT`, `VARCHAR`, `BOOLEAN`, `DATE`, `TIMESTAMP`, etc. +- **Constraints**: `NOT NULL` and `DEFAULT` are supported. Other relational constraints (e.g., `PRIMARY KEY`) are not enforced by Elasticsearch. +- **FIELDS**: Dual purpose — multi‑fields for text analysis and STRUCT for nested data modeling. +- **Options**: Column options can be specified via `OPTIONS (...)`. +- The **partition key must be of type `DATE`** and the partition column must be explicitly defined in the table schema. + +--- + ### ALTER TABLE Modify an existing table. Multiple statements can be grouped inside parentheses. @@ -146,11 +157,166 @@ This maps naturally to: --- -## 📝 Notes -- **Types**: Supported SQL types include `INT`, `BIGINT`, `VARCHAR`, `BOOLEAN`, `DATE`, `TIMESTAMP`, etc. -- **Constraints**: `NOT NULL` and `DEFAULT` are supported. Other relational constraints (e.g., `PRIMARY KEY`) are not enforced by Elasticsearch. -- **FIELDS**: Dual purpose — multi‑fields for text analysis and STRUCT for nested data modeling. -- **Options**: Column and table options can be specified via `OPTIONS (...)`. +## 🔄 MappingApi Migration Workflow + +The `MappingApi` provides intelligent mapping management with **automatic migration, validation, and rollback capabilities**. This ensures that SQL commands such as `ALTER TABLE … ALTER COLUMN SET TYPE …` are safely translated into Elasticsearch operations. + +### ✨ Features +- ✅ **Automatic Change Detection**: Compares existing mappings with new ones +- ✅ **Safe Migration Strategy**: Creates temporary indices, reindexes, and renames atomically +- ✅ **Automatic Rollback**: Reverts to original state if migration fails +- ✅ **Backup & Restore**: Preserves original mappings and settings +- ✅ **Progress Tracking**: Detailed logging of migration steps +- ✅ **Validation**: Strict JSON validation with error reporting + +--- + +### 📊 Migration Workflow + +``` +SQL Command: ALTER TABLE users ALTER COLUMN age SET TYPE BIGINT + │ + ▼ +MappingApi Execution: + 1. Backup current mapping and settings + 2. Create temporary index with new mapping (age: long) + 3. Reindex data from original → temporary + 4. Delete original index + 5. Recreate original index with new mapping + 6. Reindex data from temporary → original + 7. Delete temporary index + 8. Rollback if any step fails +``` + +--- + +### 📝 Notes +- **Atomicity**: The workflow ensures that schema changes are applied safely without downtime. +- **Transparency**: Users only see the SQL command; the migration logic is handled internally. +- **Consistency**: All data is reindexed into the new mapping, guaranteeing type correctness. +- **Resilience**: Rollback and backup mechanisms prevent data loss in case of errors. + +--- + +## 📅 Partitioned Tables + +SoftClient4ES supports **partitioned tables** via the `PARTITION BY` clause in `CREATE TABLE`. +This feature allows automatic routing of documents into indices partitioned by the value of a date column. + +--- + +### ✅ Supported Syntax + +```sql +CREATE TABLE [IF NOT EXISTS] table_name ( + column_definitions +) PARTITION BY column_name [(granularity)] +``` + +- The **partition key must be of type `DATE`**. +- The partition column must be explicitly defined in the table schema. +- Granularity is optional. If omitted, defaults to `DAY`. +- Granularity can be explicitly set with `PARTITION BY column (YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)`. +- Partitioning is typically used for time‑based data (e.g., `birthdate`, `event_date`). + +--- + +### Supported Granularities + +| SQL Granularity | ES `date_rounding` | Recommended `date_formats` | Example Index Name | +|-----------------|--------------------|------------------------------------|--------------------| +| YEAR | "y" | ["yyyy"] | users-2025 | +| MONTH | "M" | ["yyyy-MM", "yyyy-MM-dd"] | users-2025-12 | +| DAY (default) | "d" | ["yyyy-MM-dd"] | users-2025-12-10 | +| HOUR | "h" | ["yyyy-MM-dd'T'HH", "yyyy-MM-dd HH"] | users-2025-12-10-09 | +| MINUTE | "m" | ["yyyy-MM-dd'T'HH:mm"] | users-2025-12-10-09-46 | +| SECOND | "s" | ["yyyy-MM-dd'T'HH:mm:ss"] | users-2025-12-10-09-46-30 | + +--- + +### 📌 Example + +```sql +CREATE TABLE IF NOT EXISTS users ( + id INT NOT NULL, + name VARCHAR DEFAULT 'anonymous', + birthdate DATE +) PARTITION BY birthdate (MONTH); +``` + +- Creates a table `users` partitioned by the `birthdate` column. +- Documents are routed into indices partitioned by **month** of `birthdate`. + For `birthdate = 2025-12-10`, the target index will be `users-2025-12`. + +--- + +### ⚙️ Elasticsearch Translation + +- **Mapping**: the partition key must be declared as a `date` field in the index mapping. +- **Pipeline**: SoftClient4ES uses the [`date_index_name`](https://www.elastic.co/guide/en/elasticsearch/reference/current/date-index-name-processor.html) processor to route documents into partitioned indices. +- **Index Templates**: SoftClient4ES uses it to ensure consistent mappings and settings across all partitioned indices. + +--- + +### 📝 Notes + +- **Granularity**: controlled via `date_rounding` (`y`, `M`, `d`, etc.). +- **Migration**: existing documents are reindexed to be redistributed 🔀 into the correct partitions. +- **SQL vs ES**: in SQL, `PARTITION BY` is a logical clause; in Elasticsearch, it is implemented via ingest pipelines and index naming. + +--- + +## 🔑 Composite Primary Keys + +SoftClient4ES supports composite primary keys in SQL. +In SQL, a primary key can be defined on multiple columns (`PRIMARY KEY (col1, col2)`), ensuring uniqueness across the combination of values. +In Elasticsearch, uniqueness is enforced by the special `_id` field. To emulate composite primary keys, `_id` is constructed from multiple fields using the `set` processor. + +--- + +### ✅ Syntax + +```sql +CREATE TABLE users ( + id INT NOT NULL, + birthdate DATE NOT NULL, + name VARCHAR, + PRIMARY KEY (id, birthdate) +); +``` + +--- + +### ⚙️ Elasticsearch Translation + +A pipeline is automatically created to set `_id` as a concatenation of the primary key columns: + +```curl +PUT _ingest/pipeline/users-composite-id +{ + "processors": [ + { + "set": { + "field": "_id", + "value": "{{id}}|{{birthdate}}" + } + } + ] +} +``` + +- `_id` is built from the values of `id` and `birthdate`. +- Example: `id = 42`, `birthdate = 2025-12-10` → `_id = "42|2025-12-10"`. +- The separator (`|`) can be customized to avoid collisions (elastic.composite-key-separator). + +--- + +### 📝 Notes +- **Stability**: chosen fields must be immutable to preserve uniqueness. +- **Performance**: avoid overly long `_id` values. +- **SQL ↔ ES Mapping**: + - `PRIMARY KEY (id)` → `_id = id` + - `PRIMARY KEY (id, birthdate)` → `_id = "{{id}}-{{birthdate}}"` --- diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md index b2c6dc1f..f58e8f7d 100644 --- a/documentation/sql/dml_statements.md +++ b/documentation/sql/dml_statements.md @@ -61,4 +61,88 @@ DELETE FROM users WHERE age > 30; --- +## 🔄 DML Execution Strategy + +The SQL DML statements (`INSERT`, `UPDATE`, `DELETE`) are automatically translated into Elasticsearch operations. +The execution path depends on the **number of impacted rows**: + +- **Single row impacted** → direct ES operation: + - `INSERT` → `_index` + - `UPDATE` → `_update` + - `DELETE` → `_delete` + +- **Multiple rows impacted** → bulk ingestion: + - All operations are batched and executed via the `_bulk` API. + - Bulk execution is implemented using **Akka Streams**, ensuring efficient back‑pressure handling, parallelism, and resilience for large datasets. + +--- + +### 📌 Example Translation + +**SQL:** +```sql +INSERT INTO users (id, name) VALUES (1, 'Alice'); +``` + +**ES:** +```curl +PUT users/_doc/1 +{ + "id": 1, + "name": "Alice" +} +``` + +--- + +**SQL:** +```sql +UPDATE users SET name = 'Bob' WHERE id = 1; +``` + +**ES:** +```curl +POST users/_update/1 +{ + "doc": { "name": "Bob" } +} +``` + +--- + +**SQL:** +```sql +DELETE FROM users WHERE id = 1; +``` + +**ES:** +```curl +DELETE users/_doc/1 +``` + +--- + +**SQL (multi‑row):** +```sql +INSERT INTO users (id, name) VALUES (1, 'Alice'), (2, 'Bob'); +``` + +**ES (bulk via Akka Streams):** +```curl +POST _bulk +{ "index": { "_index": "users", "_id": "1" } } +{ "id": 1, "name": "Alice" } +{ "index": { "_index": "users", "_id": "2" } } +{ "id": 2, "name": "Bob" } +``` + +--- + +### ✅ Notes +- The API automatically chooses between **single‑doc operations** and **bulk operations**. +- Bulk execution is stream‑based, scalable, and fault‑tolerant thanks to Akka Streams. +- This strategy ensures optimal performance while keeping SQL semantics transparent for the user. + +--- + [Back to index](README.md) diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 13f93df5..3f4c1beb 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -99,7 +99,8 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = executeJestAction[JestResult, (Boolean, Option[Long])]( operation = "reindex", diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index b806adc4..2b651ca0 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -174,7 +174,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): result.ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 8597d9e7..39a103ae 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -179,7 +179,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): result.ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index cb325d0f..9114b42b 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -25,7 +25,6 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -160,7 +159,8 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): result.ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 6f47d492..423d4748 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -155,7 +155,8 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): result.ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 07ac268e..4a2f8af3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -29,7 +29,8 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.Column +import app.softnetwork.elastic.sql.schema.{Column, Partition} +import app.softnetwork.elastic.sql.time.TimeUnit import scala.language.implicitConversions import scala.language.existentials @@ -73,12 +74,12 @@ object Parser } def options: PackratParser[Map[String, Value[_]]] = - "OPTIONS" ~ "(" ~ repsep(option, separator) ~ ")" ^^ { case _ ~ _ ~ opts ~ _ => + "OPTIONS" ~ start ~ repsep(option, separator) ~ end ^^ { case _ ~ _ ~ opts ~ _ => opts.toMap } def multiFields: PackratParser[List[Column]] = - "FIELDS" ~ "(" ~ repsep(column, separator) ~ ")" ^^ { case _ ~ _ ~ cols ~ _ => + "FIELDS" ~ start ~ repsep(column, separator) ~ end ^^ { case _ ~ _ ~ cols ~ _ => cols } | success(Nil) @@ -114,25 +115,59 @@ object Parser } def columns: PackratParser[List[Column]] = - "(" ~ repsep(column, separator) ~ ")" ^^ { case _ ~ cols ~ _ => cols } + start ~ repsep(column, separator) ~ end ^^ { case _ ~ cols ~ _ => cols } + + def primaryKey: PackratParser[List[String]] = + separator ~ "PRIMARY" ~ "KEY" ~ start ~ repsep(ident, separator) ~ end ^^ { + case _ ~ _ ~ _ ~ _ ~ keys ~ _ => + keys + } | success(Nil) + + def granularity: PackratParser[TimeUnit] = start ~ + (("YEAR" ^^^ TimeUnit.YEARS) | + ("MONTH" ^^^ TimeUnit.MONTHS) | + ("DAY" ^^^ TimeUnit.DAYS) | + ("HOUR" ^^^ TimeUnit.HOURS) | + ("MINUTE" ^^^ TimeUnit.MINUTES) | + ("SECOND" ^^^ TimeUnit.SECONDS)) ~ end ^^ { case _ ~ gf ~ _ => gf } + + def partitionBy: PackratParser[Option[Partition]] = + opt("PARTITION" ~ "BY" ~ ident ~ opt(granularity)) ^^ { + case Some(_ ~ _ ~ pb ~ gf) => Some(Partition(pb, gf.getOrElse(TimeUnit.DAYS))) + case None => None + } + + def columnsWithPartitionBy: PackratParser[(List[Column], List[String], Option[Partition])] = + start ~ repsep(column, separator) ~ primaryKey ~ end ~ partitionBy ^^ { + case _ ~ cols ~ pk ~ _ ~ pb => + (cols, pk, pb) + } def createOrReplaceTable: PackratParser[CreateTable] = - ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columns | ("AS" ~> dqlStatement)) ^^ { + ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ name ~ lr => lr match { - case cols: List[Column] => - CreateTable(name, Right(cols), ifNotExists = false, orReplace = true) + case (cols: List[Column], pk: List[String], p: Option[Partition]) => + CreateTable( + name, + Right(cols), + ifNotExists = false, + orReplace = true, + primaryKey = pk, + partitionBy = p + ) case sel: DqlStatement => CreateTable(name, Left(sel), ifNotExists = false, orReplace = true) } } def createTable: PackratParser[CreateTable] = - ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columns | ("AS" ~> dqlStatement)) ^^ { + ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ ine ~ name ~ lr => lr match { - case cols: List[Column] => CreateTable(name, Right(cols), ine) - case sel: DqlStatement => CreateTable(name, Left(sel), ine) + case (cols: List[Column], pk: List[String], p: Option[Partition]) => + CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p) + case sel: DqlStatement => CreateTable(name, Left(sel), ine) } } @@ -233,8 +268,8 @@ object Parser /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ def insert: PackratParser[Insert] = - ("INSERT" ~ "INTO") ~ ident ~ opt("(" ~> repsep(ident, separator) <~ ")") ~ - (("VALUES" ~ "(" ~> repsep(value, separator) <~ ")") ^^ { vs => Right(vs) } + ("INSERT" ~ "INTO") ~ ident ~ opt(start ~> repsep(ident, separator) <~ end) ~ + (("VALUES" ~ start ~> repsep(value, separator) <~ end) ^^ { vs => Right(vs) } | dqlStatement ^^ { q => Left(q) }) ^^ { case _ ~ table ~ colsOpt ~ vals => Insert(table, colsOpt.getOrElse(Nil), vals) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 6d846ee6..fa1cd58e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType -import app.softnetwork.elastic.sql.schema.Column +import app.softnetwork.elastic.sql.schema.{Column, Partition} import app.softnetwork.elastic.sql.function.aggregate.WindowFunction package object query { @@ -303,7 +303,9 @@ package object query { table: String, ddl: Either[DqlStatement, List[Column]], ifNotExists: Boolean = false, - orReplace: Boolean = false + orReplace: Boolean = false, + primaryKey: List[String] = Nil, + partitionBy: Option[Partition] = None ) extends DdlStatement { override def sql: String = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 9a7b7ff2..1bbcae34 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.time.TimeUnit package object schema { case class Column( @@ -45,79 +46,140 @@ package object schema { } } + case class Partition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { + def sql: String = s"PARTITION BY $column ($granularity)" + + val dateRounding: String = granularity.script.get + + val dateFormats: List[String] = granularity match { + case TimeUnit.YEARS => List("yyyy") + case TimeUnit.MONTHS => List("yyyy-MM") + case TimeUnit.DAYS => List("yyyy-MM-dd") + case TimeUnit.HOURS => List("yyyy-MM-dd'T'HH", "yyyy-MM-dd HH") + case TimeUnit.MINUTES => + List("yyyy-MM-dd'T'HH:mm", "yyyy-MM-dd HH:mm") + case TimeUnit.SECONDS => + List("yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd HH:mm:ss") + case _ => List.empty + } + } + + case class ColumnNotFound(column: String, table: String) + extends Exception(s"Column $column does not exist in table $table") + case class Table( name: String, columns: List[Column], - options: Map[String, Value[_]] = Map.empty + primaryKey: List[String] = Nil, + partitionBy: Option[Partition] = None, + defaultPipeline: Option[String] = None, + finalPipeline: Option[String] = None ) extends Token { - lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap + private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap def sql: String = { val cols = columns.map(_.sql).mkString(", ") - val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " + val pkStr = if (primaryKey.nonEmpty) { + s", PRIMARY KEY (${primaryKey.mkString(", ")})" } else { "" } - s"CREATE OR REPLACE TABLE $name ($cols)$opts" + s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}" } def merge(statements: Seq[AlterTableStatement]): Table = { statements.foldLeft(this) { (table, statement) => statement match { case AddColumn(column, ifNotExists) => - table.copy(columns = table.columns :+ column) + if (ifNotExists && table.cols.contains(column.name)) table + else if (!table.cols.contains(column.name)) + table.copy(columns = table.columns :+ column) + else throw ColumnNotFound(column.name, table.name) case DropColumn(columnName, ifExists) => - table.copy(columns = table.columns.filterNot(_.name == columnName)) + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy(columns = table.columns.filterNot(_.name == columnName)) + else throw ColumnNotFound(columnName, table.name) case RenameColumn(oldName, newName) => - table.copy( - columns = table.columns.map { col => - if (col.name == oldName) col.copy(name = newName) else col - } - ) + if (cols.contains(oldName)) + table.copy( + columns = table.columns.map { col => + if (col.name == oldName) col.copy(name = newName) else col + } + ) + else throw ColumnNotFound(oldName, table.name) case AlterColumnType(columnName, newType, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(dataType = newType) - else col - } - ) + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(dataType = newType) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case AlterColumnDefault(columnName, newDefault, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) - else col - } - ) + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case DropColumnDefault(columnName, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(defaultValue = None) - else col - } - ) + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case AlterColumnNotNull(columnName, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(notNull = true) - else col - } - ) + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = true) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case DropColumnNotNull(columnName, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(notNull = false) - else col - } - ) + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = false) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case AlterColumnOptions(columnName, newOptions, ifExists) => - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(options = col.options ++ newOptions) - else col - } - ) + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options ++ newOptions) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case AlterColumnFields(columnName, newFields, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(multiFields = newFields.toList) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) case _ => table } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 5e928d97..3091c3af 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -2,6 +2,8 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.Partition +import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -927,15 +929,29 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DDL --- it should "parse CREATE TABLE if not exists" in { - val sql = "CREATE TABLE IF NOT EXISTS users (id INT NOT NULL, name VARCHAR DEFAULT 'anonymous')" + val sql = + """CREATE TABLE IF NOT EXISTS users ( + | id INT NOT NULL, + | name VARCHAR DEFAULT 'anonymous', + | birthdate DATE, + | PRIMARY KEY (id) + |) PARTITION BY birthdate""".stripMargin val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case CreateTable("users", Right(cols), true, false) => + case CreateTable( + "users", + Right(cols), + true, + false, + List("id"), + Some(Partition("birthdate", TimeUnit.DAYS)) + ) => cols.map(_.name) should contain allOf ("id", "name") cols.find(_.name == "id").get.notNull shouldBe true cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") + cols.find(_.name == "birthdate").get.dataType.typeId should include("DATE") case _ => fail("Expected CreateTable") } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index a6298503..8a749f90 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -110,7 +110,8 @@ trait MockElasticClientApi extends ElasticClientApi { override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ElasticResult.success((true, Some(elasticDocuments.getAll.keys.size))) From 255b489aad49400c6e85217e6ed908854e124046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 10 Dec 2025 17:22:29 +0100 Subject: [PATCH 05/95] add support for scripted column --- documentation/sql/ddl_statements.md | 88 +++++++++++++++++-- .../app/softnetwork/elastic/sql/package.scala | 47 +++++++--- .../elastic/sql/parser/Parser.scala | 35 +++++--- .../elastic/sql/query/package.scala | 17 ++-- .../elastic/sql/schema/package.scala | 60 ++++++++----- .../elastic/sql/parser/ParserSpec.scala | 9 +- 6 files changed, 195 insertions(+), 61 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index f62ae342..b738c529 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -223,14 +223,14 @@ CREATE TABLE [IF NOT EXISTS] table_name ( ### Supported Granularities -| SQL Granularity | ES `date_rounding` | Recommended `date_formats` | Example Index Name | -|-----------------|--------------------|------------------------------------|--------------------| -| YEAR | "y" | ["yyyy"] | users-2025 | -| MONTH | "M" | ["yyyy-MM", "yyyy-MM-dd"] | users-2025-12 | -| DAY (default) | "d" | ["yyyy-MM-dd"] | users-2025-12-10 | -| HOUR | "h" | ["yyyy-MM-dd'T'HH", "yyyy-MM-dd HH"] | users-2025-12-10-09 | -| MINUTE | "m" | ["yyyy-MM-dd'T'HH:mm"] | users-2025-12-10-09-46 | -| SECOND | "s" | ["yyyy-MM-dd'T'HH:mm:ss"] | users-2025-12-10-09-46-30 | +| SQL Granularity | ES `date_rounding` | Example Index Name | +|-----------------|--------------------|---------------------------| +| YEAR | "y" | users-2025 | +| MONTH | "M" | users-2025-12 | +| DAY (default) | "d" | users-2025-12-10 | +| HOUR | "h" | users-2025-12-10-09 | +| MINUTE | "m" | users-2025-12-10-09-46 | +| SECOND | "s" | users-2025-12-10-09-46-30 | --- @@ -320,4 +320,76 @@ PUT _ingest/pipeline/users-composite-id --- +## Scripted Columns 🧮 + +SoftClient4ES supports scripted columns in `CREATE TABLE`. +These columns are **computed at ingestion time** using an ingest pipeline `script` processor. +The value is persisted in `_source` and behaves like any other field. + +--- + +### ✅ Syntax + +```sql +CREATE TABLE table_name ( + column_definitions, + scripted_column TYPE SCRIPT AS (sql_expression) +); +``` + +- `scripted_column` is a regular column name. +- `TYPE` defines the target type (`INT`, `VARCHAR`, etc.). +- `SCRIPT AS (sql_expression)` defines the computation in SQL syntax. +- The expression is translated into **Painless** for Elasticsearch. + +--- + +### 📌 SQL → ES Translation + +**SQL:** +```sql +CREATE TABLE users ( + id INT NOT NULL, + name VARCHAR, + birthdate DATE, + age INT SCRIPT AS (YEAR(CURRENT_DATE) - YEAR(birthdate)), + PRIMARY KEY (id) +) PARTITION BY birthdate (MONTH); +``` + +**Elasticsearch pipeline:** +```curl +PUT _ingest/pipeline/users-pipeline +{ + "processors": [ + { + "script": { + "source": "ctx.age = ChronoUnit.YEARS.between(ctx.birthdate, Instant.now())" + } + } + ] +} +``` + +--- + +### 📖 Examples + +| SQL Script Expression | ES Painless Translation (ingest) | +|--------------------------------------------|--------------------------------------------------------------------| +| `YEAR(CURRENT_DATE) - YEAR(birthdate)` | `ctx.age = ChronoUnit.YEARS.between(ctx.birthdate, Instant.now())` | +| `UPPER(name)` | `ctx.name_upper = ctx.name.toUpperCase()` | +| `CONCAT(firstname, ' ', lastname)` | `ctx.fullname = ctx.firstname + ' ' + ctx.lastname` | +| `CASE WHEN status = 'X' THEN 1 ELSE 0 END` | `ctx.flag = ctx.status == 'X' ? 1 : 0` | + +--- + +### 📝 Notes +- Scripted columns are **evaluated once at ingestion**. +- They are **persisted** in `_source`, unlike runtime fields. +- SQL expressions are translated into equivalent **Painless** code. +- This feature allows declarative enrichment directly in the DDL. + +--- + [Back to index](README.md) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 70073314..1ce117ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -110,12 +110,6 @@ package object sql { } } - def paramValue: String = - if (nullable && checkNotNull.nonEmpty) - checkNotNull - else - s"$param${painlessMethods.mkString("")}" - private[this] var _painlessMethods: collection.mutable.Seq[String] = collection.mutable.Seq.empty @@ -129,14 +123,26 @@ package object sql { } - case class LiteralParam(param: String) extends PainlessParam { + case class LiteralParam(literal: String, maybeCheckNotNull: Option[String] = None) + extends PainlessParam { + override def param: String = literal override def sql: String = "" - override def checkNotNull: String = "" + override def nullable: Boolean = maybeCheckNotNull.nonEmpty + override def checkNotNull: String = maybeCheckNotNull.getOrElse("") + } + + sealed trait PainlessContextType + + case object PainlessContextType { + case object Processor extends PainlessContextType + case object Query extends PainlessContextType } /** Context for painless scripts + * @param context + * the context type */ - case class PainlessContext() { + case class PainlessContext(context: PainlessContextType = PainlessContextType.Query) { // List of parameter keys private[this] var _keys: collection.mutable.Seq[PainlessParam] = collection.mutable.Seq.empty @@ -146,6 +152,8 @@ package object sql { // Last parameter name added private[this] var _lastParam: Option[String] = None + def isProcessor: Boolean = context == PainlessContextType.Processor + /** Add a token parameter to the context if not already present * * @param token @@ -155,6 +163,10 @@ package object sql { */ def addParam(token: Token): Option[String] = { token match { + case identifier: Identifier if isProcessor => + addParam( + LiteralParam(identifier.processParamName, identifier.processCheckNotNull) + ) case param: PainlessParam if param.param.nonEmpty && (param.isInstanceOf[LiteralParam] || param.nullable) => get(param) match { @@ -177,6 +189,8 @@ package object sql { def get(token: Token): Option[String] = { token match { + case identifier: Identifier if isProcessor => + get(LiteralParam(identifier.processParamName, identifier.processCheckNotNull)) case param: PainlessParam => if (exists(param)) Try(_values(_keys.indexOf(param))).toOption else None @@ -205,13 +219,19 @@ package object sql { else None } + private[this] def paramValue(param: PainlessParam): String = + if (param.nullable && param.checkNotNull.nonEmpty) + param.checkNotNull + else + s"${param.param}${param.painlessMethods.mkString("")}" + override def toString: String = { if (isEmpty) "" else _keys .flatMap { param => get(param) match { - case Some(v) => Some(s"def $v = ${param.paramValue}; ") + case Some(v) => Some(s"def $v = ${paramValue(param)}; ") case None => None // should not happen } } @@ -772,6 +792,13 @@ package object sql { else s"(doc['$path'].size() == 0 ? $nullValue : doc['$path'].value${painlessMethods.mkString("")})" + lazy val processParamName: String = s"ctx.$path" + + lazy val processCheckNotNull: Option[String] = + if (path.isEmpty || !nullable) None + else + Option(s"(ctx.$path == null ? $nullValue : ctx.$path${painlessMethods.mkString("")})") + override def painless(context: Option[PainlessContext]): String = { val base = context match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 4a2f8af3..add401ee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -29,7 +29,7 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.{Column, Partition} +import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition} import app.softnetwork.elastic.sql.time.TimeUnit import scala.language.implicitConversions @@ -78,7 +78,7 @@ object Parser opts.toMap } - def multiFields: PackratParser[List[Column]] = + def multiFields: PackratParser[List[DdlColumn]] = "FIELDS" ~ start ~ repsep(column, separator) ~ end ^^ { case _ ~ _ ~ cols ~ _ => cols } | success(Nil) @@ -107,14 +107,25 @@ object Parser case None => None } - def column: PackratParser[Column] = - ident ~ sql_type ~ (options | success( + def script: PackratParser[PainlessScript] = + ("SCRIPT" ~ "AS") ~ start ~ (identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction) ~ end ^^ { case _ ~ _ ~ s ~ _ => s } + + def column: PackratParser[DdlColumn] = + ident ~ sql_type ~ (script | multiFields) ~ notNull ~ defaultVal ~ (options | success( Map.empty[String, Value[_]] - )) ~ notNull ~ defaultVal ~ multiFields ^^ { case name ~ dt ~ opts ~ nn ~ dv ~ mfs => - Column(name, dt, opts, mfs, nn, dv) + )) ^^ { case name ~ dt ~ mfs ~ nn ~ dv ~ opts => + mfs match { + case script: PainlessScript => + DdlColumn(name, dt, Some(script), Nil, nn, dv, opts) + case cols: List[DdlColumn] => + DdlColumn(name, dt, None, cols, nn, dv, opts) + } } - def columns: PackratParser[List[Column]] = + def columns: PackratParser[List[DdlColumn]] = start ~ repsep(column, separator) ~ end ^^ { case _ ~ cols ~ _ => cols } def primaryKey: PackratParser[List[String]] = @@ -131,13 +142,13 @@ object Parser ("MINUTE" ^^^ TimeUnit.MINUTES) | ("SECOND" ^^^ TimeUnit.SECONDS)) ~ end ^^ { case _ ~ gf ~ _ => gf } - def partitionBy: PackratParser[Option[Partition]] = + def partitionBy: PackratParser[Option[DdlPartition]] = opt("PARTITION" ~ "BY" ~ ident ~ opt(granularity)) ^^ { - case Some(_ ~ _ ~ pb ~ gf) => Some(Partition(pb, gf.getOrElse(TimeUnit.DAYS))) + case Some(_ ~ _ ~ pb ~ gf) => Some(DdlPartition(pb, gf.getOrElse(TimeUnit.DAYS))) case None => None } - def columnsWithPartitionBy: PackratParser[(List[Column], List[String], Option[Partition])] = + def columnsWithPartitionBy: PackratParser[(List[DdlColumn], List[String], Option[DdlPartition])] = start ~ repsep(column, separator) ~ primaryKey ~ end ~ partitionBy ^^ { case _ ~ cols ~ pk ~ _ ~ pb => (cols, pk, pb) @@ -147,7 +158,7 @@ object Parser ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ name ~ lr => lr match { - case (cols: List[Column], pk: List[String], p: Option[Partition]) => + case (cols: List[DdlColumn], pk: List[String], p: Option[DdlPartition]) => CreateTable( name, Right(cols), @@ -165,7 +176,7 @@ object Parser ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ ine ~ name ~ lr => lr match { - case (cols: List[Column], pk: List[String], p: Option[Partition]) => + case (cols: List[DdlColumn], pk: List[String], p: Option[DdlPartition]) => CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p) case sel: DqlStatement => CreateTable(name, Left(sel), ine) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index fa1cd58e..641b9266 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType -import app.softnetwork.elastic.sql.schema.{Column, Partition} +import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition} import app.softnetwork.elastic.sql.function.aggregate.WindowFunction package object query { @@ -301,11 +301,11 @@ package object query { case class CreateTable( table: String, - ddl: Either[DqlStatement, List[Column]], + ddl: Either[DqlStatement, List[DdlColumn]], ifNotExists: Boolean = false, orReplace: Boolean = false, primaryKey: List[String] = Nil, - partitionBy: Option[Partition] = None + partitionBy: Option[DdlPartition] = None ) extends DdlStatement { override def sql: String = { @@ -320,16 +320,16 @@ package object query { } } - lazy val columns: Seq[Column] = { + lazy val columns: Seq[DdlColumn] = { ddl match { case Left(select) => select match { case s: SingleSearch => - s.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + s.select.fields.map(f => DdlColumn(f.identifier.aliasOrName, f.out)) case m: MultiSearch => m.requests.headOption .map { req => - req.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + req.select.fields.map(f => DdlColumn(f.identifier.aliasOrName, f.out)) } .getOrElse(Nil) case _ => Nil @@ -354,7 +354,8 @@ package object query { } sealed trait AlterTableStatement extends Token - case class AddColumn(column: Column, ifNotExists: Boolean = false) extends AlterTableStatement { + case class AddColumn(column: DdlColumn, ifNotExists: Boolean = false) + extends AlterTableStatement { override def sql: String = { val ifNotExistsClause = if (ifNotExists) " IF NOT EXISTS" else "" s"ADD COLUMN$ifNotExistsClause ${column.sql}" @@ -421,7 +422,7 @@ package object query { } case class AlterColumnFields( columnName: String, - fields: Seq[Column], + fields: Seq[DdlColumn], ifExists: Boolean = false ) extends AlterTableStatement { override def sql: String = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 1bbcae34..c5e0536e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -21,13 +21,14 @@ import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.time.TimeUnit package object schema { - case class Column( + case class DdlColumn( name: String, dataType: SQLType, - options: Map[String, Value[_]] = Map.empty, - multiFields: List[Column] = Nil, + script: Option[PainlessScript] = None, + multiFields: List[DdlColumn] = Nil, notNull: Boolean = false, - defaultValue: Option[Value[_]] = None + defaultValue: Option[Value[_]] = None, + options: Map[String, Value[_]] = Map.empty ) extends Token { def sql: String = { val opts = if (options.nonEmpty) { @@ -46,7 +47,7 @@ package object schema { } } - case class Partition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { + case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { def sql: String = s"PARTITION BY $column ($granularity)" val dateRounding: String = granularity.script.get @@ -64,18 +65,18 @@ package object schema { } } - case class ColumnNotFound(column: String, table: String) + case class DdlColumnNotFound(column: String, table: String) extends Exception(s"Column $column does not exist in table $table") - case class Table( + case class DdlTable( name: String, - columns: List[Column], + columns: List[DdlColumn], primaryKey: List[String] = Nil, - partitionBy: Option[Partition] = None, + partitionBy: Option[DdlPartition] = None, defaultPipeline: Option[String] = None, finalPipeline: Option[String] = None ) extends Token { - private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap + private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap def sql: String = { val cols = columns.map(_.sql).mkString(", ") @@ -87,19 +88,19 @@ package object schema { s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}" } - def merge(statements: Seq[AlterTableStatement]): Table = { + def merge(statements: Seq[AlterTableStatement]): DdlTable = { statements.foldLeft(this) { (table, statement) => statement match { case AddColumn(column, ifNotExists) => if (ifNotExists && table.cols.contains(column.name)) table else if (!table.cols.contains(column.name)) table.copy(columns = table.columns :+ column) - else throw ColumnNotFound(column.name, table.name) + else throw DdlColumnNotFound(column.name, table.name) case DropColumn(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) table.copy(columns = table.columns.filterNot(_.name == columnName)) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case RenameColumn(oldName, newName) => if (cols.contains(oldName)) table.copy( @@ -107,7 +108,7 @@ package object schema { if (col.name == oldName) col.copy(name = newName) else col } ) - else throw ColumnNotFound(oldName, table.name) + else throw DdlColumnNotFound(oldName, table.name) case AlterColumnType(columnName, newType, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -117,7 +118,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnDefault(columnName, newDefault, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -127,7 +128,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case DropColumnDefault(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -137,7 +138,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnNotNull(columnName, ifExists) => if (!table.cols.contains(columnName) && ifExists) table else if (table.cols.contains(columnName)) @@ -147,7 +148,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case DropColumnNotNull(columnName, ifExists) => if (!table.cols.contains(columnName) && ifExists) table else if (table.cols.contains(columnName)) @@ -157,7 +158,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnOptions(columnName, newOptions, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -168,7 +169,7 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnFields(columnName, newFields, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -179,11 +180,28 @@ package object schema { else col } ) - else throw ColumnNotFound(columnName, table.name) + else throw DdlColumnNotFound(columnName, table.name) case _ => table } } } + + override def validate(): Either[SQL, Unit] = { + var errors = Seq[String]() + // check that primary key columns exist + primaryKey.foreach { pk => + if (!cols.contains(pk)) { + errors = errors :+ s"Primary key column $pk does not exist in table $name" + } + } + // check that partition column exists + partitionBy.foreach { partition => + if (!cols.contains(partition.column)) { + errors = errors :+ s"Partition column ${partition.column} does not exist in table $name" + } + } + if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) + } } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 3091c3af..a93292ae 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,8 +1,10 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.Partition +import app.softnetwork.elastic.sql.schema.DdlPartition import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -934,6 +936,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { | id INT NOT NULL, | name VARCHAR DEFAULT 'anonymous', | birthdate DATE, + | age INT SCRIPT AS (date_diff(CURRENT_DATE, birthdate, YEAR)), | PRIMARY KEY (id) |) PARTITION BY birthdate""".stripMargin val result = Parser(sql) @@ -946,12 +949,14 @@ class ParserSpec extends AnyFlatSpec with Matchers { true, false, List("id"), - Some(Partition("birthdate", TimeUnit.DAYS)) + Some(DdlPartition("birthdate", TimeUnit.DAYS)) ) => cols.map(_.name) should contain allOf ("id", "name") cols.find(_.name == "id").get.notNull shouldBe true cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") cols.find(_.name == "birthdate").get.dataType.typeId should include("DATE") + cols.find(_.name == "age").get.script.nonEmpty shouldBe true + //.map(_.painless(Some(PainlessContext(Processor)))).getOrElse("") should include("...") FIXME case _ => fail("Expected CreateTable") } } From 8a5c91e4b9cc6e4bf763cc079a48a42da2e07363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 10 Dec 2025 18:45:08 +0100 Subject: [PATCH 06/95] fix painless script for datediff, add ddl column processorScript utility method, fix parameter name when context is Processor --- .../app/softnetwork/elastic/sql/SQLQuerySpec.scala | 4 ++-- .../app/softnetwork/elastic/sql/SQLQuerySpec.scala | 4 ++-- .../elastic/sql/function/time/package.scala | 4 ++-- .../scala/app/softnetwork/elastic/sql/package.scala | 10 +++++++--- .../app/softnetwork/elastic/sql/schema/package.scala | 12 +++++++++++- .../softnetwork/elastic/sql/parser/ParserSpec.scala | 11 ++++++++--- 6 files changed, 32 insertions(+), 13 deletions(-) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index fb8a7d9e..58c071c2 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1481,7 +1481,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" | } | } | }, @@ -1532,7 +1532,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param3 = (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param3, param2)" | } | } | } diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 1d13b242..856e7bca 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1481,7 +1481,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" | } | } | }, @@ -1532,7 +1532,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param3 = (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param3, param2)" | } | } | } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 55a9bd44..6ddb905d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -388,7 +388,7 @@ package object time { override lazy val words: List[String] = List(sql, "DATEDIFF") } - case class DateDiff(end: PainlessScript, start: PainlessScript, unit: TimeUnit) + case class DateDiff(start: PainlessScript, end: PainlessScript, unit: TimeUnit) extends DateTimeFunction with BinaryFunction[SQLDateTime, SQLDateTime, SQLNumeric] with PainlessScript { @@ -402,7 +402,7 @@ package object time { override def sql: String = DateDiff.sql - override def toSQL(base: String): String = s"$sql(${end.sql}, ${start.sql}, ${unit.sql})" + override def toSQL(base: String): String = s"$sql(${start.sql}, ${end.sql}, ${unit.sql})" override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = s"${unit.painless(context)}${DateDiff.painless(context)}(${callArgs.mkString(", ")})" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 1ce117ee..146841b9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -165,7 +165,7 @@ package object sql { token match { case identifier: Identifier if isProcessor => addParam( - LiteralParam(identifier.processParamName, identifier.processCheckNotNull) + LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) ) case param: PainlessParam if param.param.nonEmpty && (param.isInstanceOf[LiteralParam] || param.nullable) => @@ -190,7 +190,7 @@ package object sql { def get(token: Token): Option[String] = { token match { case identifier: Identifier if isProcessor => - get(LiteralParam(identifier.processParamName, identifier.processCheckNotNull)) + get(LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ )) case param: PainlessParam => if (exists(param)) Try(_values(_keys.indexOf(param))).toOption else None @@ -792,7 +792,11 @@ package object sql { else s"(doc['$path'].size() == 0 ? $nullValue : doc['$path'].value${painlessMethods.mkString("")})" - lazy val processParamName: String = s"ctx.$path" + lazy val processParamName: String = { + if (path.nonEmpty) + s"ctx.$path" + else "" + } lazy val processCheckNotNull: Option[String] = if (path.isEmpty || !nullable) None diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index c5e0536e..d5f6cc18 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.time.TimeUnit @@ -43,7 +44,16 @@ package object schema { } else { "" } - s"$name $dataType$opts$notNullOpt$defaultOpt$fieldsOpt" + val scriptOpt = script.map(s => s" SCRIPT AS ($s)").getOrElse("") + s"$name $dataType$fieldsOpt$scriptOpt$notNullOpt$defaultOpt$opts" + } + + def processorScript: Option[String] = { + script.map { s => + val context = PainlessContext(Processor) + val script = s.painless(Some(context)) + s"$context$script" + } } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index a93292ae..316bd450 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.PainlessContext +import app.softnetwork.elastic.sql.{Identifier, PainlessContext} import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ @@ -936,7 +936,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { | id INT NOT NULL, | name VARCHAR DEFAULT 'anonymous', | birthdate DATE, - | age INT SCRIPT AS (date_diff(CURRENT_DATE, birthdate, YEAR)), + | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), | PRIMARY KEY (id) |) PARTITION BY birthdate""".stripMargin val result = Parser(sql) @@ -956,7 +956,12 @@ class ParserSpec extends AnyFlatSpec with Matchers { cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") cols.find(_.name == "birthdate").get.dataType.typeId should include("DATE") cols.find(_.name == "age").get.script.nonEmpty shouldBe true - //.map(_.painless(Some(PainlessContext(Processor)))).getOrElse("") should include("...") FIXME + cols.find(_.name == "age").get.processorScript.getOrElse("") should include( + """def param1 = ctx.birthdate; + |def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); + |(param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin + .replaceAll("\n", " ") + ) case _ => fail("Expected CreateTable") } } From 7e0279b2861c532d25ff86d7673712bef745dc0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 12 Dec 2025 08:57:40 +0100 Subject: [PATCH 07/95] add text and keyword types - add ddl processors to remove, rename, set pk, set date_index_name, set column as script - add Es schema to load Es index as a ddl Table with its columns and processors --- .../elastic/client/file/package.scala | 38 +- .../elastic/client/file/FileSourceSpec.scala | 1 + .../softnetwork/elastic/schema/package.scala | 163 ++++++ .../app/softnetwork/elastic/sql/package.scala | 164 +++---- .../elastic/sql/parser/Parser.scala | 37 +- .../elastic/sql/parser/type/package.scala | 7 + .../elastic/sql/query/package.scala | 49 +- .../elastic/sql/schema/package.scala | 462 +++++++++++++++++- .../elastic/sql/serialization/package.scala | 37 ++ .../elastic/sql/time/package.scala | 11 + .../elastic/sql/type/SQLType.scala | 6 + .../elastic/sql/type/SQLTypeUtils.scala | 32 ++ .../elastic/sql/type/SQLTypes.scala | 44 +- .../elastic/sql/StringValueSpec.scala | 18 - .../elastic/sql/parser/ParserSpec.scala | 18 +- 15 files changed, 915 insertions(+), 172 deletions(-) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/schema/package.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala delete mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala index d7aef09e..dd0c6a38 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala @@ -18,17 +18,10 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.stream.scaladsl.Source -import com.fasterxml.jackson.annotation.JsonInclude +import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.fasterxml.jackson.core.{JsonFactory, JsonParser, JsonToken} import com.fasterxml.jackson.databind.node.ObjectNode -import com.fasterxml.jackson.databind.{ - DeserializationFeature, - JsonNode, - ObjectMapper, - SerializationFeature -} -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} @@ -86,33 +79,6 @@ package object file { conf } - /** Jackson ObjectMapper configuration */ - object JacksonConfig { - lazy val objectMapper: ObjectMapper = { - val mapper = new ObjectMapper() - - // Scala module for native support of Scala types - mapper.registerModule(DefaultScalaModule) - - // Java Time module for java.time.Instant, LocalDateTime, etc. - mapper.registerModule(new JavaTimeModule()) - - // Setup for performance - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - - // Ignores null values in serialization - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) - - // Optimizations - mapper.configure(SerializationFeature.INDENT_OUTPUT, false) // No pretty print - mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, false) - - mapper - } - } - /** Base trait for file sources */ sealed trait FileSource { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala index c24cb834..453441c9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.client.file import akka.actor.ActorSystem import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.sql.serialization.JacksonConfig import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.ScalaFutures import org.scalatest.matchers.should.Matchers diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala new file mode 100644 index 00000000..3cd09ecd --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -0,0 +1,163 @@ +package app.softnetwork.elastic + +import app.softnetwork.elastic.sql.Value +import com.fasterxml.jackson.databind.JsonNode + +import scala.jdk.CollectionConverters._ + +package object schema { + final case class EsField( + name: String, + `type`: String, + null_value: Option[Value[_]] = None, + fields: List[EsField] = Nil, + options: Map[String, Value[_]] = Map.empty + ) + + object EsField { + def apply(name: String, node: JsonNode): EsField = { + val tpe = Option(node.get("type")).map(_.asText()).getOrElse("object") + + val nullValue = + Option(node.get("null_value")).flatMap(Value(_)).map(Value(_)) + + val fields = + Option(node.get("fields")) + .map(_.properties().asScala.map { entry => + val name = entry.getKey + val value = entry.getValue + apply(name, value) + }.toList) + .getOrElse(Nil) + + val options = extractOptions(node) + + EsField( + name = name, + `type` = tpe, + null_value = nullValue, + fields = fields, + options = options + ) + + } + + private[this] def extractOptions(node: JsonNode): Map[String, Value[_]] = { + val ignoredKeys = Set("type", "null_value", "fields") + + node + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val value = entry.getValue + + if (ignoredKeys.contains(key)) { + None + } else { + Value(value).map(key -> Value(_)) + } + } + .toMap + } + + } + + final case class EsMapping( + fields: List[EsField] = Nil + ) + + object EsMapping { + def apply(root: JsonNode): EsMapping = { + val fields = Option(root.path("mappings").path("properties")) + .map(_.properties().asScala.map { entry => + val name = entry.getKey + val value = entry.getValue + EsField(name, value) + }.toList) + .getOrElse(Nil) + + EsMapping(fields = fields) + } + } + + final case class EsPartitionMeta( + column: String, + granularity: String // "d", "M", "y", etc. + ) + + final case class EsDdlMeta( + primary_key: List[String] = Nil, + partition_by: Option[EsPartitionMeta] = None, + default_pipeline: Option[String] = None, + final_pipeline: Option[String] = None + ) + + object EsDdlMeta { + def apply(settings: JsonNode): EsDdlMeta = { + val index = settings.path("settings").path("index") + + val defaultPipeline = Option(index.get("default_pipeline")).map(_.asText()) + val finalPipeline = Option(index.get("final_pipeline")).map(_.asText()) + + val ddlNode = index + .path("meta") + .path("ddl") + + if (ddlNode.isMissingNode) + return EsDdlMeta( + default_pipeline = defaultPipeline, + final_pipeline = finalPipeline + ) + + val primaryKey = Option(ddlNode.path("primary_key")) + .filter(_.isArray) + .map(_.elements().asScala.map(_.asText()).toList) + .getOrElse(Nil) + + val partitionBy = + Option(ddlNode.path("partition_by")).filter(_.isObject).map { partitionNode => + val column = Option(partitionNode.get("column")) + .map(_.asText()) + .getOrElse("date") + val granularity = Option(partitionNode.get("granularity")) + .map(_.asText()) + .getOrElse("d") + EsPartitionMeta(column, granularity) + } + + EsDdlMeta( + primary_key = primaryKey, + partition_by = partitionBy, + default_pipeline = defaultPipeline, + final_pipeline = finalPipeline + ) + } + } + + final case class EsIndexMeta( + name: String, + mapping: EsMapping, + ddlMeta: EsDdlMeta, + defaultPipeline: Option[JsonNode] = None // existing "DDL default" pipeline + ) + + object EsIndexMeta { + def apply( + name: String, + settings: JsonNode, + mappings: JsonNode, + defaultPipeline: Option[JsonNode] + ): EsIndexMeta = { + val mapping = EsMapping(mappings) + val ddlMeta = EsDdlMeta(settings) + + EsIndexMeta( + name = name, + mapping = mapping, + ddlMeta = ddlMeta, + defaultPipeline = defaultPipeline + ) + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 146841b9..805febc3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -16,21 +16,14 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.function.aggregate.{ - AggregateFunction, - COUNT, - MAX, - MIN, - WindowFunction -} +import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, COUNT, WindowFunction} import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.function.time.CurrentFunction -import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.parser.{Validation, Validator} import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.JsonNode import java.security.MessageDigest -import java.util.regex.Pattern import scala.reflect.runtime.universe._ import scala.util.Try import scala.util.matching.Regex @@ -275,28 +268,10 @@ package object sql { case object Distinct extends Expr("DISTINCT") with TokenRegex - abstract class Value[+T](val value: T)(implicit ev$1: T => Ordered[T]) + abstract class Value[+T](val value: T) extends Token with PainlessScript with FunctionWithValue[T] { - def choose[R >: T]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - if (values.isEmpty) - None - else - operator match { - case Some(EQ) => values.find(_ == value) - case Some(NE | DIFF) => values.find(_ != value) - case Some(GE) => values.filter(_ >= value).sorted.reverse.headOption - case Some(GT) => values.filter(_ > value).sorted.reverse.headOption - case Some(LE) => values.filter(_ <= value).sorted.headOption - case Some(LT) => values.filter(_ < value).sorted.headOption - case _ => values.headOption - } - } override def painless(context: Option[PainlessContext]): String = SQLTypeUtils.coerce( value match { @@ -314,6 +289,69 @@ package object sql { override def nullable: Boolean = false } + object Value { + def apply[R: TypeTag, T <: Value[R]](value: Any): Value[_] = { + value match { + case null => Null + case b: Boolean => BooleanValue(b) + case c: Char => CharValue(c) + case s: String => StringValue(s) + case b: Byte => ByteValue(b) + case s: Short => ShortValue(s) + case i: Int => IntValue(i) + case l: Long => LongValue(l) + case f: Float => FloatValue(f) + case d: Double => DoubleValue(d) + case a: Array[T] => + val values = a.toSeq.map(apply) + values.headOption match { + case Some(_: StringValue) => + StringValues(values.asInstanceOf[Seq[StringValue]]).asInstanceOf[Values[R, T]] + case Some(_: ByteValue) => + ByteValues(values.asInstanceOf[Seq[ByteValue]]).asInstanceOf[Values[R, T]] + case Some(_: ShortValue) => + ShortValues(values.asInstanceOf[Seq[ShortValue]]).asInstanceOf[Values[R, T]] + case Some(_: IntValue) => + IntValues(values.asInstanceOf[Seq[IntValue]]).asInstanceOf[Values[R, T]] + case Some(_: LongValue) => + LongValues(values.asInstanceOf[Seq[LongValue]]).asInstanceOf[Values[R, T]] + case Some(_: FloatValue) => + FloatValues(values.asInstanceOf[Seq[FloatValue]]).asInstanceOf[Values[R, T]] + case Some(_: DoubleValue) => + DoubleValues(values.asInstanceOf[Seq[DoubleValue]]).asInstanceOf[Values[R, T]] + case _ => throw new IllegalArgumentException("Unsupported Values type") + } + case other => StringValue(other.toString) + } + } + + def apply(node: JsonNode): Option[Any] = { + node match { + case n if n.isNull => Some(null) + case n if n.isTextual => Some(n.asText()) + case n if n.isBoolean => Some(n.asBoolean()) + case n if n.isShort => Some(node.asInstanceOf[Short]) + case n if n.isInt => Some(n.asInt()) + case n if n.isLong => Some(n.asLong()) + case n if n.isDouble => Some(n.asDouble()) + case n if n.isFloat => Some(node.asInstanceOf[Float]) + case n if n.isArray => + import scala.jdk.CollectionConverters._ + val arr = n + .elements() + .asScala + .flatMap(apply) + .toList + Some(arr) + case n if n.isObject => + // The raw JSON object is stored as a string. TODO consider mapping to a Map[String, Any] + Some(n.toString) + case _ => + None + } + } + } + case object Null extends Value[Null](null) with TokenRegex { override def sql: String = "NULL" override def painless(context: Option[PainlessContext]): String = "null" @@ -340,47 +378,11 @@ package object sql { case class StringValue(override val value: String) extends Value[String](value) { override def sql: String = s"""'$value'""" - import SQLImplicits._ - private lazy val pattern: Pattern = value.pattern - def like: Seq[String] => Boolean = { - _.exists { pattern.matcher(_).matches() } - } - def eq: Seq[String] => Boolean = { - _.exists { _.contentEquals(value) } - } - def ne: Seq[String] => Boolean = { - _.forall { !_.contentEquals(value) } - } - override def choose[R >: String]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - operator match { - case Some(EQ) => values.find(v => v.toString contentEquals value) - case Some(NE | DIFF) => values.find(v => !(v.toString contentEquals value)) - case Some(LIKE | RLIKE) => values.find(v => pattern.matcher(v.toString).matches()) - case None => Some(values.mkString(separator)) - case _ => super.choose(values, operator, separator) - } - } override def baseType: SQLType = SQLTypes.Varchar } - sealed abstract class NumericValue[T: Numeric](override val value: T)(implicit - ev$1: T => Ordered[T] - ) extends Value[T](value) { + sealed abstract class NumericValue[T: Numeric](override val value: T) extends Value[T](value) { override def sql: String = value.toString - override def choose[R >: T]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - operator match { - case None => if (values.isEmpty) None else Some(values.max) - case _ => super.choose(values, operator, separator) - } - } private[this] val num: Numeric[T] = implicitly[Numeric[T]] def toDouble: Double = num.toDouble(value) def toEither: Either[Long, Double] = value match { @@ -390,14 +392,6 @@ package object sql { case f: Float => Right(f.toDouble) case _ => Right(toDouble) } - def max: Seq[T] => T = x => Try(x.max).getOrElse(num.zero) - def min: Seq[T] => T = x => Try(x.min).getOrElse(num.zero) - def eq: Seq[T] => Boolean = { - _.exists { _ == value } - } - def ne: Seq[T] => Boolean = { - _.forall { _ != value } - } override def baseType: SQLNumeric = SQLTypes.Numeric } @@ -505,8 +499,7 @@ package object sql { extends FromTo(from, to) sealed abstract class Values[+R: TypeTag, +T <: Value[R]](val values: Seq[T]) - extends Token - with PainlessScript { + extends Value[Seq[T]](values) { override def sql = s"(${values.map(_.sql).mkString(",")})" override def painless(context: Option[PainlessContext]): String = s"[${values.map(_.painless(context)).mkString(",")}]" @@ -564,25 +557,6 @@ package object sql { override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Double) } - def choose[T]( - values: Seq[T], - criteria: Option[Criteria], - function: Option[Function] = None - )(implicit ev$1: T => Ordered[T]): Option[T] = { - criteria match { - case Some(GenericExpression(_, operator, value: Value[T] @unchecked, _)) => - value.choose[T](values, Some(operator)) - case _ => - function match { - case Some(MIN) => Some(values.min) - case Some(MAX) => Some(values.max) - // FIXME case Some(SQLSum) => Some(values.sum) - // FIXME case Some(SQLAvg) => Some(values.sum / values.length ) - case _ => values.headOption - } - } - } - def toRegex(value: String): String = { value.replaceAll("%", ".*").replaceAll("_", ".") } @@ -794,7 +768,7 @@ package object sql { lazy val processParamName: String = { if (path.nonEmpty) - s"ctx.$path" + s"ctx['$path']" else "" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index add401ee..6a4e091f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.operator._ @@ -29,7 +30,7 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition} +import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition, DdlScriptProcessor} import app.softnetwork.elastic.sql.time.TimeUnit import scala.language.implicitConversions @@ -114,12 +115,40 @@ object Parser identifierWithFunction) ~ end ^^ { case _ ~ _ ~ s ~ _ => s } def column: PackratParser[DdlColumn] = - ident ~ sql_type ~ (script | multiFields) ~ notNull ~ defaultVal ~ (options | success( + ident ~ extension_type ~ (script | multiFields) ~ notNull ~ defaultVal ~ (options | success( Map.empty[String, Value[_]] )) ^^ { case name ~ dt ~ mfs ~ nn ~ dv ~ opts => mfs match { case script: PainlessScript => - DdlColumn(name, dt, Some(script), Nil, nn, dv, opts) + val ctx = PainlessContext(Processor) + val scr = script.painless(Some(ctx)) + val temp = s"$ctx$scr" + val ret = + temp.split(";") match { + case Array(single) if single.trim.startsWith("return ") => + val stripReturn = single.trim.stripPrefix("return ").trim + s"ctx.$name = $stripReturn" + case multiple => + val last = multiple.last.trim + val temp = multiple.dropRight(1) :+ s" ctx.$name = $last" + temp.mkString(";") + } + DdlColumn( + name, + dt, + Some( + DdlScriptProcessor( + script = script.sql, + column = name, + dataType = dt, + source = ret + ) + ), + Nil, + nn, + dv, + opts + ) case cols: List[DdlColumn] => DdlColumn(name, dt, None, cols, nn, dv, opts) } @@ -223,7 +252,7 @@ object Parser } def setColumnType: PackratParser[AlterColumnType] = - alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ sql_type ^^ { + alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ extension_type ^^ { case ie ~ name ~ _ ~ newType => AlterColumnType(name, newType, ifExists = ie) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index fac388a8..4e5ee847 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -99,5 +99,12 @@ package object `type` { def sql_type: PackratParser[SQLType] = char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type | array_type + def text_type: PackratParser[SQLTypes.Text.type] = + "(?i)text".r ^^ (_ => SQLTypes.Text) + + def keyword_type: PackratParser[SQLTypes.Keyword.type] = + "(?i)keyword".r ^^ (_ => SQLTypes.Keyword) + + def extension_type: PackratParser[SQLType] = sql_type | text_type | keyword_type } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 641b9266..641c08fb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -17,9 +17,21 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType -import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition} +import app.softnetwork.elastic.sql.schema.{ + DdlColumn, + DdlDefaultValueProcessor, + DdlPartition, + DdlPipeline, + DdlPipelineType, + DdlProcessor, + DdlRemoveProcessor, + DdlRenameProcessor, + DdlTable +} import app.softnetwork.elastic.sql.function.aggregate.WindowFunction +import java.time.Instant + package object query { sealed trait Statement extends Token @@ -337,6 +349,15 @@ package object query { case Right(cols) => cols } } + + lazy val ddlTable: DdlTable = DdlTable( + name = table, + columns = columns.toList, + primaryKey = primaryKey, + partitionBy = partitionBy + ) + + lazy val ddlPipeline: DdlPipeline = ddlTable.ddlPipeline } case class AlterTable(table: String, ifExists: Boolean, statements: List[AlterTableStatement]) @@ -351,9 +372,16 @@ package object query { } s"ALTER TABLE $table$ifExistsClause $statementsSql" } + + lazy val ddlProcessors: Seq[DdlProcessor] = statements.flatMap(_.ddlProcessor) + + lazy val pipeline: DdlPipeline = + DdlPipeline(s"alter-$table-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) } - sealed trait AlterTableStatement extends Token + sealed trait AlterTableStatement extends Token { + def ddlProcessor: Option[DdlProcessor] = None + } case class AddColumn(column: DdlColumn, ifNotExists: Boolean = false) extends AlterTableStatement { override def sql: String = { @@ -366,9 +394,11 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"DROP COLUMN$ifExistsClause $columnName" } + override def ddlProcessor: Option[DdlProcessor] = Some(DdlRemoveProcessor(sql, columnName)) } case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { override def sql: String = s"RENAME COLUMN $oldName TO $newName" + override def ddlProcessor: Option[DdlProcessor] = Some(DdlRenameProcessor(oldName, newName)) } case class AlterColumnOptions( columnName: String, @@ -398,6 +428,14 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"ALTER COLUMN$ifExistsClause $columnName SET DEFAULT $defaultValue" } + override def ddlProcessor: Option[DdlProcessor] = + Some( + DdlDefaultValueProcessor( + sql, + columnName, + defaultValue + ) + ) } case class DropColumnDefault(columnName: String, ifExists: Boolean = false) extends AlterTableStatement { @@ -405,6 +443,13 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"ALTER COLUMN$ifExistsClause $columnName DROP DEFAULT" } + override def ddlProcessor: Option[DdlProcessor] = + Some( + DdlRemoveProcessor( + sql, + columnName + ) + ) } case class AlterColumnNotNull(columnName: String, ifExists: Boolean = false) extends AlterTableStatement { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index d5f6cc18..2409d3b6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,16 +16,352 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.PainlessContextType.Processor -import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.schema.{EsField, EsIndexMeta} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.serialization.JacksonConfig import app.softnetwork.elastic.sql.time.TimeUnit +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.ObjectNode + +import scala.language.implicitConversions +import scala.jdk.CollectionConverters._ package object schema { + val mapper: ObjectMapper = JacksonConfig.objectMapper + + sealed trait DdlProcessorType { + def name: String + } + + object DdlProcessorType { + case object Script extends DdlProcessorType { + def name: String = "script" + } + case object Rename extends DdlProcessorType { + def name: String = "rename" + } + case object Remove extends DdlProcessorType { + def name: String = "remove" + } + case object Set extends DdlProcessorType { + def name: String = "set" + } + case object DateIndexName extends DdlProcessorType { + def name: String = "date_index_name" + } + } + + sealed trait DdlProcessor extends Token { + def column: String + def ignoreFailure: Boolean + final def node: ObjectNode = { + val node = mapper.createObjectNode() + val props = mapper.createObjectNode() + for ((key, value) <- properties) { + value match { + case v: String => props.put(key, v) + case v: Boolean => props.put(key, v) + case v: Int => props.put(key, v) + case v: Long => props.put(key, v) + case v: Double => props.put(key, v) + case v: ObjectNode => props.set(key, v) + case v: Seq[_] => + val arrayNode = mapper.createArrayNode() + v.foreach { + case s: String => arrayNode.add(s) + case b: Boolean => arrayNode.add(b) + case i: Int => arrayNode.add(i) + case l: Long => arrayNode.add(l) + case d: Double => arrayNode.add(d) + case o: ObjectNode => arrayNode.add(o) + case _ => + } + props.set(key, arrayNode) + case _ => + } + } + node.set(processorType.name, props) + node + } + def json: String = node.toString + def processorType: DdlProcessorType + def description: String = sql + def name: String = processorType.name + def properties: Map[String, Any] + } + + case class DdlScriptProcessor( + script: String, + column: String, + dataType: SQLType, + source: String, + ignoreFailure: Boolean = true + ) extends DdlProcessor { + override def sql: SQL = s"$column $dataType SCRIPT AS ($script)" + + override def baseType: SQLType = dataType + + def processorType: DdlProcessorType = DdlProcessorType.Script + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "lang" -> "painless", + "source" -> source, + "ignore_failure" -> ignoreFailure + ) + + } + + case class DdlRenameProcessor( + column: String, + newName: String, + ignoreFailure: Boolean = true, + ignoreMissing: Boolean = true + ) extends DdlProcessor { + def processorType: DdlProcessorType = DdlProcessorType.Rename + + def sql: String = s"$column RENAME TO $newName" + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "field" -> column, + "target_field" -> newName, + "ignore_failure" -> ignoreFailure, + "ignore_missing" -> ignoreMissing + ) + + } + + case class DdlRemoveProcessor( + sql: String, + column: String, + ignoreFailure: Boolean = true, + ignoreMissing: Boolean = true + ) extends DdlProcessor { + def processorType: DdlProcessorType = DdlProcessorType.Remove + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "field" -> column, + "ignore_failure" -> ignoreFailure, + "ignore_missing" -> ignoreMissing + ) + + } + + case class DdlPrimaryKeyProcessor( + sql: String, + column: String, + value: Set[String], + ignoreFailure: Boolean = false, + ignoreEmptyValue: Boolean = false, + separator: String = "|" + ) extends DdlProcessor { + def processorType: DdlProcessorType = DdlProcessorType.Set + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "field" -> column, + "value" -> value.mkString("{{", separator, "}}"), + "ignore_failure" -> ignoreFailure, + "ignore_empty_value" -> ignoreEmptyValue + ) + + } + + case class DdlDefaultValueProcessor( + sql: String, + column: String, + value: Value[_], + ignoreFailure: Boolean = true + ) extends DdlProcessor { + def processorType: DdlProcessorType = DdlProcessorType.Set + + def _if: String = { + if (column.contains(".")) + s"""ctx.${column.split(".").mkString("?.")} == null""" + else + s"""ctx.$column == null""" + } + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "field" -> column, + "value" -> value.value, + "ignore_failure" -> ignoreFailure, + "if" -> _if + ) + } + + case class DdlDateIndexNameProcessor( + sql: String, + column: String, + dateRounding: String, + dateFormats: List[String], + prefix: String, + separator: String = "-", + ignoreFailure: Boolean = true + ) extends DdlProcessor { + def processorType: DdlProcessorType = DdlProcessorType.DateIndexName + + override def properties: Map[SQL, Any] = Map( + "description" -> description, + "field" -> column, + "date_rounding" -> dateRounding, + "date_formats" -> dateFormats, + "index_name_prefix" -> prefix, + "separator" -> separator, + "ignore_failure" -> ignoreFailure + ) + + } + + implicit def primaryKeyToDdlProcessor( + primaryKey: List[String] + ): Seq[DdlProcessor] = { + if (primaryKey.nonEmpty) { + Seq( + DdlPrimaryKeyProcessor( + sql = s"PRIMARY KEY (${primaryKey.mkString(", ")})", + column = "_id", + value = primaryKey.toSet + ) + ) + } else { + Nil + } + } + + object DdlProcessor { + private val ScriptDescRegex = + """^\s*([a-zA-Z0-9_]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r + + def apply(node: JsonNode): Option[DdlProcessor] = { + val processorType = node.fieldNames().next() // "set", "script", "date_index_name", etc. + val props = node.get(processorType) + + processorType match { + case "set" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val valueNode = props.get("value") + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + if (field == "_id" && desc.startsWith("PRIMARY KEY")) { + // DdlPrimaryKeyProcessor + // description: "PRIMARY KEY (id)" + val inside = desc.stripPrefix("PRIMARY KEY").trim.stripPrefix("(").stripSuffix(")") + val cols = inside.split(",").map(_.trim).filter(_.nonEmpty).toSet + Some( + DdlPrimaryKeyProcessor( + sql = desc, + column = "_id", + value = cols, + ignoreFailure = ignoreFailure + ) + ) + } else if (desc.startsWith(s"$field DEFAULT")) { + Some( + DdlDefaultValueProcessor( + sql = desc, + column = field, + value = Value(valueNode.asText()), + ignoreFailure = ignoreFailure + ) + ) + } else { + None + } + + case "script" => + val desc = props.get("description").asText() + val lang = props.get("lang").asText() + require(lang == "painless", s"Only painless supported, got $lang") + val source = props.get("source").asText() + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + desc match { + case ScriptDescRegex(col, dataType, script) => + Some( + DdlScriptProcessor( + script = script, + column = col, + dataType = SQLTypes(dataType), + source = source, + ignoreFailure = ignoreFailure + ) + ) + case _ => + None + } + + case "date_index_name" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val rounding = props.get("date_rounding").asText() + val formats = Option(props.get("date_formats")) + .map(_.elements().asScala.toList.map(_.asText())) + .getOrElse(Nil) + val prefix = props.get("index_name_prefix").asText() + + Some( + DdlDateIndexNameProcessor( + sql = desc, + column = field, + dateRounding = rounding, + dateFormats = formats, + prefix = prefix + ) + ) + + case _ => None + } + } + } + + sealed trait DdlPipelineType { + def name: String + } + + object DdlPipelineType { + case object Default extends DdlPipelineType { + def name: String = "DEFAULT" + } + case object Final extends DdlPipelineType { + def name: String = "FINAL" + } + case object Custom extends DdlPipelineType { + def name: String = "CUSTOM" + } + } + + case class DdlPipeline( + name: String, + ddlPipelineType: DdlPipelineType, + ddlProcessors: Seq[DdlProcessor] + ) extends Token { + def sql: String = + s"CREATE OR REPLACE ${ddlPipelineType.name} PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql).mkString(", ")})" + + def node: ObjectNode = { + val node = mapper.createObjectNode() + val processorsNode = mapper.createArrayNode() + ddlProcessors.foreach { processor => + processorsNode.add(processor.node) + } + node.put("description", sql) + node.set("processors", processorsNode) + node + } + + def json: String = node.toString + } + case class DdlColumn( name: String, dataType: SQLType, - script: Option[PainlessScript] = None, + script: Option[DdlScriptProcessor] = None, multiFields: List[DdlColumn] = Nil, notNull: Boolean = false, defaultValue: Option[Value[_]] = None, @@ -48,15 +384,27 @@ package object schema { s"$name $dataType$fieldsOpt$scriptOpt$notNullOpt$defaultOpt$opts" } - def processorScript: Option[String] = { - script.map { s => - val context = PainlessContext(Processor) - val script = s.painless(Some(context)) - s"$context$script" - } - } + def ddlProcessors: Seq[DdlProcessor] = script.toSeq ++ + defaultValue.map { dv => + DdlDefaultValueProcessor( + sql = s"$name DEFAULT $dv", + column = name, + value = dv + ) + }.toSeq } + object DdlColumn { + def apply(field: EsField): DdlColumn = { + DdlColumn( + name = field.name, + dataType = SQLTypes(field), + multiFields = field.fields.map(apply), + defaultValue = field.null_value, + options = field.options + ) + } + } case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { def sql: String = s"PARTITION BY $column ($granularity)" @@ -73,6 +421,15 @@ package object schema { List("yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd HH:mm:ss") case _ => List.empty } + + def ddlProcessor(table: DdlTable): DdlDateIndexNameProcessor = + DdlDateIndexNameProcessor( + sql, + column, + dateRounding, + dateFormats, + prefix = s"${table.name}-" + ) } case class DdlColumnNotFound(column: String, table: String) @@ -98,6 +455,9 @@ package object schema { s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}" } + def ddlProcessors: Seq[DdlProcessor] = + columns.flatMap(_.ddlProcessors) ++ partitionBy.map(_.ddlProcessor(this)).toSeq ++ primaryKey + def merge(statements: Seq[AlterTableStatement]): DdlTable = { statements.foldLeft(this) { (table, statement) => statement match { @@ -194,6 +554,7 @@ package object schema { case _ => table } } + } override def validate(): Either[SQL, Unit] = { @@ -212,6 +573,87 @@ package object schema { } if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) } + + lazy val ddlPipeline: DdlPipeline = DdlPipeline( + name = s"${name}_ddl_default_pipeline", + ddlPipelineType = DdlPipelineType.Default, + ddlProcessors = ddlProcessors + ) } + object DdlTable { + def apply(indexMeta: EsIndexMeta): DdlTable = { + // 1. Columns from the mapping + val initialCols: Map[String, DdlColumn] = + indexMeta.mapping.fields.map { field => + val name = field.name + name -> DdlColumn( + name = name, + dataType = SQLTypes(field), + script = None, + multiFields = field.fields.map(DdlColumn(_)), + notNull = false, // TODO add required + defaultValue = field.null_value, + options = field.options + ) + }.toMap + + // 2. PK + partition + pipelines from meta + var primaryKey: List[String] = indexMeta.ddlMeta.primary_key + var partitionBy: Option[DdlPartition] = indexMeta.ddlMeta.partition_by.map { p => + val granularity = TimeUnit(p.granularity) + DdlPartition(p.column, granularity) + } + val defaultPipelineName = indexMeta.ddlMeta.default_pipeline + val finalPipelineName = indexMeta.ddlMeta.final_pipeline + + // 3. Enrichment from the default pipeline (if provided) + val enrichedCols = scala.collection.mutable.Map.from(initialCols) + + indexMeta.defaultPipeline.foreach { pipeline => + val processorsNode = pipeline.get("processors") + if (processorsNode != null && processorsNode.isArray) { + val processors: Seq[DdlProcessor] = + processorsNode.elements().asScala.toSeq.flatMap(DdlProcessor(_)) + + processors.foreach { + case p: DdlScriptProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(script = Some(p))) + } + + case p: DdlDefaultValueProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) + } + + case p: DdlDateIndexNameProcessor => + if (partitionBy.isEmpty) { + val granularity = TimeUnit(p.dateRounding) + partitionBy = Some(DdlPartition(p.column, granularity)) + } + + case p: DdlPrimaryKeyProcessor => + if (primaryKey.isEmpty) { + primaryKey = p.value.toList + } + + case _ => // ignore others (rename/remove...) ou gère-les si tu veux les remonter en DDL + } + } + } + + // 4. Construction finale du DdlTable + DdlTable( + name = indexMeta.name, + columns = enrichedCols.values.toList.sortBy(_.name), + primaryKey = primaryKey, + partitionBy = partitionBy, + defaultPipeline = defaultPipelineName, + finalPipeline = finalPipelineName + ) + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala new file mode 100644 index 00000000..1ab8a9dc --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -0,0 +1,37 @@ +package app.softnetwork.elastic.sql + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature} +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.DefaultScalaModule + +package object serialization { + + /** Jackson ObjectMapper configuration */ + object JacksonConfig { + lazy val objectMapper: ObjectMapper = { + val mapper = new ObjectMapper() + + // Scala module for native support of Scala types + mapper.registerModule(DefaultScalaModule) + + // Java Time module for java.time.Instant, LocalDateTime, etc. + mapper.registerModule(new JavaTimeModule()) + + // Setup for performance + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + + // Ignores null values in serialization + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + + // Optimizations + mapper.configure(SerializationFeature.INDENT_OUTPUT, false) // No pretty print + mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, false) + + mapper + } + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index ec373753..1efa5099 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -113,6 +113,17 @@ package object time { sealed trait FixedUnit extends TimeUnit object TimeUnit { + def apply(script: String): TimeUnit = script match { + case "y" => YEARS + case "M" => MONTHS + case "w" => WEEKS + case "d" => DAYS + case "H" => HOURS + case "m" => MINUTES + case "s" => SECONDS + case _ => throw new IllegalArgumentException(s"Invalid time unit script: $script") + } + case object YEARS extends Expr("YEAR") with CalendarUnit { override def script: Option[String] = Some("y") } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index a61d4f57..e5b273c1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -50,3 +50,9 @@ trait SQLBool extends SQLType trait SQLArray extends SQLType { def elementType: SQLType } trait SQLStruct extends SQLType + +sealed trait EsqlType extends SQLType + +trait EsqlText extends EsqlType with SQLVarchar + +trait EsqlKeyword extends EsqlType with SQLVarchar diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index eafb23b3..a0b35d81 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -21,6 +21,38 @@ import app.softnetwork.elastic.sql.`type`.SQLTypes._ object SQLTypeUtils { + def painlessType(sqlType: SQLType): String = sqlType match { + case TinyInt => "byte" + case SmallInt => "short" + case Int => "int" + case BigInt => "long" + case Double => "double" + case Real => "float" + case Numeric => "java.math.BigDecimal" + case Varchar => "String" + case Boolean => "boolean" + case Date => "LocalDate" + case Time => "LocalTime" + case DateTime => "LocalDateTime" + case Timestamp => "ZonedDateTime" + case Temporal => "ZonedDateTime" + case Array(inner) => + inner match { + case TinyInt => "byte[]" + case SmallInt => "short[]" + case Int => "int[]" + case BigInt => "long[]" + case Double => "double[]" + case Real => "float[]" + case Boolean => "boolean[]" + case _ => s"java.util.List<${painlessType(inner)}>" + } + case Struct => "Map" + case Any => "Object" + case Null => "Object" + case _ => "Object" + } + def matches(out: SQLType, in: SQLType): Boolean = out.typeId == in.typeId || (out.typeId == Temporal.typeId && Set( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 52cca678..fa142132 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -16,6 +16,8 @@ package app.softnetwork.elastic.sql.`type` +import app.softnetwork.elastic.schema.EsField + object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } @@ -41,12 +43,52 @@ object SQLTypes { case object Char extends SQLChar { val typeId = "CHAR" } case object Varchar extends SQLVarchar { val typeId = "VARCHAR" } + case object Text extends EsqlText { val typeId = "TEXT" } + case object Keyword extends EsqlKeyword { val typeId = "KEYWORD" } case object Boolean extends SQLBool { val typeId = "BOOLEAN" } case class Array(elementType: SQLType) extends SQLArray { - val typeId = s"array<${elementType.typeId}>" + val typeId = s"ARRAY<${elementType.typeId}>" } case object Struct extends SQLStruct { val typeId = "STRUCT" } + + def apply(typeName: String): SQLType = typeName.toLowerCase match { + case "null" => Null + case "boolean" => Boolean + case "int" => Int + case "long" | "bigint" => BigInt + case "short" | "smallint" => SmallInt + case "byte" | "tinyint" => TinyInt + case "keyword" => Keyword + case "text" => Text + case "varchar" => Varchar + case "datetime" | "timestamp" => DateTime + case "date" => Date + case "time" => Time + case "double" => Double + case "float" | "real" => Real + case "object" | "struct" => Struct + case "nested" | "array" => Array(Struct) + case _ => Any + } + + def apply(field: EsField): SQLType = field.`type` match { + case "null" => Null + case "boolean" => Boolean + case "integer" => Int + case "long" => BigInt + case "short" => SmallInt + case "byte" => TinyInt + case "keyword" => Keyword + case "text" => Text + case "date" => DateTime + case "double" => Double + case "float" => Real + case "object" => Struct + case "nested" => Array(Struct) + case _ => Any + } + } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala deleted file mode 100644 index 9bbf49cb..00000000 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala +++ /dev/null @@ -1,18 +0,0 @@ -package app.softnetwork.elastic.sql - -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** Created by smanciot on 17/02/17. - */ -class StringValueSpec extends AnyFlatSpec with Matchers { - - "SQLLiteral" should "perform sql like" in { - val l = StringValue("%dummy%") - l.like(Seq("dummy")) should ===(true) - l.like(Seq("aa dummy")) should ===(true) - l.like(Seq("dummy bbb")) should ===(true) - l.like(Seq("aaa dummy bbb")) should ===(true) - l.like(Seq("dummY")) should ===(false) - } -} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 316bd450..5130e168 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,7 +1,5 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.{Identifier, PainlessContext} -import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.DdlPartition @@ -943,7 +941,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case CreateTable( + case ct @ CreateTable( "users", Right(cols), true, @@ -956,12 +954,20 @@ class ParserSpec extends AnyFlatSpec with Matchers { cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") cols.find(_.name == "birthdate").get.dataType.typeId should include("DATE") cols.find(_.name == "age").get.script.nonEmpty shouldBe true - cols.find(_.name == "age").get.processorScript.getOrElse("") should include( - """def param1 = ctx.birthdate; + cols + .find(_.name == "age") + .get + .script + .map(p => p.source) + .getOrElse("") should include( + """def param1 = ctx['birthdate']; |def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); - |(param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin + |ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin .replaceAll("\n", " ") ) + val json = ct.ddlTable.ddlPipeline.json + print(json) + json shouldBe "{\"description\":\"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), PARTITION BY birthdate (DAY), PRIMARY KEY (id))\",\"processors\":[{\"set\":{\"field\":\"name\",\"if\":\"ctx.name == null\",\"description\":\"name DEFAULT 'anonymous'\",\"ignore_failure\":true,\"value\":\"anonymous\"}},{\"script\":{\"description\":\"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))\",\"lang\":\"painless\",\"source\":\"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)\",\"ignore_failure\":true}},{\"date_index_name\":{\"field\":\"birthdate\",\"index_name_prefix\":\"users-\",\"date_formats\":[\"yyyy-MM-dd\"],\"ignore_failure\":true,\"date_rounding\":\"d\",\"description\":\"PARTITION BY birthdate (DAY)\",\"separator\":\"-\"}},{\"set\":{\"field\":\"_id\",\"description\":\"PRIMARY KEY (id)\",\"ignore_failure\":false,\"ignore_empty_value\":false,\"value\":\"{{id}}\"}}]}" case _ => fail("Expected CreateTable") } } From 111c091c4fd6f14ca215b3d2d2617ff8ddaa08da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 13 Dec 2025 12:44:20 +0100 Subject: [PATCH 08/95] update index creation api adding optional mappings and aliases --- .../client/ElasticClientDelegator.scala | 15 ++- .../elastic/client/ElasticClientHelpers.scala | 10 ++ .../elastic/client/IndicesApi.scala | 44 +++++++- .../elastic/client/MappingApi.scala | 41 +++---- .../elastic/client/NopeClientApi.scala | 4 +- .../client/metrics/MetricsElasticClient.scala | 9 +- .../elastic/client/AliasApiSpec.scala | 8 +- .../elastic/client/IndicesApiSpec.scala | 103 ++++++++++-------- .../elastic/client/MappingApiSpec.scala | 4 +- .../elastic/client/SettingsApiSpec.scala | 28 +++-- documentation/client/indices.md | 8 +- .../elastic/client/jest/JestIndicesApi.scala | 6 +- .../client/rest/RestHighLevelClientApi.scala | 11 +- .../client/rest/RestHighLevelClientApi.scala | 10 +- .../elastic/client/java/JavaClientApi.scala | 13 ++- .../elastic/client/java/JavaClientApi.scala | 13 ++- .../elastic/client/ElasticClientSpec.scala | 24 ++-- .../elastic/client/MockElasticClientApi.scala | 4 +- .../elastic/client/WindowFunctionSpec.scala | 2 +- 19 files changed, 246 insertions(+), 111 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 55caf5ed..6346b23b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -80,8 +80,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * true if the index was created successfully, false otherwise */ - override def createIndex(index: String, settings: String): ElasticResult[Boolean] = - delegate.createIndex(index, settings) + override def createIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[String] + ): ElasticResult[Boolean] = + delegate.createIndex(index, settings, mappings, aliases) /** Delete an index with the provided name. * @@ -145,9 +150,11 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = - delegate.executeCreateIndex(index, settings) + delegate.executeCreateIndex(index, settings, None, Nil) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = delegate.executeDeleteIndex(index) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 2ce3a1b2..9a6f4bac 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -176,6 +176,16 @@ trait ElasticClientHelpers { validateJson("validateJsonSettings", settings) } + /** Validate the JSON mappings. + * @param mappings + * mappings in JSON format + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateJsonMappings(mappings: String): Option[ElasticError] = { + validateJson("validateJsonMappings", mappings) + } + /** Validate the alias name. Aliases follow the same rules as indexes. * @param alias * alias name to validate diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index e3032dc9..9b31fc3c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -84,10 +84,19 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => * - the name of the index to create * @param settings * - the settings to apply to the index (default is defaultSettings) + * @param mappings + * - optional mappings to apply to the index + * @param aliases + * - optional aliases to apply to the index * @return * true if the index was created successfully, false otherwise */ - def createIndex(index: String, settings: String = defaultSettings): ElasticResult[Boolean] = { + def createIndex( + index: String, + settings: String = defaultSettings, + mappings: Option[String] = None, + aliases: Seq[String] = Nil + ): ElasticResult[Boolean] = { validateIndexName(index) match { case Some(error) => return ElasticFailure( @@ -113,9 +122,33 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => case None => // OK } + mappings.map(validateJsonMappings) match { + case Some(Some(error)) => + return ElasticFailure( + error.copy( + operation = Some("createIndex"), + statusCode = Some(400), + message = s"Invalid mappings: ${error.message}" + ) + ) + case _ => // OK + } + + aliases.flatMap(alias => validateAliasName(alias)) match { + case error :: _ => + return ElasticFailure( + error.copy( + operation = Some("createIndex"), + statusCode = Some(400), + message = s"Invalid alias: ${error.message}" + ) + ) + case Nil => // OK + } + logger.info(s"Creating index '$index' with settings: $settings") - executeCreateIndex(index, settings) match { + executeCreateIndex(index, settings, mappings, aliases) match { case success @ ElasticSuccess(true) => logger.info(s"✅ Index '$index' created successfully") success @@ -397,7 +430,12 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => // METHODS TO IMPLEMENT // ======================================================================== - private[client] def executeCreateIndex(index: String, settings: String): ElasticResult[Boolean] + private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[String] + ): ElasticResult[Boolean] private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index d274bac1..0c2b4114 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -158,7 +158,19 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w indexExists(index).flatMap { case false => // Scenario 1: Index doesn't exist - createIndexWithMapping(index, mapping, settings) + createIndex(index, settings, Some(mapping), Nil).flatMap { + case true => + logger.info(s"✅ Index '$index' created with mapping successfully") + ElasticResult.success(true) + case false => + ElasticResult.failure( + ElasticError( + message = s"Failed to create index '$index' with mapping", + index = Some(index), + operation = Some("updateMapping") + ) + ) + } case true => // Check if mapping needs update @@ -177,27 +189,6 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w } } - /** Create a new index with the given mapping. - */ - private def createIndexWithMapping( - index: String, - mapping: String, - settings: String - ): ElasticResult[Boolean] = { - logger.info(s"Creating new index '$index' with mapping") - - for { - _ <- createIndex(index, settings) - .filter(_ == true, s"Failed to create index '$index'") - .logSuccess(logger, _ => s"✅ Index '$index' created successfully") - - _ <- setMapping(index, mapping) - .filter(_ == true, s"Failed to set mapping for index '$index'") - .logSuccess(logger, _ => s"✅ Mapping for index '$index' set successfully") - - } yield true - } - private def migrateMappingWithRollback( index: String, newMapping: String, @@ -266,7 +257,7 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w for { // Create temp index - _ <- createIndex(tempIndex, settings) + _ <- createIndex(tempIndex, settings, None, Nil) .filter(_ == true, s"❌ Failed to create temp index '$tempIndex'") _ <- setMapping(tempIndex, mapping) @@ -281,7 +272,7 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w .filter(_ == true, s"❌ Failed to delete original index") // Recreate original with new mapping - _ <- createIndex(index, settings) + _ <- createIndex(index, settings, None, Nil) .filter(_ == true, s"❌ Failed to recreate original index") _ <- setMapping(index, mapping) @@ -323,7 +314,7 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w } // Recreate with original settings and mapping - _ <- createIndex(index, originalSettings) + _ <- createIndex(index, originalSettings, None, Nil) .filter(_ == true, s"❌ Rollback: Failed to recreate index") _ <- setMapping(index, originalMapping) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index 279301aa..536ec097 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -104,7 +104,9 @@ trait NopeClientApi extends ElasticClientApi { override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ElasticResult.success(false) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index c74d1fb4..43b69045 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -87,9 +87,14 @@ class MetricsElasticClient( // ==================== IndicesApi ==================== - override def createIndex(index: String, settings: String): ElasticResult[Boolean] = { + override def createIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[String] + ): ElasticResult[Boolean] = { measureResult("createIndex", Some(index)) { - delegate.createIndex(index, settings) + delegate.createIndex(index, settings, mappings, aliases) } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index 81bc6902..3d38d302 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -71,7 +71,9 @@ class AliasApiSpec // Other required methods override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? @@ -1656,7 +1658,9 @@ class AliasApiSpec ): ElasticResult[Boolean] = ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: JSONQuery, + mappings: Option[JSONQuery], + aliases: Seq[JSONQuery] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index 4217295a..bd42e076 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -37,7 +37,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = { executeCreateIndexResult } @@ -144,7 +146,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result shouldBe ElasticSuccess(true) @@ -161,7 +163,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", customSettings) + val result = indicesApi.createIndex("my-index", customSettings, None, Nil) // Then result.isSuccess shouldBe true @@ -176,7 +178,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(false) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result shouldBe ElasticSuccess(false) @@ -191,7 +193,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -202,7 +204,7 @@ class IndicesApiSpec "reject invalid index name" in { // When - val result = indicesApi.createIndex("INVALID") + val result = indicesApi.createIndex("INVALID", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -216,7 +218,7 @@ class IndicesApiSpec "reject empty index name" in { // When - val result = indicesApi.createIndex("") + val result = indicesApi.createIndex("", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -229,7 +231,7 @@ class IndicesApiSpec val invalidJson = """{"index": invalid json}""" // When - val result = indicesApi.createIndex("my-index", invalidJson) + val result = indicesApi.createIndex("my-index", invalidJson, None, Nil) // Then result.isFailure shouldBe true @@ -243,7 +245,7 @@ class IndicesApiSpec "reject empty settings" in { // When - val result = indicesApi.createIndex("my-index", "") + val result = indicesApi.createIndex("my-index", "", None, Nil) // Then result.isFailure shouldBe true @@ -253,7 +255,7 @@ class IndicesApiSpec "reject null settings" in { // When - val result = indicesApi.createIndex("my-index", null) + val result = indicesApi.createIndex("my-index", null, None, Nil) // Then result.isFailure shouldBe true @@ -270,7 +272,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -284,7 +286,7 @@ class IndicesApiSpec val invalidJson = """invalid""" // When - Invalid index name should fail first - val result = indicesApi.createIndex("INVALID", invalidJson) + val result = indicesApi.createIndex("INVALID", invalidJson, None, Nil) // Then result.error.get.message should include("Invalid index") @@ -679,7 +681,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -731,7 +735,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -763,9 +769,10 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index").map { success => - if (success) "Created" else "Not created" - } + val result = + indicesApi.createIndex("my-index", mappings = None, aliases = Nil).map { success => + if (success) "Created" else "Not created" + } // Then result shouldBe ElasticSuccess("Created") @@ -777,9 +784,10 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index").flatMap { _ => - indicesApi.indexExists("my-index") - } + val result = + indicesApi.createIndex("my-index", mappings = None, aliases = Nil).flatMap { _ => + indicesApi.indexExists("my-index") + } // Then result shouldBe ElasticSuccess(true) @@ -792,7 +800,7 @@ class IndicesApiSpec // When val result = for { - created <- indicesApi.createIndex("my-index") + created <- indicesApi.createIndex("my-index", mappings = None, aliases = Nil) opened <- indicesApi.openIndex("my-index") } yield created && opened @@ -807,7 +815,7 @@ class IndicesApiSpec // When val result = indicesApi - .createIndex("my-index") + .createIndex("my-index", mappings = None, aliases = Nil) .map(!_) .flatMap(v => ElasticSuccess(s"Result: $v")) @@ -827,7 +835,7 @@ class IndicesApiSpec indicesApi.executeDeleteIndexResult = ElasticSuccess(true) // When - val created = indicesApi.createIndex("test-index") + val created = indicesApi.createIndex("test-index", mappings = None, aliases = Nil) val closed = indicesApi.closeIndex("test-index") val opened = indicesApi.openIndex("test-index") val deleted = indicesApi.deleteIndex("test-index") @@ -849,9 +857,9 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result1 = indicesApi.createIndex("index1") - val result2 = indicesApi.createIndex("index2") - val result3 = indicesApi.createIndex("index3") + val result1 = indicesApi.createIndex("index1", mappings = None, aliases = Nil) + val result2 = indicesApi.createIndex("index2", mappings = None, aliases = Nil) + val result3 = indicesApi.createIndex("index3", mappings = None, aliases = Nil) // Then result1.isSuccess shouldBe true @@ -870,7 +878,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -931,7 +939,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) @@ -954,7 +964,7 @@ class IndicesApiSpec } // When - validatingApi.createIndex("INVALID") + validatingApi.createIndex("INVALID", mappings = None, aliases = Nil) // Then executeCalled shouldBe false @@ -968,7 +978,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) @@ -991,7 +1003,7 @@ class IndicesApiSpec } // When - validatingApi.createIndex("valid-index", "invalid json") + validatingApi.createIndex("valid-index", "invalid json", None, Nil) // Then executeCalled shouldBe false @@ -1010,7 +1022,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -1046,7 +1060,9 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -1077,7 +1093,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - indicesApi.createIndex("my-index") + indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then verify(mockLogger, atLeastOnce).info(any[String]) @@ -1089,7 +1105,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(ElasticError("Failed")) // When - indicesApi.createIndex("my-index") + indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then verify(mockLogger).error(any[String]) @@ -1130,7 +1146,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex(maxName) + val result = indicesApi.createIndex(maxName, mappings = None, aliases = Nil) // Then result.isSuccess shouldBe true @@ -1186,7 +1202,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", complexSettings) + val result = indicesApi.createIndex("my-index", complexSettings, None, Nil) // Then result.isSuccess shouldBe true @@ -1199,7 +1215,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", settings) + val result = indicesApi.createIndex("my-index", settings, None, Nil) // Then result.isSuccess shouldBe true @@ -1216,7 +1232,7 @@ class IndicesApiSpec indicesApi.executeDeleteIndexResult = ElasticSuccess(true) // When - val created = indicesApi.createIndex("new-index") + val created = indicesApi.createIndex("new-index", mappings = None, aliases = Nil) val reindexed = indicesApi.reindex("old-index", "new-index") val deleted = indicesApi.deleteIndex("old-index") @@ -1229,13 +1245,13 @@ class IndicesApiSpec "handle multiple operations with mixed results" in { // Given indicesApi.executeCreateIndexResult = ElasticSuccess(true) - val result1 = indicesApi.createIndex("index1") + val result1 = indicesApi.createIndex("index1", mappings = None, aliases = Nil) indicesApi.executeCreateIndexResult = ElasticFailure(ElasticError("Already exists")) - val result2 = indicesApi.createIndex("index2") + val result2 = indicesApi.createIndex("index2", mappings = None, aliases = Nil) indicesApi.executeCreateIndexResult = ElasticSuccess(false) - val result3 = indicesApi.createIndex("index3") + val result3 = indicesApi.createIndex("index3", mappings = None, aliases = Nil) // Then result1.isSuccess shouldBe true @@ -1270,7 +1286,8 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - Simulate concurrent calls - val results = (1 to 5).map(i => indicesApi.createIndex(s"index-$i")) + val results = + (1 to 5).map(i => indicesApi.createIndex(s"index-$i", mappings = None, aliases = Nil)) // Then results.foreach(_.isSuccess shouldBe true) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index e3b3100e..26a125f2 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -60,7 +60,9 @@ class MappingApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = { executeCreateIndexResult } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index 394256fa..bbf5b3ad 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -55,7 +55,9 @@ class SettingsApiSpec // Other required methods override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -737,7 +739,9 @@ class SettingsApiSpec ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -790,7 +794,9 @@ class SettingsApiSpec ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -840,7 +846,9 @@ class SettingsApiSpec ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -1406,7 +1414,9 @@ class SettingsApiSpec ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -1448,7 +1458,9 @@ class SettingsApiSpec ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? @@ -1490,7 +1502,9 @@ class SettingsApiSpec override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? diff --git a/documentation/client/indices.md b/documentation/client/indices.md index cd472f81..c8de7f50 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -82,13 +82,17 @@ Creates a new index with specified settings. ```scala def createIndex( index: String, - settings: String = defaultSettings + settings: String = defaultSettings, + mappings: Option[String] = None, + aliases: Seq[String] = Seq.empty ): ElasticResult[Boolean] ``` **Parameters:** - `index` - Name of the index to create - `settings` - JSON settings for the index (defaults to `defaultSettings`) +- `mappings` - Optional JSON mappings for the index +- `aliases` - Optional list of aliases to assign to the index **Returns:** - `ElasticSuccess[Boolean]` with `true` if created, `false` otherwise @@ -97,6 +101,8 @@ def createIndex( **Validation:** - Index name format validation - JSON settings syntax validation +- JSON mappings syntax validation if provided +- Alias name format validation **Examples:** diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 3f4c1beb..ae90f749 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -37,7 +37,9 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe */ private[client] def executeCreateIndex( index: String, - settings: String = defaultSettings + settings: String = defaultSettings, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = { executeJestBooleanAction( operation = "createIndex", @@ -46,6 +48,8 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe ) { new CreateIndex.Builder(index) .settings(settings) + .aliases(aliases.map(alias => s"""{"$alias":{}}""").mkString(",")) + .mappings(mappings.getOrElse("{}")) .build() } } diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 2b651ca0..9955aac3 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -26,7 +26,7 @@ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import com.google.gson.JsonParser import org.apache.http.util.EntityUtils -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest +import org.elasticsearch.action.admin.indices.alias.{Alias, IndicesAliasesRequest} import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest import org.elasticsearch.action.admin.indices.close.CloseIndexRequest @@ -125,14 +125,19 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH _: RefreshApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): result.ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), retryable = false )( - request = new CreateIndexRequest(index).settings(settings, XContentType.JSON) + request = new CreateIndexRequest(index) + .settings(settings, XContentType.JSON) + .aliases(aliases.map(alias => new Alias(alias)).asJava) + .mapping(mappings.getOrElse("{}"), XContentType.JSON) )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 39a103ae..5a581a6c 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -26,6 +26,7 @@ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import com.google.gson.JsonParser import org.apache.http.util.EntityUtils +import org.elasticsearch.action.admin.indices.alias.Alias import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest @@ -130,14 +131,19 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): result.ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), retryable = false )( - request = new CreateIndexRequest(index).settings(settings, XContentType.JSON) + request = new CreateIndexRequest(index) + .settings(settings, XContentType.JSON) + .aliases(aliases.map(alias => new Alias(alias)).asJava) + .mapping(mappings.getOrElse("{}"), XContentType.JSON) )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 9114b42b..c79b9167 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -26,6 +26,7 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, FieldValue, @@ -106,7 +107,9 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel _: JavaClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): result.ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", @@ -119,6 +122,14 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) + .aliases(aliases.map(key => (key, new Alias.Builder().build())).toMap.asJava) + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(mappings.getOrElse("{}")) + ) + .build() + ) .build() ) )(_.acknowledged()) diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 423d4748..a458e39b 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -27,6 +27,7 @@ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, FieldValue, @@ -102,7 +103,9 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel _: JavaClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): result.ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", @@ -115,6 +118,14 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) + .aliases(aliases.map(key => (key, new Alias.Builder().build())).toMap.asJava) + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(mappings.getOrElse("{}")) + ) + .build() + ) .build() ) )(_.acknowledged()) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 287b20bf..bdfc660e 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -80,7 +80,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M override def beforeAll(): Unit = { super.beforeAll() - pClient.createIndex("person") + pClient.createIndex("person", mappings = None, aliases = Nil) } override def afterAll(): Unit = { @@ -103,7 +103,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M ) "Creating an index and then delete it" should "work fine" in { - pClient.createIndex("create_delete") + pClient.createIndex("create_delete", mappings = None, aliases = Nil) blockUntilIndexExists("create_delete") "create_delete" should beCreated @@ -174,7 +174,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Setting a mapping" should "work" in { - pClient.createIndex("person_mapping") + pClient.createIndex("person_mapping", mappings = None, aliases = Nil) blockUntilIndexExists("person_mapping") "person_mapping" should beCreated() @@ -270,7 +270,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Updating a mapping" should "work" in { - pClient.createIndex("person_migration").get shouldBe true + pClient.createIndex("person_migration", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -528,7 +528,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Bulk upsert valid json with an id key and a suffix key" should "work" in { - pClient.createIndex("person5").get shouldBe true + pClient.createIndex("person5", mappings = None, aliases = Nil).get shouldBe true implicit val bulkOptions: BulkOptions = BulkOptions("person5") val result = pClient .bulk[String]( @@ -735,7 +735,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Index" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -758,7 +758,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Update" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.updateAs(sample, uuid, Some(index)).get result shouldBe true @@ -781,7 +781,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Delete" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -799,7 +799,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Delete asynchronously" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -819,7 +819,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Index binary data" should "work" in { - bClient.createIndex("binaries").get shouldBe true + bClient.createIndex("binaries", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -875,7 +875,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Aggregations" should "work" in { - pClient.createIndex("person10").get shouldBe true + pClient.createIndex("person10", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -1136,7 +1136,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Nested queries" should "work" in { - parentClient.createIndex("parent").get shouldBe true + parentClient.createIndex("parent", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 8a749f90..9aee0e72 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -94,7 +94,9 @@ trait MockElasticClientApi extends ElasticClientApi { override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[String] ): ElasticResult[Boolean] = ElasticResult.success(true) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala index 83952c62..16f1f1ad 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala @@ -79,7 +79,7 @@ trait WindowFunctionSpec | } |}""".stripMargin - client.createIndex("emp").get shouldBe true + client.createIndex("emp", mappings = None, aliases = Nil).get shouldBe true client.setMapping("emp", mapping).get shouldBe true From f581d4318a9a81130b30fcc12954f55bc8212112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 13 Dec 2025 23:24:51 +0100 Subject: [PATCH 09/95] add support for _id and _ingest.timestamp as default values --- .../app/softnetwork/elastic/sql/package.scala | 33 +++++++++++++++---- .../elastic/sql/parser/Parser.scala | 6 +++- .../elastic/sql/schema/package.scala | 21 +++++++++--- .../elastic/sql/parser/ParserSpec.scala | 6 +++- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 805febc3..abc51db3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -295,13 +295,19 @@ package object sql { case null => Null case b: Boolean => BooleanValue(b) case c: Char => CharValue(c) - case s: String => StringValue(s) - case b: Byte => ByteValue(b) - case s: Short => ShortValue(s) - case i: Int => IntValue(i) - case l: Long => LongValue(l) - case f: Float => FloatValue(f) - case d: Double => DoubleValue(d) + case s: String => + s match { + case "null" => Null + case "_id" => IdValue + case "_ingest.timestamp" => IngestTimestampValue + case _ => StringValue(s) + } + case b: Byte => ByteValue(b) + case s: Short => ShortValue(s) + case i: Int => IntValue(i) + case l: Long => LongValue(l) + case f: Float => FloatValue(f) + case d: Double => DoubleValue(d) case a: Array[T] => val values = a.toSeq.map(apply) values.headOption match { @@ -381,6 +387,19 @@ package object sql { override def baseType: SQLType = SQLTypes.Varchar } + case object IdValue extends Value[String]("_id") with TokenRegex { + override def sql: String = value + override def painless(context: Option[PainlessContext]): String = s"{{$value}}" + override def baseType: SQLType = SQLTypes.Varchar + } + + case object IngestTimestampValue extends Value[String]("_ingest.timestamp") with TokenRegex { + override def sql: String = value + override def painless(context: Option[PainlessContext]): String = + s"{{$value}}" + override def baseType: SQLType = SQLTypes.Timestamp + } + sealed abstract class NumericValue[T: Numeric](override val value: T) extends Value[T](value) { override def sql: String = value.toString private[this] val num: Numeric[T] = implicitly[Numeric[T]] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 6a4e091f..e629f9c7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -102,8 +102,12 @@ object Parser case None => false } + def ingest_id: PackratParser[Value[_]] = "_id" ^^ (_ => IdValue) + + def ingest_timestamp: PackratParser[Value[_]] = "_ingest.timestamp" ^^ (_ => IngestTimestampValue) + def defaultVal: PackratParser[Option[Value[_]]] = - opt("DEFAULT" ~ value) ^^ { + opt("DEFAULT" ~ (value | ingest_id | ingest_timestamp)) ^^ { case Some(_ ~ v) => Some(v) case None => None } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 2409d3b6..bddd8e80 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -186,9 +186,14 @@ package object schema { } override def properties: Map[SQL, Any] = Map( - "description" -> description, - "field" -> column, - "value" -> value.value, + "description" -> description, + "field" -> column, + "value" -> { + value match { + case IdValue | IngestTimestampValue => s"{{${value.value}}}" + case _ => value.value + } + }, "ignore_failure" -> ignoreFailure, "if" -> _if ) @@ -441,18 +446,24 @@ package object schema { primaryKey: List[String] = Nil, partitionBy: Option[DdlPartition] = None, defaultPipeline: Option[String] = None, - finalPipeline: Option[String] = None + finalPipeline: Option[String] = None, + options: Map[String, Value[_]] = Map.empty ) extends Token { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap def sql: String = { + val opts = if (options.nonEmpty) { + s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " + } else { + "" + } val cols = columns.map(_.sql).mkString(", ") val pkStr = if (primaryKey.nonEmpty) { s", PRIMARY KEY (${primaryKey.mkString(", ")})" } else { "" } - s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}" + s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}$opts" } def ddlProcessors: Seq[DdlProcessor] = diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 5130e168..e04d1c51 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -935,6 +935,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { | name VARCHAR DEFAULT 'anonymous', | birthdate DATE, | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), + | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, | PRIMARY KEY (id) |) PARTITION BY birthdate""".stripMargin val result = Parser(sql) @@ -965,9 +966,12 @@ class ParserSpec extends AnyFlatSpec with Matchers { |ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin .replaceAll("\n", " ") ) + cols.find(_.name == "ingested_at").get.defaultValue.map(_.value) shouldBe Some( + "_ingest.timestamp" + ) val json = ct.ddlTable.ddlPipeline.json print(json) - json shouldBe "{\"description\":\"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), PARTITION BY birthdate (DAY), PRIMARY KEY (id))\",\"processors\":[{\"set\":{\"field\":\"name\",\"if\":\"ctx.name == null\",\"description\":\"name DEFAULT 'anonymous'\",\"ignore_failure\":true,\"value\":\"anonymous\"}},{\"script\":{\"description\":\"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))\",\"lang\":\"painless\",\"source\":\"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)\",\"ignore_failure\":true}},{\"date_index_name\":{\"field\":\"birthdate\",\"index_name_prefix\":\"users-\",\"date_formats\":[\"yyyy-MM-dd\"],\"ignore_failure\":true,\"date_rounding\":\"d\",\"description\":\"PARTITION BY birthdate (DAY)\",\"separator\":\"-\"}},{\"set\":{\"field\":\"_id\",\"description\":\"PRIMARY KEY (id)\",\"ignore_failure\":false,\"ignore_empty_value\":false,\"value\":\"{{id}}\"}}]}" + json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (DAY), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM-dd"],"ignore_failure":true,"date_rounding":"d","description":"PARTITION BY birthdate (DAY)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" case _ => fail("Expected CreateTable") } } From fa2f6abb51d8aef2c76b78ba6b9d14ec6f61c2c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 15 Dec 2025 21:29:49 +0100 Subject: [PATCH 10/95] add object value, generate index mappings, settings and pipeline from DdlTable, support mappings and settings options from ddl --- .../softnetwork/elastic/schema/package.scala | 218 +++++++++++------- .../app/softnetwork/elastic/sql/package.scala | 48 +++- .../elastic/sql/parser/Parser.scala | 39 +++- .../elastic/sql/parser/type/package.scala | 24 +- .../elastic/sql/query/package.scala | 26 ++- .../elastic/sql/schema/package.scala | 155 ++++++++++--- .../elastic/sql/type/SQLTypeUtils.scala | 22 ++ .../elastic/sql/type/SQLTypes.scala | 4 +- 8 files changed, 404 insertions(+), 132 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 3cd09ecd..8b91b243 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -6,16 +6,37 @@ import com.fasterxml.jackson.databind.JsonNode import scala.jdk.CollectionConverters._ package object schema { - final case class EsField( + + private[schema] def extractOptions( + node: JsonNode, + ignoredKeys: Set[String] = Set.empty + ): Map[String, Value[_]] = { + node + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val value = entry.getValue + + if (ignoredKeys.contains(key)) { + None + } else { + Value(value).map(key -> Value(_)) + } + } + .toMap + } + + final case class Field( name: String, `type`: String, null_value: Option[Value[_]] = None, - fields: List[EsField] = Nil, + fields: List[Field] = Nil, options: Map[String, Value[_]] = Map.empty ) - object EsField { - def apply(name: String, node: JsonNode): EsField = { + object Field { + def apply(name: String, node: JsonNode): Field = { val tpe = Option(node.get("type")).map(_.asText()).getOrElse("object") val nullValue = @@ -30,9 +51,9 @@ package object schema { }.toList) .getOrElse(Nil) - val options = extractOptions(node) + val options = extractOptions(node, ignoredKeys = Set("type", "null_value", "fields")) - EsField( + Field( name = name, `type` = tpe, null_value = nullValue, @@ -41,122 +62,141 @@ package object schema { ) } - - private[this] def extractOptions(node: JsonNode): Map[String, Value[_]] = { - val ignoredKeys = Set("type", "null_value", "fields") - - node - .properties() - .asScala - .flatMap { entry => - val key = entry.getKey - val value = entry.getValue - - if (ignoredKeys.contains(key)) { - None - } else { - Value(value).map(key -> Value(_)) - } - } - .toMap - } - } - final case class EsMapping( - fields: List[EsField] = Nil + final case class Mappings( + fields: List[Field] = Nil, + primaryKey: List[String] = Nil, + partitionBy: Option[EsPartitionBy] = None, + options: Map[String, Value[_]] = Map.empty ) - object EsMapping { - def apply(root: JsonNode): EsMapping = { - val fields = Option(root.path("mappings").path("properties")) + object Mappings { + def apply(root: JsonNode): Mappings = { + val mappings = root.path("mappings") + val fields = Option(mappings.get("properties")) + .orElse(Option(mappings.path("_doc").get("properties"))) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue - EsField(name, value) + Field(name, value) }.toList) .getOrElse(Nil) - EsMapping(fields = fields) + val options = extractOptions(mappings, ignoredKeys = Set("properties", "_doc")) + val meta = options.get("_meta") + val primaryKey: List[String] = meta + .map { + case m: Map[_, _] => + m.asInstanceOf[Map[String, Value[_]]].get("primary_key") match { + case Some(pk: Value[_]) => + pk.value match { + case list: List[_] => list.map(_.toString) + case str: String => List(str) + case _ => List.empty + } + case _ => List.empty + } + case _ => List.empty + } + .getOrElse(List.empty) + + val partitionBy: Option[EsPartitionBy] = meta.flatMap { + case m: Map[_, _] => + m.asInstanceOf[Map[String, Value[_]]].get("partition_by") match { + case Some(pb: Value[_]) => + pb.value match { + case map: Map[_, _] => + val partitionMap = map.asInstanceOf[Map[String, String]] + val column = partitionMap.getOrElse("column", "date") + val granularity = partitionMap.getOrElse("granularity", "d") + Some(EsPartitionBy(column, granularity)) + case _ => None + } + case _ => None + } + case _ => None + } + + Mappings( + fields = fields, + primaryKey = primaryKey, + partitionBy = partitionBy, + options = options + ) } + } - final case class EsPartitionMeta( + final case class EsPartitionBy( column: String, granularity: String // "d", "M", "y", etc. ) - final case class EsDdlMeta( - primary_key: List[String] = Nil, - partition_by: Option[EsPartitionMeta] = None, - default_pipeline: Option[String] = None, - final_pipeline: Option[String] = None - ) - - object EsDdlMeta { - def apply(settings: JsonNode): EsDdlMeta = { - val index = settings.path("settings").path("index") - - val defaultPipeline = Option(index.get("default_pipeline")).map(_.asText()) - val finalPipeline = Option(index.get("final_pipeline")).map(_.asText()) + final case class Settings( + options: Map[String, Value[_]] = Map.empty + ) { - val ddlNode = index - .path("meta") - .path("ddl") + lazy val defaultPipeline: Option[String] = { + options.get("default_pipeline").map(_.value).flatMap { + case v: String => Some(v) + case _ => None + } + } - if (ddlNode.isMissingNode) - return EsDdlMeta( - default_pipeline = defaultPipeline, - final_pipeline = finalPipeline - ) + lazy val finalPipeline: Option[String] = { + options.get("final_pipeline").map(_.value).flatMap { + case v: String => Some(v) + case _ => None + } + } + } - val primaryKey = Option(ddlNode.path("primary_key")) - .filter(_.isArray) - .map(_.elements().asScala.map(_.asText()).toList) - .getOrElse(Nil) + object Settings { + def apply(settings: JsonNode): Settings = { + val index = settings.path("settings").path("index") - val partitionBy = - Option(ddlNode.path("partition_by")).filter(_.isObject).map { partitionNode => - val column = Option(partitionNode.get("column")) - .map(_.asText()) - .getOrElse("date") - val granularity = Option(partitionNode.get("granularity")) - .map(_.asText()) - .getOrElse("d") - EsPartitionMeta(column, granularity) - } + val options = extractOptions(index) - EsDdlMeta( - primary_key = primaryKey, - partition_by = partitionBy, - default_pipeline = defaultPipeline, - final_pipeline = finalPipeline + Settings( + options = options ) } } - final case class EsIndexMeta( + final case class Index( name: String, - mapping: EsMapping, - ddlMeta: EsDdlMeta, - defaultPipeline: Option[JsonNode] = None // existing "DDL default" pipeline - ) + mappings: Mappings, + settings: Settings, + pipeline: Option[JsonNode] = None + ) { + lazy val defaultPipeline: Option[String] = { + settings.options.get("default_pipeline").map(_.value).flatMap { + case v: String => Some(v) + case _ => None + } + } - object EsIndexMeta { + lazy val finalPipeline: Option[String] = { + settings.options.get("final_pipeline").map(_.value).flatMap { + case v: String => Some(v) + case _ => None + } + } + } + + object Index { def apply( name: String, settings: JsonNode, mappings: JsonNode, - defaultPipeline: Option[JsonNode] - ): EsIndexMeta = { - val mapping = EsMapping(mappings) - val ddlMeta = EsDdlMeta(settings) - - EsIndexMeta( + pipeline: Option[JsonNode] + ): Index = { + Index( name = name, - mapping = mapping, - ddlMeta = ddlMeta, - defaultPipeline = defaultPipeline + mappings = Mappings(mappings), + settings = Settings(settings), + pipeline = pipeline ) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index abc51db3..be135cc1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.sql.query._ import com.fasterxml.jackson.databind.JsonNode import java.security.MessageDigest +import scala.jdk.CollectionConverters._ import scala.reflect.runtime.universe._ import scala.util.Try import scala.util.matching.Regex @@ -325,8 +326,19 @@ package object sql { FloatValues(values.asInstanceOf[Seq[FloatValue]]).asInstanceOf[Values[R, T]] case Some(_: DoubleValue) => DoubleValues(values.asInstanceOf[Seq[DoubleValue]]).asInstanceOf[Values[R, T]] + case Some(_: BooleanValue) => + BooleanValues(values.asInstanceOf[Seq[BooleanValue]]) + .asInstanceOf[Values[R, T]] + case Some(_: ObjectValue) => + ObjectValues( + values + .asInstanceOf[Seq[ObjectValue]] + ).asInstanceOf[Values[R, T]] case _ => throw new IllegalArgumentException("Unsupported Values type") } + case o: Map[_, _] => + val map = o.asInstanceOf[Map[String, Any]].map { case (k, v) => k -> apply(v) } + ObjectValue(map) case other => StringValue(other.toString) } } @@ -350,14 +362,30 @@ package object sql { .toList Some(arr) case n if n.isObject => - // The raw JSON object is stored as a string. TODO consider mapping to a Map[String, Any] - Some(n.toString) + val map = n + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val valueNode = entry.getValue + apply(valueNode).map(value => key -> value) + } + .toMap + Some(map) case _ => None } } } + case class ObjectValue(override val value: Map[String, Value[_]]) + extends Value[Map[String, Value[_]]](value) { + override def sql: String = value + .map { case (k, v) => s""""$k" = ${v.sql}""" } + .mkString("(", ", ", ")") + override def baseType: SQLType = SQLTypes.Struct + } + case object Null extends Value[Null](null) with TokenRegex { override def sql: String = "NULL" override def painless(context: Option[PainlessContext]): String = "null" @@ -576,6 +604,22 @@ package object sql { override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Double) } + case class BooleanValues(override val values: Seq[BooleanValue]) + extends Values[Boolean, Value[Boolean]](values) { + def eq: Seq[Boolean] => Boolean = { + _.exists { b => innerValues.contains(b) } + } + def ne: Seq[Boolean] => Boolean = { + _.forall { b => !innerValues.contains(b) } + } + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Boolean) + } + + case class ObjectValues(override val values: Seq[ObjectValue]) + extends Values[Map[String, Value[_]], ObjectValue](values) { + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Struct) + } + def toRegex(value: String): String = { value.replaceAll("%", ".*").replaceAll("_", ".") } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index e629f9c7..7d96a2bd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -69,8 +69,13 @@ object Parser def ident: Parser[String] = """[a-zA-Z_][a-zA-Z0-9_]*""".r + def objectValue: PackratParser[ObjectValue] = + start ~ repsep(option, separator) ~ end ^^ { case _ ~ opts ~ _ => + ObjectValue(opts.toMap) + } + def option: PackratParser[(String, Value[_])] = - ident ~ "=" ~ value ^^ { case key ~ _ ~ value => + ident ~ "=" ~ (value | objectValue) ^^ { case key ~ _ ~ value => (key, value) } @@ -181,24 +186,35 @@ object Parser case None => None } - def columnsWithPartitionBy: PackratParser[(List[DdlColumn], List[String], Option[DdlPartition])] = - start ~ repsep(column, separator) ~ primaryKey ~ end ~ partitionBy ^^ { - case _ ~ cols ~ pk ~ _ ~ pb => - (cols, pk, pb) + def columnsWithPartitionBy + : PackratParser[(List[DdlColumn], List[String], Option[DdlPartition], Map[String, Any])] = + start ~ repsep( + column, + separator + ) ~ primaryKey ~ end ~ partitionBy ~ ((separator.? ~> options) | success( + Map.empty[String, Value[_]] + )) ^^ { case _ ~ cols ~ pk ~ _ ~ pb ~ opts => + (cols, pk, pb, opts) } def createOrReplaceTable: PackratParser[CreateTable] = ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ name ~ lr => lr match { - case (cols: List[DdlColumn], pk: List[String], p: Option[DdlPartition]) => + case ( + cols: List[DdlColumn], + pk: List[String], + p: Option[DdlPartition], + opts: Map[String, Value[_]] + ) => CreateTable( name, Right(cols), ifNotExists = false, orReplace = true, primaryKey = pk, - partitionBy = p + partitionBy = p, + options = opts ) case sel: DqlStatement => CreateTable(name, Left(sel), ifNotExists = false, orReplace = true) @@ -209,8 +225,13 @@ object Parser ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { case _ ~ ine ~ name ~ lr => lr match { - case (cols: List[DdlColumn], pk: List[String], p: Option[DdlPartition]) => - CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p) + case ( + cols: List[DdlColumn], + pk: List[String], + p: Option[DdlPartition], + opts: Map[String, Value[_]] + ) => + CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p, options = opts) case sel: DqlStatement => CreateTable(name, Left(sel), ine) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 4e5ee847..d94ba023 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -18,12 +18,16 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.{ BooleanValue, + BooleanValues, DoubleValue, + DoubleValues, Identifier, LongValue, + LongValues, ParamValue, PiValue, StringValue, + StringValues, Value } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} @@ -52,8 +56,26 @@ package object `type` { def param: PackratParser[ParamValue.type] = "?" ^^ (_ => ParamValue) + def literals: PackratParser[Value[_]] = "[" ~> repsep(literal, ",") <~ "]" ^^ { list => + StringValues(list) + } + + def longs: PackratParser[Value[_]] = "[" ~> repsep(long, ",") <~ "]" ^^ { list => + LongValues(list) + } + + def doubles: PackratParser[Value[_]] = "[" ~> repsep(double, ",") <~ "]" ^^ { list => + DoubleValues(list) + } + + def booleans: PackratParser[BooleanValues] = "[" ~> repsep(boolean, ",") <~ "]" ^^ { list => + BooleanValues(list) + } + + def array: PackratParser[Value[_]] = literals | longs | doubles | booleans + def value: PackratParser[Value[_]] = - literal | pi | double | long | boolean | param + literal | pi | double | long | boolean | param | array def identifierWithValue: Parser[Identifier] = (value ^^ functionAsIdentifier) >> cast diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 641c08fb..d77e5db2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -317,7 +317,8 @@ package object query { ifNotExists: Boolean = false, orReplace: Boolean = false, primaryKey: List[String] = Nil, - partitionBy: Option[DdlPartition] = None + partitionBy: Option[DdlPartition] = None, + options: Map[String, Value[_]] = Map.empty ) extends DdlStatement { override def sql: String = { @@ -350,14 +351,35 @@ package object query { } } + lazy val mappings: Map[String, Value[_]] = options.get("mappings") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + + lazy val settings: Map[String, Value[_]] = options.get("settings") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + lazy val ddlTable: DdlTable = DdlTable( name = table, columns = columns.toList, primaryKey = primaryKey, - partitionBy = partitionBy + partitionBy = partitionBy, + mappings = mappings, + settings = settings ) lazy val ddlPipeline: DdlPipeline = ddlTable.ddlPipeline + } case class AlterTable(table: String, ifExists: Boolean, statements: List[AlterTableStatement]) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index bddd8e80..2f38ac4a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,8 +16,8 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.schema.{EsField, EsIndexMeta} -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.schema.{Field, Index} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization.JacksonConfig import app.softnetwork.elastic.sql.time.TimeUnit @@ -86,7 +86,7 @@ package object schema { } def json: String = node.toString def processorType: DdlProcessorType - def description: String = sql + def description: String = sql.trim def name: String = processorType.name def properties: Map[String, Any] } @@ -347,7 +347,7 @@ package object schema { ddlProcessors: Seq[DdlProcessor] ) extends Token { def sql: String = - s"CREATE OR REPLACE ${ddlPipelineType.name} PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql).mkString(", ")})" + s"CREATE OR REPLACE ${ddlPipelineType.name} PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql.trim).mkString(", ")})" def node: ObjectNode = { val node = mapper.createObjectNode() @@ -363,6 +363,27 @@ package object schema { def json: String = node.toString } + private[schema] def update(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { + updates.foreach { case (k, v) => + v match { + case Null => node.putNull(k) + case BooleanValue(b) => node.put(k, b) + case StringValue(s) => node.put(k, s) + case ByteValue(b) => node.put(k, b) + case ShortValue(s) => node.put(k, s) + case IntValue(i) => node.put(k, i) + case LongValue(l) => node.put(k, l) + case DoubleValue(d) => node.put(k, d) + case FloatValue(f) => node.put(k, f) + case ObjectValue(value) => + if (value.nonEmpty) + node.set(k, update(mapper.createObjectNode(), value)) + case _ => // do nothing + } + } + node + } + case class DdlColumn( name: String, dataType: SQLType, @@ -385,7 +406,7 @@ package object schema { } else { "" } - val scriptOpt = script.map(s => s" SCRIPT AS ($s)").getOrElse("") + val scriptOpt = script.map(s => s" SCRIPT AS (${s.script})").getOrElse("") s"$name $dataType$fieldsOpt$scriptOpt$notNullOpt$defaultOpt$opts" } @@ -397,10 +418,33 @@ package object schema { value = dv ) }.toSeq + + def node: ObjectNode = { + val root = mapper.createObjectNode() + val esType = SQLTypeUtils.elasticType(dataType) + root.put("type", esType) + defaultValue.foreach { dv => + update(root, Map("null_value" -> dv)) + } + if (multiFields.nonEmpty) { + val name = + esType match { + case "object" | "nested" => "properties" + case _ => "fields" + } + val fieldsNode = mapper.createObjectNode() + multiFields.foreach { field => + fieldsNode.replace(field.name, field.node) + } + root.set(name, fieldsNode) + } + update(root, options) + root + } } object DdlColumn { - def apply(field: EsField): DdlColumn = { + def apply(field: Field): DdlColumn = { DdlColumn( name = field.name, dataType = SQLTypes(field), @@ -410,8 +454,9 @@ package object schema { ) } } + case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { - def sql: String = s"PARTITION BY $column ($granularity)" + def sql: String = s" PARTITION BY $column ($granularity)" val dateRounding: String = granularity.script.get @@ -447,23 +492,38 @@ package object schema { partitionBy: Option[DdlPartition] = None, defaultPipeline: Option[String] = None, finalPipeline: Option[String] = None, - options: Map[String, Value[_]] = Map.empty + mappings: Map[String, Value[_]] = Map.empty, + settings: Map[String, Value[_]] = Map.empty ) extends Token { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap def sql: String = { - val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " - } else { - "" - } - val cols = columns.map(_.sql).mkString(", ") + val opts = + if (mappings.nonEmpty || settings.nonEmpty) { + val mappingOpts = + if (mappings.nonEmpty) { + s"mappings = (${mappings.map { case (k, v) => s"$k = $v" }.mkString(", ")})" + } else { + "" + } + val settingsOpts = + if (settings.nonEmpty) { + s"settings = (${mappings.map { case (k, v) => s"$k = $v" }.mkString(", ")})" + } else { + "" + } + val separator = if (partitionBy.nonEmpty) "," else "" + s"$separator OPTIONS = (${Seq(mappingOpts, settingsOpts).filter(_.nonEmpty).mkString(", ")})" + } else { + "" + } + val cols = columns.map(_.sql).mkString(",\n\t") val pkStr = if (primaryKey.nonEmpty) { - s", PRIMARY KEY (${primaryKey.mkString(", ")})" + s",\n\tPRIMARY KEY (${primaryKey.mkString(", ")})\n" } else { "" } - s"CREATE OR REPLACE TABLE $name ($cols$pkStr)${partitionBy.getOrElse("")}$opts" + s"CREATE OR REPLACE TABLE $name (\n\t$cols$pkStr)${partitionBy.getOrElse("")}$opts" } def ddlProcessors: Seq[DdlProcessor] = @@ -590,13 +650,54 @@ package object schema { ddlPipelineType = DdlPipelineType.Default, ddlProcessors = ddlProcessors ) + + lazy val indexMappings: ObjectNode = { + val node = mapper.createObjectNode() + val fields = mapper.createObjectNode() + columns.foreach { column => + fields.replace(column.name, column.node) + } + node.set("properties", fields) + update(node, mappings) + if (primaryKey.nonEmpty || partitionBy.nonEmpty) { + val meta = Option(node.get("_meta")).getOrElse(mapper.createObjectNode()) + if (meta != null && meta.isObject) { + val metaObj = meta.asInstanceOf[ObjectNode] + if (primaryKey.nonEmpty) { + val pkArray = mapper.createArrayNode() + primaryKey.foreach(pk => pkArray.add(pk)) + metaObj.replace("primary_key", pkArray) + } + partitionBy.foreach { partition => + val partitionObj = mapper.createObjectNode() + partitionObj.put("column", partition.column) + partitionObj.put("granularity", partition.granularity.script.get) + metaObj.replace("partition_by", partitionObj) + } + node.replace("_meta", metaObj) + } + } + node + } + + lazy val indexSettings: ObjectNode = { + val node = mapper.createObjectNode() + val index = mapper.createObjectNode() + update(index, settings) + node.set("index", index) + node + } + + lazy val pipeline: ObjectNode = { + ddlPipeline.node + } } object DdlTable { - def apply(indexMeta: EsIndexMeta): DdlTable = { + def apply(index: Index): DdlTable = { // 1. Columns from the mapping val initialCols: Map[String, DdlColumn] = - indexMeta.mapping.fields.map { field => + index.mappings.fields.map { field => val name = field.name name -> DdlColumn( name = name, @@ -609,19 +710,19 @@ package object schema { ) }.toMap - // 2. PK + partition + pipelines from meta - var primaryKey: List[String] = indexMeta.ddlMeta.primary_key - var partitionBy: Option[DdlPartition] = indexMeta.ddlMeta.partition_by.map { p => + // 2. PK + partition + pipelines from index mappings and settings + var primaryKey: List[String] = index.mappings.primaryKey + var partitionBy: Option[DdlPartition] = index.mappings.partitionBy.map { p => val granularity = TimeUnit(p.granularity) DdlPartition(p.column, granularity) } - val defaultPipelineName = indexMeta.ddlMeta.default_pipeline - val finalPipelineName = indexMeta.ddlMeta.final_pipeline + val defaultPipelineName = index.settings.defaultPipeline + val finalPipelineName = index.settings.finalPipeline - // 3. Enrichment from the default pipeline (if provided) + // 3. Enrichment from the pipeline (if provided) val enrichedCols = scala.collection.mutable.Map.from(initialCols) - indexMeta.defaultPipeline.foreach { pipeline => + index.pipeline.foreach { pipeline => val processorsNode = pipeline.get("processors") if (processorsNode != null && processorsNode.isArray) { val processors: Seq[DdlProcessor] = @@ -656,9 +757,9 @@ package object schema { } } - // 4. Construction finale du DdlTable + // 4. Final construction of the DdlTable DdlTable( - name = indexMeta.name, + name = index.name, columns = enrichedCols.values.toList.sortBy(_.name), primaryKey = primaryKey, partitionBy = partitionBy, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index a0b35d81..7f23cdfc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -53,6 +53,28 @@ object SQLTypeUtils { case _ => "Object" } + def elasticType(sqlType: SQLType): String = sqlType match { + case Null => "null" + case TinyInt => "byte" + case SmallInt => "short" + case Int => "integer" + case BigInt => "long" + case Double => "double" + case Real => "float" + case Numeric => "scaled_float" + case Varchar | Text => "text" + case Keyword => "keyword" + case Boolean => "boolean" + case Date => "date" + case Time => "date" + case DateTime => "date" + case Timestamp => "date" + case Temporal => "date" + case Array(Struct) => "nested" + case Struct => "object" + case _ => "object" + } + def matches(out: SQLType, in: SQLType): Boolean = out.typeId == in.typeId || (out.typeId == Temporal.typeId && Set( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index fa142132..11bc13a5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.`type` -import app.softnetwork.elastic.schema.EsField +import app.softnetwork.elastic.schema.Field object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } @@ -74,7 +74,7 @@ object SQLTypes { case _ => Any } - def apply(field: EsField): SQLType = field.`type` match { + def apply(field: Field): SQLType = field.`type` match { case "null" => Null case "boolean" => Boolean case "integer" => Int From 65766e7009b9c1991c5b5a207f20614a08c32237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 15 Dec 2025 21:31:30 +0100 Subject: [PATCH 11/95] update parser specifications --- .../elastic/sql/parser/ParserSpec.scala | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index e04d1c51..0cf2a475 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.{ObjectValue, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.DdlPartition @@ -932,12 +933,12 @@ class ParserSpec extends AnyFlatSpec with Matchers { val sql = """CREATE TABLE IF NOT EXISTS users ( | id INT NOT NULL, - | name VARCHAR DEFAULT 'anonymous', + | name VARCHAR FIELDS(raw Keyword) DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), | birthdate DATE, | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, | PRIMARY KEY (id) - |) PARTITION BY birthdate""".stripMargin + |) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false))""".stripMargin val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get @@ -948,7 +949,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { true, false, List("id"), - Some(DdlPartition("birthdate", TimeUnit.DAYS)) + Some(DdlPartition("birthdate", TimeUnit.MONTHS)), + _ ) => cols.map(_.name) should contain allOf ("id", "name") cols.find(_.name == "id").get.notNull shouldBe true @@ -969,9 +971,21 @@ class ParserSpec extends AnyFlatSpec with Matchers { cols.find(_.name == "ingested_at").get.defaultValue.map(_.value) shouldBe Some( "_ingest.timestamp" ) + ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) + val sql = ct.ddlTable.sql + println(sql) val json = ct.ddlTable.ddlPipeline.json - print(json) - json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (DAY), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM-dd"],"ignore_failure":true,"date_rounding":"d","description":"PARTITION BY birthdate (DAY)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + println(json) + json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + val indexMappings = ct.ddlTable.indexMappings + println(indexMappings) + indexMappings.toString shouldBe "{\"properties\":{\"id\":{\"type\":\"integer\"},\"name\":{\"type\":\"text\",\"null_value\":\"anonymous\",\"fields\":{\"raw\":{\"type\":\"keyword\"}},\"analyzer\":\"french\",\"search_analyzer\":\"french\"},\"birthdate\":{\"type\":\"date\"},\"age\":{\"type\":\"integer\"},\"ingested_at\":{\"type\":\"date\"}},\"dynamic\":false,\"_meta\":{\"primary_key\":[\"id\"],\"partition_by\":{\"column\":\"birthdate\",\"granularity\":\"M\"}}}" + val indexSettings = ct.ddlTable.indexSettings + println(indexSettings) + indexSettings.toString shouldBe """{"index":{}}""" + val pipeline = ct.ddlTable.pipeline + println(pipeline) + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" case _ => fail("Expected CreateTable") } } From 0d1930a05e468fa67f314344410abd82d3567b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 15 Dec 2025 21:50:48 +0100 Subject: [PATCH 12/95] update elastic index definition --- .../softnetwork/elastic/schema/package.scala | 39 +++---------------- .../elastic/sql/schema/package.scala | 28 +++++++++---- 2 files changed, 27 insertions(+), 40 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 8b91b243..d18c2a54 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -43,7 +43,8 @@ package object schema { Option(node.get("null_value")).flatMap(Value(_)).map(Value(_)) val fields = - Option(node.get("fields")) + Option(node.get("fields")) // multi-fields + .orElse(Option(node.get("properties"))) // object/nested fields .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue @@ -51,7 +52,8 @@ package object schema { }.toList) .getOrElse(Nil) - val options = extractOptions(node, ignoredKeys = Set("type", "null_value", "fields")) + val options = + extractOptions(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) Field( name = name, @@ -135,22 +137,7 @@ package object schema { final case class Settings( options: Map[String, Value[_]] = Map.empty - ) { - - lazy val defaultPipeline: Option[String] = { - options.get("default_pipeline").map(_.value).flatMap { - case v: String => Some(v) - case _ => None - } - } - - lazy val finalPipeline: Option[String] = { - options.get("final_pipeline").map(_.value).flatMap { - case v: String => Some(v) - case _ => None - } - } - } + ) object Settings { def apply(settings: JsonNode): Settings = { @@ -169,21 +156,7 @@ package object schema { mappings: Mappings, settings: Settings, pipeline: Option[JsonNode] = None - ) { - lazy val defaultPipeline: Option[String] = { - settings.options.get("default_pipeline").map(_.value).flatMap { - case v: String => Some(v) - case _ => None - } - } - - lazy val finalPipeline: Option[String] = { - settings.options.get("final_pipeline").map(_.value).flatMap { - case v: String => Some(v) - case _ => None - } - } - } + ) object Index { def apply( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 2f38ac4a..85597768 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -490,8 +490,6 @@ package object schema { columns: List[DdlColumn], primaryKey: List[String] = Nil, partitionBy: Option[DdlPartition] = None, - defaultPipeline: Option[String] = None, - finalPipeline: Option[String] = None, mappings: Map[String, Value[_]] = Map.empty, settings: Map[String, Value[_]] = Map.empty ) extends Token { @@ -651,6 +649,26 @@ package object schema { ddlProcessors = ddlProcessors ) + lazy val defaultPipeline: String = + settings + .get("default_pipeline") + .map(_.value) + .flatMap { + case v: String => Some(v) + case _ => None + } + .getOrElse("_none") + + lazy val finalPipeline: String = + settings + .get("final_pipeline") + .map(_.value) + .flatMap { + case v: String => Some(v) + case _ => None + } + .getOrElse("_none") + lazy val indexMappings: ObjectNode = { val node = mapper.createObjectNode() val fields = mapper.createObjectNode() @@ -716,8 +734,6 @@ package object schema { val granularity = TimeUnit(p.granularity) DdlPartition(p.column, granularity) } - val defaultPipelineName = index.settings.defaultPipeline - val finalPipelineName = index.settings.finalPipeline // 3. Enrichment from the pipeline (if provided) val enrichedCols = scala.collection.mutable.Map.from(initialCols) @@ -762,9 +778,7 @@ package object schema { name = index.name, columns = enrichedCols.values.toList.sortBy(_.name), primaryKey = primaryKey, - partitionBy = partitionBy, - defaultPipeline = defaultPipelineName, - finalPipeline = finalPipelineName + partitionBy = partitionBy ) } } From edaa37b01f593ffc0fb622ba0b069a55c592a2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 16 Dec 2025 12:59:00 +0100 Subject: [PATCH 13/95] convert elastic index mappings and settings to DdlTable --- .../softnetwork/elastic/schema/package.scala | 321 +++++++++++++++--- .../app/softnetwork/elastic/sql/package.scala | 37 +- .../elastic/sql/parser/Parser.scala | 15 +- .../elastic/sql/schema/package.scala | 259 ++++---------- .../elastic/sql/type/SQLTypes.scala | 6 +- .../elastic/sql/parser/ParserSpec.scala | 25 +- 6 files changed, 399 insertions(+), 264 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index d18c2a54..49004786 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -1,6 +1,18 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.Value +import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.schema.{ + DdlColumn, + DdlDateIndexNameProcessor, + DdlDefaultValueProcessor, + DdlPartition, + DdlPrimaryKeyProcessor, + DdlProcessor, + DdlScriptProcessor, + DdlTable +} +import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode import scala.jdk.CollectionConverters._ @@ -27,16 +39,32 @@ package object schema { .toMap } - final case class Field( + final case class EsField( name: String, `type`: String, + script: Option[DdlScriptProcessor] = None, null_value: Option[Value[_]] = None, - fields: List[Field] = Nil, + not_null: Option[Boolean] = None, + comment: Option[String] = None, + fields: List[EsField] = Nil, options: Map[String, Value[_]] = Map.empty - ) + ) { + lazy val ddlColumn: DdlColumn = { + DdlColumn( + name = name, + dataType = SQLTypes(this), + script = script, + multiFields = fields.map(_.ddlColumn), + defaultValue = null_value, + notNull = not_null.getOrElse(false), + comment = comment, + options = options + ) + } + } - object Field { - def apply(name: String, node: JsonNode): Field = { + object EsField { + def apply(name: String, node: JsonNode): EsField = { val tpe = Option(node.get("type")).map(_.asText()).getOrElse("object") val nullValue = @@ -55,10 +83,54 @@ package object schema { val options = extractOptions(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) - Field( + val meta = options.get("meta") + val comment = meta.flatMap { + case m: ObjectValue => + m.value.get("comment") match { + case Some(c: StringValue) => Some(c.value) + case _ => None + } + case _ => None + } + val notNull = meta.flatMap { + case m: ObjectValue => + m.value.get("not_null") match { + case Some(c: BooleanValue) => Some(c.value) + case _ => None + } + case _ => None + } + val script = meta.flatMap { + case m: ObjectValue => + m.value.get("script") match { + case Some(st: ObjectValue) => + val map = st.value + map.get("sql") match { + case Some(script: StringValue) => + map.get("painless") match { + case Some(source: StringValue) => + Some( + DdlScriptProcessor( + script = script.value, + column = name, + dataType = SQLTypes(tpe), + source = source.value + ) + ) + } + case _ => None + } + case _ => None + } + case _ => None + } + EsField( name = name, `type` = tpe, + script = script, null_value = nullValue, + not_null = notNull, + comment = comment, fields = fields, options = options ) @@ -66,22 +138,22 @@ package object schema { } } - final case class Mappings( - fields: List[Field] = Nil, + final case class EsMappings( + fields: List[EsField] = Nil, primaryKey: List[String] = Nil, partitionBy: Option[EsPartitionBy] = None, options: Map[String, Value[_]] = Map.empty ) - object Mappings { - def apply(root: JsonNode): Mappings = { + object EsMappings { + def apply(root: JsonNode): EsMappings = { val mappings = root.path("mappings") val fields = Option(mappings.get("properties")) .orElse(Option(mappings.path("_doc").get("properties"))) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue - Field(name, value) + EsField(name, value) }.toList) .getOrElse(Nil) @@ -89,30 +161,27 @@ package object schema { val meta = options.get("_meta") val primaryKey: List[String] = meta .map { - case m: Map[_, _] => - m.asInstanceOf[Map[String, Value[_]]].get("primary_key") match { - case Some(pk: Value[_]) => - pk.value match { - case list: List[_] => list.map(_.toString) - case str: String => List(str) - case _ => List.empty - } - case _ => List.empty + case m: ObjectValue => + m.value.get("primary_key") match { + case Some(pk: StringValues) => pk.values.map(_.ddl).toList + case Some(pk: StringValue) => List(pk.ddl) + case _ => List.empty } case _ => List.empty } .getOrElse(List.empty) val partitionBy: Option[EsPartitionBy] = meta.flatMap { - case m: Map[_, _] => - m.asInstanceOf[Map[String, Value[_]]].get("partition_by") match { - case Some(pb: Value[_]) => - pb.value match { - case map: Map[_, _] => - val partitionMap = map.asInstanceOf[Map[String, String]] - val column = partitionMap.getOrElse("column", "date") - val granularity = partitionMap.getOrElse("granularity", "d") - Some(EsPartitionBy(column, granularity)) + case m: ObjectValue => + m.value.get("partition_by") match { + case Some(pb: ObjectValue) => + pb.value.get("column") match { + case Some(column: StringValue) => // valid + pb.value.get("granularity") match { + case Some(granularity: StringValue) => + Some(EsPartitionBy(column.value, granularity.value)) + case _ => Some(EsPartitionBy(column.value, "d")) + } case _ => None } case _ => None @@ -120,7 +189,7 @@ package object schema { case _ => None } - Mappings( + EsMappings( fields = fields, primaryKey = primaryKey, partitionBy = partitionBy, @@ -135,42 +204,194 @@ package object schema { granularity: String // "d", "M", "y", etc. ) - final case class Settings( + final case class EsSettings( options: Map[String, Value[_]] = Map.empty ) - object Settings { - def apply(settings: JsonNode): Settings = { + object EsSettings { + def apply(settings: JsonNode): EsSettings = { val index = settings.path("settings").path("index") val options = extractOptions(index) - Settings( + EsSettings( options = options ) } } - final case class Index( + final case class EsProcessor( + processor: JsonNode + ) { + private val ScriptDescRegex = + """^\s*([a-zA-Z0-9_]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r + + lazy val ddlProcesor: Option[DdlProcessor] = { + val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. + val props = processor.get(processorType) + + processorType match { + case "set" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val valueNode = props.get("value") + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + if (field == "_id" && desc.startsWith("PRIMARY KEY")) { + // DdlPrimaryKeyProcessor + // description: "PRIMARY KEY (id)" + val inside = desc.stripPrefix("PRIMARY KEY").trim.stripPrefix("(").stripSuffix(")") + val cols = inside.split(",").map(_.trim).filter(_.nonEmpty).toSet + Some( + DdlPrimaryKeyProcessor( + sql = desc, + column = "_id", + value = cols, + ignoreFailure = ignoreFailure + ) + ) + } else if (desc.startsWith(s"$field DEFAULT")) { + Some( + DdlDefaultValueProcessor( + sql = desc, + column = field, + value = Value(valueNode.asText()), + ignoreFailure = ignoreFailure + ) + ) + } else { + None + } + + case "script" => + val desc = props.get("description").asText() + val lang = props.get("lang").asText() + require(lang == "painless", s"Only painless supported, got $lang") + val source = props.get("source").asText() + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + desc match { + case ScriptDescRegex(col, dataType, script) => + Some( + DdlScriptProcessor( + script = script, + column = col, + dataType = SQLTypes(dataType), + source = source, + ignoreFailure = ignoreFailure + ) + ) + case _ => + None + } + + case "date_index_name" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val rounding = props.get("date_rounding").asText() + val formats = Option(props.get("date_formats")) + .map(_.elements().asScala.toList.map(_.asText())) + .getOrElse(Nil) + val prefix = props.get("index_name_prefix").asText() + + Some( + DdlDateIndexNameProcessor( + sql = desc, + column = field, + dateRounding = rounding, + dateFormats = formats, + prefix = prefix + ) + ) + + case _ => None + } + } + } + + final case class EsPipeline( + pipeline: JsonNode + ) { + lazy val processors: Seq[DdlProcessor] = { + val processorsNode = pipeline.get("processors") + if (processorsNode != null && processorsNode.isArray) { + processorsNode.elements().asScala.toSeq.flatMap(EsProcessor(_).ddlProcesor) + } else { + Seq.empty + } + } + } + + final case class EsIndex( name: String, - mappings: Mappings, - settings: Settings, + mappings: JsonNode, + settings: JsonNode, pipeline: Option[JsonNode] = None - ) + ) { + + lazy val esMappings: EsMappings = EsMappings(mappings) + + lazy val esSettings: EsSettings = EsSettings(settings) + + lazy val esPipeline: Option[EsPipeline] = pipeline.map(EsPipeline(_)) + + lazy val ddlTable: DdlTable = { + // 1. Columns from the mapping + val initialCols: Map[String, DdlColumn] = + esMappings.fields.map { field => + val name = field.name + name -> field.ddlColumn + }.toMap + + // 2. PK + partition + pipelines from index mappings and settings + var primaryKey: List[String] = esMappings.primaryKey + var partitionBy: Option[DdlPartition] = esMappings.partitionBy.map { p => + val granularity = TimeUnit(p.granularity) + DdlPartition(p.column, granularity) + } - object Index { - def apply( - name: String, - settings: JsonNode, - mappings: JsonNode, - pipeline: Option[JsonNode] - ): Index = { - Index( + // 3. Enrichment from the pipeline (if provided) + val enrichedCols = scala.collection.mutable.Map.from(initialCols) + + esPipeline.foreach { pipeline => + pipeline.processors.foreach { + case p: DdlScriptProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(script = Some(p))) + } + + case p: DdlDefaultValueProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) + } + + case p: DdlDateIndexNameProcessor => + if (partitionBy.isEmpty) { + val granularity = TimeUnit(p.dateRounding) + partitionBy = Some(DdlPartition(p.column, granularity)) + } + + case p: DdlPrimaryKeyProcessor => + if (primaryKey.isEmpty) { + primaryKey = p.value.toList + } + + case _ => // ignore others (rename/remove...) ou gère-les si tu veux les remonter en DDL + } + } + + // 4. Final construction of the DdlTable + DdlTable( name = name, - mappings = Mappings(mappings), - settings = Settings(settings), - pipeline = pipeline + columns = enrichedCols.values.toList.sortBy(_.name), + primaryKey = primaryKey, + partitionBy = partitionBy, + esMappings.options, + esSettings.options ) } } + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index be135cc1..55567e55 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -288,6 +288,7 @@ package object sql { ) override def nullable: Boolean = false + def ddl: String = sql } object Value { @@ -298,19 +299,20 @@ package object sql { case c: Char => CharValue(c) case s: String => s match { - case "null" => Null - case "_id" => IdValue - case "_ingest.timestamp" => IngestTimestampValue - case _ => StringValue(s) + case "null" => Null + case "_id" | "{{_id]]" => IdValue + case "_ingest.timestamp" | "{{_ingest.timestamp}}" => IngestTimestampValue + case _ => StringValue(s) } - case b: Byte => ByteValue(b) - case s: Short => ShortValue(s) - case i: Int => IntValue(i) - case l: Long => LongValue(l) - case f: Float => FloatValue(f) - case d: Double => DoubleValue(d) - case a: Array[T] => - val values = a.toSeq.map(apply) + case b: Byte => ByteValue(b) + case s: Short => ShortValue(s) + case i: Int => IntValue(i) + case l: Long => LongValue(l) + case f: Float => FloatValue(f) + case d: Double => DoubleValue(d) + case a: Array[T] => apply(a.toSeq) + case a: Seq[T] => + val values = a.map(apply) values.headOption match { case Some(_: StringValue) => StringValues(values.asInstanceOf[Seq[StringValue]]).asInstanceOf[Values[R, T]] @@ -381,9 +383,12 @@ package object sql { case class ObjectValue(override val value: Map[String, Value[_]]) extends Value[Map[String, Value[_]]](value) { override def sql: String = value - .map { case (k, v) => s""""$k" = ${v.sql}""" } + .map { case (k, v) => s"""$k = ${v.sql}""" } .mkString("(", ", ", ")") override def baseType: SQLType = SQLTypes.Struct + override def ddl: String = value + .map { case (k, v) => s"""$k: ${v.ddl}""" } + .mkString("(", ", ", ")") } case object Null extends Value[Null](null) with TokenRegex { @@ -413,12 +418,15 @@ package object sql { case class StringValue(override val value: String) extends Value[String](value) { override def sql: String = s"""'$value'""" override def baseType: SQLType = SQLTypes.Varchar + + override def ddl: String = value } case object IdValue extends Value[String]("_id") with TokenRegex { override def sql: String = value override def painless(context: Option[PainlessContext]): String = s"{{$value}}" override def baseType: SQLType = SQLTypes.Varchar + override def ddl: String = value } case object IngestTimestampValue extends Value[String]("_ingest.timestamp") with TokenRegex { @@ -426,6 +434,7 @@ package object sql { override def painless(context: Option[PainlessContext]): String = s"{{$value}}" override def baseType: SQLType = SQLTypes.Timestamp + override def ddl: String = value } sealed abstract class NumericValue[T: Numeric](override val value: T) extends Value[T](value) { @@ -553,6 +562,8 @@ package object sql { lazy val innerValues: Seq[R] = values.map(_.value) override def nullable: Boolean = values.exists(_.nullable) override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Any) + + override def ddl: String = s"[${values.map(_.ddl).mkString(",")}]" } case class StringValues(override val values: Seq[StringValue]) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 7d96a2bd..9cf31935 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -117,6 +117,12 @@ object Parser case None => None } + def comment: PackratParser[Option[String]] = + opt("COMMENT" ~ literal) ^^ { + case Some(_ ~ v) => Some(v.value) + case None => None + } + def script: PackratParser[PainlessScript] = ("SCRIPT" ~ "AS") ~ start ~ (identifierWithArithmeticExpression | identifierWithTransformation | @@ -124,9 +130,9 @@ object Parser identifierWithFunction) ~ end ^^ { case _ ~ _ ~ s ~ _ => s } def column: PackratParser[DdlColumn] = - ident ~ extension_type ~ (script | multiFields) ~ notNull ~ defaultVal ~ (options | success( + ident ~ extension_type ~ (script | multiFields) ~ defaultVal ~ notNull ~ comment ~ (options | success( Map.empty[String, Value[_]] - )) ^^ { case name ~ dt ~ mfs ~ nn ~ dv ~ opts => + )) ^^ { case name ~ dt ~ mfs ~ dv ~ nn ~ ct ~ opts => mfs match { case script: PainlessScript => val ctx = PainlessContext(Processor) @@ -154,12 +160,13 @@ object Parser ) ), Nil, - nn, dv, + nn, + ct, opts ) case cols: List[DdlColumn] => - DdlColumn(name, dt, None, cols, nn, dv, opts) + DdlColumn(name, dt, None, cols, dv, nn, ct, opts) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 85597768..f6336cfa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,16 +16,14 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.schema.{Field, Index} -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization.JacksonConfig import app.softnetwork.elastic.sql.time.TimeUnit -import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode import scala.language.implicitConversions -import scala.jdk.CollectionConverters._ package object schema { val mapper: ObjectMapper = JacksonConfig.objectMapper @@ -238,93 +236,6 @@ package object schema { } } - object DdlProcessor { - private val ScriptDescRegex = - """^\s*([a-zA-Z0-9_]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r - - def apply(node: JsonNode): Option[DdlProcessor] = { - val processorType = node.fieldNames().next() // "set", "script", "date_index_name", etc. - val props = node.get(processorType) - - processorType match { - case "set" => - val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") - val valueNode = props.get("value") - val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) - - if (field == "_id" && desc.startsWith("PRIMARY KEY")) { - // DdlPrimaryKeyProcessor - // description: "PRIMARY KEY (id)" - val inside = desc.stripPrefix("PRIMARY KEY").trim.stripPrefix("(").stripSuffix(")") - val cols = inside.split(",").map(_.trim).filter(_.nonEmpty).toSet - Some( - DdlPrimaryKeyProcessor( - sql = desc, - column = "_id", - value = cols, - ignoreFailure = ignoreFailure - ) - ) - } else if (desc.startsWith(s"$field DEFAULT")) { - Some( - DdlDefaultValueProcessor( - sql = desc, - column = field, - value = Value(valueNode.asText()), - ignoreFailure = ignoreFailure - ) - ) - } else { - None - } - - case "script" => - val desc = props.get("description").asText() - val lang = props.get("lang").asText() - require(lang == "painless", s"Only painless supported, got $lang") - val source = props.get("source").asText() - val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) - - desc match { - case ScriptDescRegex(col, dataType, script) => - Some( - DdlScriptProcessor( - script = script, - column = col, - dataType = SQLTypes(dataType), - source = source, - ignoreFailure = ignoreFailure - ) - ) - case _ => - None - } - - case "date_index_name" => - val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") - val rounding = props.get("date_rounding").asText() - val formats = Option(props.get("date_formats")) - .map(_.elements().asScala.toList.map(_.asText())) - .getOrElse(Nil) - val prefix = props.get("index_name_prefix").asText() - - Some( - DdlDateIndexNameProcessor( - sql = desc, - column = field, - dateRounding = rounding, - dateFormats = formats, - prefix = prefix - ) - ) - - case _ => None - } - } - } - sealed trait DdlPipelineType { def name: String } @@ -366,15 +277,33 @@ package object schema { private[schema] def update(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { updates.foreach { case (k, v) => v match { - case Null => node.putNull(k) - case BooleanValue(b) => node.put(k, b) - case StringValue(s) => node.put(k, s) - case ByteValue(b) => node.put(k, b) - case ShortValue(s) => node.put(k, s) - case IntValue(i) => node.put(k, i) - case LongValue(l) => node.put(k, l) - case DoubleValue(d) => node.put(k, d) - case FloatValue(f) => node.put(k, f) + case Null => node.putNull(k) + case BooleanValue(b) => node.put(k, b) + case StringValue(s) => node.put(k, s) + case ByteValue(b) => node.put(k, b) + case ShortValue(s) => node.put(k, s) + case IntValue(i) => node.put(k, i) + case LongValue(l) => node.put(k, l) + case DoubleValue(d) => node.put(k, d) + case FloatValue(f) => node.put(k, f) + case IdValue => node.put(k, s"${v.value}") + case IngestTimestampValue => node.put(k, s"${v.value}") + case v: Values[_, _] => + val arrayNode = mapper.createArrayNode() + v.values.foreach { + case Null => arrayNode.addNull() + case BooleanValue(b) => arrayNode.add(b) + case StringValue(s) => arrayNode.add(s) + case ByteValue(b) => arrayNode.add(b) + case ShortValue(s) => arrayNode.add(s) + case IntValue(i) => arrayNode.add(i) + case LongValue(l) => arrayNode.add(l) + case DoubleValue(d) => arrayNode.add(d) + case FloatValue(f) => arrayNode.add(f) + case ObjectValue(o) => arrayNode.add(update(mapper.createObjectNode(), o)) + case _ => // do nothing + } + node.set(k, arrayNode) case ObjectValue(value) => if (value.nonEmpty) node.set(k, update(mapper.createObjectNode(), value)) @@ -389,25 +318,27 @@ package object schema { dataType: SQLType, script: Option[DdlScriptProcessor] = None, multiFields: List[DdlColumn] = Nil, - notNull: Boolean = false, defaultValue: Option[Value[_]] = None, + notNull: Boolean = false, + comment: Option[String] = None, options: Map[String, Value[_]] = Map.empty ) extends Token { def sql: String = { val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = $v" }.mkString(", ")}) " + s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")}) " } else { "" } + val defaultOpt = defaultValue.map(v => s" DEFAULT ${v.ddl}").getOrElse("") val notNullOpt = if (notNull) " NOT NULL" else "" - val defaultOpt = defaultValue.map(v => s" DEFAULT $v").getOrElse("") + val commentOpt = comment.map(c => s" COMMENT '$c'").getOrElse("") val fieldsOpt = if (multiFields.nonEmpty) { s" FIELDS (${multiFields.mkString(", ")})" } else { "" } val scriptOpt = script.map(s => s" SCRIPT AS (${s.script})").getOrElse("") - s"$name $dataType$fieldsOpt$scriptOpt$notNullOpt$defaultOpt$opts" + s"$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" } def ddlProcessors: Seq[DdlProcessor] = script.toSeq ++ @@ -438,23 +369,38 @@ package object schema { } root.set(name, fieldsNode) } - update(root, options) + val sql_script = + script match { + case Some(s) => + Some( + ObjectValue( + Map( + "sql" -> StringValue(s.script), + "painless" -> StringValue(s.source) + ) + ) + ) + case _ => None + } + val meta = options.get("meta") match { + case Some(ObjectValue(value)) => + ObjectValue( + value ++ Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ sql_script.map(st => "script" -> st) + ) + case _ => + ObjectValue( + Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ sql_script.map(st => "script" -> st) + ) + } + update(root, options ++ Map("meta" -> meta)) root } } - object DdlColumn { - def apply(field: Field): DdlColumn = { - DdlColumn( - name = field.name, - dataType = SQLTypes(field), - multiFields = field.fields.map(apply), - defaultValue = field.null_value, - options = field.options - ) - } - } - case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { def sql: String = s" PARTITION BY $column ($granularity)" @@ -500,13 +446,17 @@ package object schema { if (mappings.nonEmpty || settings.nonEmpty) { val mappingOpts = if (mappings.nonEmpty) { - s"mappings = (${mappings.map { case (k, v) => s"$k = $v" }.mkString(", ")})" + s"mappings = (${mappings + .map { case (k, v) => + s"$k = ${v.ddl}" + } + .mkString(", ")})" } else { "" } val settingsOpts = if (settings.nonEmpty) { - s"settings = (${mappings.map { case (k, v) => s"$k = $v" }.mkString(", ")})" + s"settings = (${mappings.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")})" } else { "" } @@ -711,75 +661,4 @@ package object schema { } } - object DdlTable { - def apply(index: Index): DdlTable = { - // 1. Columns from the mapping - val initialCols: Map[String, DdlColumn] = - index.mappings.fields.map { field => - val name = field.name - name -> DdlColumn( - name = name, - dataType = SQLTypes(field), - script = None, - multiFields = field.fields.map(DdlColumn(_)), - notNull = false, // TODO add required - defaultValue = field.null_value, - options = field.options - ) - }.toMap - - // 2. PK + partition + pipelines from index mappings and settings - var primaryKey: List[String] = index.mappings.primaryKey - var partitionBy: Option[DdlPartition] = index.mappings.partitionBy.map { p => - val granularity = TimeUnit(p.granularity) - DdlPartition(p.column, granularity) - } - - // 3. Enrichment from the pipeline (if provided) - val enrichedCols = scala.collection.mutable.Map.from(initialCols) - - index.pipeline.foreach { pipeline => - val processorsNode = pipeline.get("processors") - if (processorsNode != null && processorsNode.isArray) { - val processors: Seq[DdlProcessor] = - processorsNode.elements().asScala.toSeq.flatMap(DdlProcessor(_)) - - processors.foreach { - case p: DdlScriptProcessor => - val col = p.column - enrichedCols.get(col).foreach { c => - enrichedCols.update(col, c.copy(script = Some(p))) - } - - case p: DdlDefaultValueProcessor => - val col = p.column - enrichedCols.get(col).foreach { c => - enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) - } - - case p: DdlDateIndexNameProcessor => - if (partitionBy.isEmpty) { - val granularity = TimeUnit(p.dateRounding) - partitionBy = Some(DdlPartition(p.column, granularity)) - } - - case p: DdlPrimaryKeyProcessor => - if (primaryKey.isEmpty) { - primaryKey = p.value.toList - } - - case _ => // ignore others (rename/remove...) ou gère-les si tu veux les remonter en DDL - } - } - } - - // 4. Final construction of the DdlTable - DdlTable( - name = index.name, - columns = enrichedCols.values.toList.sortBy(_.name), - primaryKey = primaryKey, - partitionBy = partitionBy - ) - } - } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 11bc13a5..6a15c837 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.`type` -import app.softnetwork.elastic.schema.Field +import app.softnetwork.elastic.schema.EsField object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } @@ -57,7 +57,7 @@ object SQLTypes { def apply(typeName: String): SQLType = typeName.toLowerCase match { case "null" => Null case "boolean" => Boolean - case "int" => Int + case "int" | "integer" => Int case "long" | "bigint" => BigInt case "short" | "smallint" => SmallInt case "byte" | "tinyint" => TinyInt @@ -74,7 +74,7 @@ object SQLTypes { case _ => Any } - def apply(field: Field): SQLType = field.`type` match { + def apply(field: EsField): SQLType = field.`type` match { case "null" => Null case "boolean" => Boolean case "integer" => Int diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 0cf2a475..ad926c31 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,9 +1,10 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.schema.EsIndex import app.softnetwork.elastic.sql.{ObjectValue, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.DdlPartition +import app.softnetwork.elastic.sql.schema.{mapper, DdlPartition} import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -932,8 +933,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { it should "parse CREATE TABLE if not exists" in { val sql = """CREATE TABLE IF NOT EXISTS users ( - | id INT NOT NULL, - | name VARCHAR FIELDS(raw Keyword) DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), + | id INT NOT NULL COMMENT 'user identifier', + | name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), | birthdate DATE, | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, @@ -979,13 +980,29 @@ class ParserSpec extends AnyFlatSpec with Matchers { json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings println(indexMappings) - indexMappings.toString shouldBe "{\"properties\":{\"id\":{\"type\":\"integer\"},\"name\":{\"type\":\"text\",\"null_value\":\"anonymous\",\"fields\":{\"raw\":{\"type\":\"keyword\"}},\"analyzer\":\"french\",\"search_analyzer\":\"french\"},\"birthdate\":{\"type\":\"date\"},\"age\":{\"type\":\"integer\"},\"ingested_at\":{\"type\":\"date\"}},\"dynamic\":false,\"_meta\":{\"primary_key\":[\"id\"],\"partition_by\":{\"column\":\"birthdate\",\"granularity\":\"M\"}}}" + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","painless":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" val indexSettings = ct.ddlTable.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = ct.ddlTable.pipeline println(pipeline) pipeline.toString shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + // Reconstruct EsIndex + val mappings = mapper.createObjectNode() + mappings.set("mappings", indexMappings) + val settings = mapper.createObjectNode() + settings.set("settings", indexSettings) + val esIndex = EsIndex( + name = "users", + mappings = mappings, + settings = settings, + pipeline = None + ) + val ddlTable = esIndex.ddlTable + println(s"""esIndex ddl -> ${ddlTable.sql}""") + println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") + println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") + println(s"""esIndex pipeline -> ${ddlTable.pipeline.toString}""") case _ => fail("Expected CreateTable") } } From de794890cce1ceed1ee8090a6ba4efc49702e1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 16 Dec 2025 15:38:24 +0100 Subject: [PATCH 14/95] add struct type to the parser, add support for scripts within fields, fix painless within processor script --- documentation/sql/ddl_statements.md | 59 +++++++++++++++---- .../softnetwork/elastic/schema/package.scala | 2 +- .../app/softnetwork/elastic/sql/package.scala | 2 +- .../elastic/sql/parser/Parser.scala | 4 +- .../elastic/sql/parser/type/package.scala | 5 +- .../elastic/sql/query/package.scala | 2 +- .../elastic/sql/schema/package.scala | 49 ++++++++++----- .../elastic/sql/parser/ParserSpec.scala | 14 +++-- 8 files changed, 101 insertions(+), 36 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index b738c529..a4f77322 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -13,10 +13,10 @@ Create a new table with explicit column definitions or from a `SELECT` query. **Syntax:** ```sql CREATE [OR REPLACE] TABLE [IF NOT EXISTS] table_name ( - column_name data_type [NOT NULL] [DEFAULT value] [OPTIONS (...)] [FIELDS (...)], + column_name data_type [SCRIPT AS (sql) | FIELDS (...)] [DEFAULT value] [NOT NULL] [COMMENT 'comment'] [OPTIONS (...)], [... more columns ...], [PRIMARY KEY (column1, column2, ...)] -) [PARTITION BY column_name] +) [PARTITION BY column_name] OPTIONS ([mappings = (...)] , [settings = (...)] ); ``` - `FIELDS (...)` can define **multi‑fields** (alternative analyzers for text) or **STRUCT** (nested objects). @@ -24,9 +24,13 @@ CREATE [OR REPLACE] TABLE [IF NOT EXISTS] table_name ( **Examples:** ```sql CREATE TABLE IF NOT EXISTS users ( - id INT NOT NULL, - name VARCHAR DEFAULT 'anonymous' -); + id INT NOT NULL COMMENT 'user identifier', + name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), + birthdate DATE, + age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), + ingested_at TIMESTAMP DEFAULT _ingest.timestamp, -- special field + PRIMARY KEY (id) +) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false)); CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; ``` @@ -36,7 +40,9 @@ CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; ## 📝 Notes - **Types**: Supported SQL types include `INT`, `BIGINT`, `VARCHAR`, `BOOLEAN`, `DATE`, `TIMESTAMP`, etc. - **Constraints**: `NOT NULL` and `DEFAULT` are supported. Other relational constraints (e.g., `PRIMARY KEY`) are not enforced by Elasticsearch. +- **SCRIPT AS (sql)**: Defines a scripted column computed at ingestion time. - **FIELDS**: Dual purpose — multi‑fields for text analysis and STRUCT for nested data modeling. +- **Comments**: Column comments can be added via `COMMENT 'text'`. - **Options**: Column options can be specified via `OPTIONS (...)`. - The **partition key must be of type `DATE`** and the partition column must be explicitly defined in the table schema. @@ -129,8 +135,8 @@ CREATE TABLE docs ( --- -### FIELDS for STRUCT -`FIELDS (...)` also enables the definition of **STRUCT** types, which represent nested objects with their own fields. This is how you model hierarchical data. +### FIELDS for STRUCT or NESTED OBJECTS +`FIELDS (...)` also enables the definition of **STRUCT** types, which may represent either object or nested objects with their own fields. This is how you model hierarchical data. **Example:** ```sql @@ -143,7 +149,9 @@ CREATE TABLE users ( street VARCHAR, city VARCHAR, zip VARCHAR - ) + ), + join_date DATE, + seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) ) ) ``` @@ -151,9 +159,38 @@ CREATE TABLE users ( - `profile` is a `STRUCT` column containing multiple fields. - `address` is a nested `STRUCT` inside `profile`. -This maps naturally to: -- **Elasticsearch**: `object` or `nested` type. -- **Avro**: `record`. +This maps naturally to **Elasticsearch** `object` type. + +**Example:** +```sql +CREATE TABLE store ( + id INT NOT NULL, + products ARRAY FIELDS ( + name VARCHAR NOT NULL, + description VARCHAR NOT NULL, + price BIGINT NOT NULL + ) +) +``` + +- `products` is an `ARRAY` column containing multiple fields. + +This maps naturally to **Elasticsearch** `nested` type. + +### 📝 Notes +- The meaning of `FIELDS (...)` depends on the column type: + - On `VARCHAR` types, it defines **multi‑fields**. + - On `STRUCT`, it defines the **fields of the struct**. + - On `ARRAY`, it defines the **fields of each element**. +- Sub‑fields defined inside `FIELDS (...)` support the full DDL syntax: + - nested `FIELDS` + - `SCRIPT AS (sql)` (for scripted sub‑fields) + - `DEFAULT` + - `NOT NULL` + - `COMMENT` + - `OPTIONS` +- Multi‑level nesting is supported. +- `SCRIPT AS (sql)` can not be used for `ARRAY`. --- diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 49004786..371e8930 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -390,7 +390,7 @@ package object schema { partitionBy = partitionBy, esMappings.options, esSettings.options - ) + ).update() } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 55567e55..80159492 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -842,7 +842,7 @@ package object sql { lazy val processParamName: String = { if (path.nonEmpty) - s"ctx['$path']" + s"ctx.$path" else "" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 9cf31935..d362969d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -85,9 +85,7 @@ object Parser } def multiFields: PackratParser[List[DdlColumn]] = - "FIELDS" ~ start ~ repsep(column, separator) ~ end ^^ { case _ ~ _ ~ cols ~ _ => - cols - } | success(Nil) + "FIELDS" ~ start ~> repsep(column, separator) <~ end ^^ (cols => cols) | success(Nil) def ifExists: PackratParser[Boolean] = opt("IF" ~ "EXISTS") ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index d94ba023..b25f96dd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -113,13 +113,16 @@ package object `type` { def float_type: PackratParser[SQLTypes.Real.type] = "(?i)float|real".r ^^ (_ => SQLTypes.Real) + def struct_type: PackratParser[SQLTypes.Struct.type] = + "(?i)struct".r ^^ (_ => SQLTypes.Struct) + def array_type: PackratParser[SQLTypes.Array] = "(?i)array<".r ~> sql_type <~ ">" ^^ { elementType => SQLTypes.Array(elementType) } def sql_type: PackratParser[SQLType] = - char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type | array_type + char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type | struct_type | array_type def text_type: PackratParser[SQLTypes.Text.type] = "(?i)text".r ^^ (_ => SQLTypes.Text) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index d77e5db2..7a27acfc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -376,7 +376,7 @@ package object query { partitionBy = partitionBy, mappings = mappings, settings = settings - ) + ).update() lazy val ddlPipeline: DdlPipeline = ddlTable.ddlPipeline diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index f6336cfa..abf6f903 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -274,7 +274,7 @@ package object schema { def json: String = node.toString } - private[schema] def update(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { + private[schema] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { updates.foreach { case (k, v) => v match { case Null => node.putNull(k) @@ -300,13 +300,13 @@ package object schema { case LongValue(l) => arrayNode.add(l) case DoubleValue(d) => arrayNode.add(d) case FloatValue(f) => arrayNode.add(f) - case ObjectValue(o) => arrayNode.add(update(mapper.createObjectNode(), o)) + case ObjectValue(o) => arrayNode.add(updateNode(mapper.createObjectNode(), o)) case _ => // do nothing } node.set(k, arrayNode) case ObjectValue(value) => if (value.nonEmpty) - node.set(k, update(mapper.createObjectNode(), value)) + node.set(k, updateNode(mapper.createObjectNode(), value)) case _ => // do nothing } } @@ -321,11 +321,28 @@ package object schema { defaultValue: Option[Value[_]] = None, notNull: Boolean = false, comment: Option[String] = None, - options: Map[String, Value[_]] = Map.empty + options: Map[String, Value[_]] = Map.empty, + struct: Option[DdlColumn] = None ) extends Token { + def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) + def level: Int = struct.map(_.level + 1).getOrElse(0) + def update(struct: Option[DdlColumn] = None): DdlColumn = { + val updated = this.copy(struct = struct) + updated.copy( + multiFields = multiFields.map { field => + field.update(Some(updated)) + }, + script = script.map { sc => + sc.copy( + column = updated.path, + source = sc.source.replace(s"ctx.$name", s"ctx.${updated.path}") + ) + } + ) + } def sql: String = { val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")}) " + s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")})" } else { "" } @@ -333,29 +350,30 @@ package object schema { val notNullOpt = if (notNull) " NOT NULL" else "" val commentOpt = comment.map(c => s" COMMENT '$c'").getOrElse("") val fieldsOpt = if (multiFields.nonEmpty) { - s" FIELDS (${multiFields.mkString(", ")})" + s" FIELDS (\n\t${multiFields.mkString(s",\n\t")}\n\t)" } else { "" } val scriptOpt = script.map(s => s" SCRIPT AS (${s.script})").getOrElse("") - s"$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" + val tabs = "\t" * level + s"$tabs$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" } - def ddlProcessors: Seq[DdlProcessor] = script.toSeq ++ + def ddlProcessors: Seq[DdlProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => DdlDefaultValueProcessor( sql = s"$name DEFAULT $dv", - column = name, + column = path, value = dv ) - }.toSeq + }.toSeq ++ multiFields.flatMap(_.ddlProcessors) def node: ObjectNode = { val root = mapper.createObjectNode() val esType = SQLTypeUtils.elasticType(dataType) root.put("type", esType) defaultValue.foreach { dv => - update(root, Map("null_value" -> dv)) + updateNode(root, Map("null_value" -> dv)) } if (multiFields.nonEmpty) { val name = @@ -376,6 +394,7 @@ package object schema { ObjectValue( Map( "sql" -> StringValue(s.script), + "column" -> StringValue(path), "painless" -> StringValue(s.source) ) ) @@ -396,7 +415,7 @@ package object schema { ) ++ sql_script.map(st => "script" -> st) ) } - update(root, options ++ Map("meta" -> meta)) + updateNode(root, options ++ Map("meta" -> meta)) root } } @@ -441,6 +460,8 @@ package object schema { ) extends Token { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap + def update(): DdlTable = this.copy(columns = columns.map(_.update())) + def sql: String = { val opts = if (mappings.nonEmpty || settings.nonEmpty) { @@ -626,7 +647,7 @@ package object schema { fields.replace(column.name, column.node) } node.set("properties", fields) - update(node, mappings) + updateNode(node, mappings) if (primaryKey.nonEmpty || partitionBy.nonEmpty) { val meta = Option(node.get("_meta")).getOrElse(mapper.createObjectNode()) if (meta != null && meta.isObject) { @@ -651,7 +672,7 @@ package object schema { lazy val indexSettings: ObjectNode = { val node = mapper.createObjectNode() val index = mapper.createObjectNode() - update(index, settings) + updateNode(index, settings) node.set("index", index) node } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index ad926c31..3f446ad1 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -938,6 +938,12 @@ class ParserSpec extends AnyFlatSpec with Matchers { | birthdate DATE, | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) COMMENT 'user profile', | PRIMARY KEY (id) |) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false))""".stripMargin val result = Parser(sql) @@ -964,7 +970,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { .script .map(p => p.source) .getOrElse("") should include( - """def param1 = ctx['birthdate']; + """def param1 = ctx.birthdate; |def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); |ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin .replaceAll("\n", " ") @@ -977,16 +983,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(sql) val json = ct.ddlTable.ddlPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","painless":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" val indexSettings = ct.ddlTable.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = ct.ddlTable.pipeline println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx['birthdate']; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) From 47d0abb42fd2c0a708bd49fb5414a4bc80221802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 17 Dec 2025 07:19:43 +0100 Subject: [PATCH 15/95] add alter table statements (alter/drop column script, field, comment, option - alter/drop mapping, setting) --- documentation/sql/ddl_statements.md | 36 +++-- .../elastic/sql/parser/Parser.scala | 112 +++++++++++-- .../elastic/sql/query/package.scala | 101 +++++++++++- .../elastic/sql/schema/package.scala | 147 ++++++++++++++++++ 4 files changed, 368 insertions(+), 28 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index a4f77322..3f450e22 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -52,17 +52,31 @@ CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; Modify an existing table. Multiple statements can be grouped inside parentheses. **Supported statements:** -- `ADD COLUMN [IF NOT EXISTS] column_definition` -- `DROP COLUMN [IF EXISTS] column_name` -- `RENAME COLUMN old_name TO new_name` -- `ALTER COLUMN [IF EXISTS] column_name SET OPTIONS (...)` -- `ALTER COLUMN [IF EXISTS] column_name SET DEFAULT value` -- `ALTER COLUMN [IF EXISTS] column_name DROP DEFAULT` -- `ALTER COLUMN [IF EXISTS] column_name SET NOT NULL` -- `ALTER COLUMN [IF EXISTS] column_name DROP NOT NULL` -- `ALTER COLUMN [IF EXISTS] column_name SET DATA TYPE new_type` -- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` - → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column. +- `ADD COLUMN [IF NOT EXISTS] column_definition` → Add a new column. +- `DROP COLUMN [IF EXISTS] column_name` → Remove an existing column. +- `RENAME COLUMN old_name TO new_name` → Rename an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET SCRIPT AS (sql)` → Define or update a scripted column. +- `ALTER COLUMN [IF EXISTS] column_name DROP SCRIPT` → Remove a scripted column. +- `ALTER COLUMN [IF EXISTS] column_name SET|ADD OPTION (key = value)` → Set a specific option for an existing column. +- `ALTER COLUMN [IF EXISTS] column_name DROP OPTION key` → Remove a specific option from an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET COMMENT 'comment'` → Set or update the comment for an existing column. +- `ALTER COLUMN [IF EXISTS] column_name DROP COMMENT` → Remove the comment from an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET DEFAULT value` → Set or update the default value for an existing column. +- `ALTER COLUMN [IF EXISTS] column_name DROP DEFAULT` → Remove the default value from an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET NOT NULL` → Make an existing column NOT NULL. +- `ALTER COLUMN [IF EXISTS] column_name DROP NOT NULL` → Remove the NOT NULL constraint from an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET DATA TYPE new_type` → Change the data type of an existing column. +- `ALTER COLUMN [IF EXISTS] column_name SET|ADD FIELD field_definition` → Add or update a field inside a STRUCT or multi‑field. +- `ALTER COLUMN [IF EXISTS] column_name DROP FIELD field_name` → Remove a field from a STRUCT or multi‑field. +- `SET|ADD MAPPING (key = value)` → Set table‑level mapping. +- `DROP MAPPING key` → Remove table‑level mapping. +- `SET|ADD SETTING (key = value)` → Set table‑level setting. +- `DROP SETTING key` → Remove table‑level setting. + +[//]: # (- `ALTER COLUMN [IF EXISTS] column_name SET OPTIONS (...)`) +[//]: # ( → Set multiple options for an existing column.) +[//]: # (- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` ) +[//]: # ( → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column.) **Examples:** ```sql diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index d362969d..d80a88f0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -271,22 +271,76 @@ object Parser ie } - def setColumnOptions: PackratParser[AlterColumnOptions] = + def alterColumnOptions: PackratParser[AlterColumnOptions] = alterColumnIfExists ~ ident ~ "SET" ~ options ^^ { case ie ~ col ~ _ ~ opts => AlterColumnOptions(col, opts, ifExists = ie) } - def setColumnFields: PackratParser[AlterColumnFields] = + def alterColumnOption: PackratParser[AlterColumnOption] = + alterColumnIfExists ~ ident ~ (("SET" | "ADD") ~ "OPTION") ~ start ~ option ~ end ^^ { + case ie ~ col ~ _ ~ _ ~ opt ~ _ => + AlterColumnOption(col, opt._1, opt._2, ifExists = ie) + } + + def dropColumnOption: PackratParser[DropColumnOption] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "OPTION") ~ ident ^^ { case ie ~ col ~ _ ~ optionName => + DropColumnOption(col, optionName, ifExists = ie) + } + + def alterColumnFields: PackratParser[AlterColumnFields] = alterColumnIfExists ~ ident ~ "SET" ~ multiFields ^^ { case ie ~ col ~ _ ~ fields => AlterColumnFields(col, fields, ifExists = ie) } - def setColumnType: PackratParser[AlterColumnType] = + def alterColumnField: PackratParser[AlterColumnField] = + alterColumnIfExists ~ ident ~ (("SET" | "ADD") ~ "FIELD") ~ column ^^ { + case ie ~ col ~ _ ~ field => + AlterColumnField(col, field, ifExists = ie) + } + + def dropColumnField: PackratParser[DropColumnField] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "FIELD") ~ ident ^^ { case ie ~ col ~ _ ~ fieldName => + DropColumnField(col, fieldName, ifExists = ie) + } + + def alterColumnType: PackratParser[AlterColumnType] = alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ extension_type ^^ { case ie ~ name ~ _ ~ newType => AlterColumnType(name, newType, ifExists = ie) } - def setColumnDefault: PackratParser[AlterColumnDefault] = + def alterColumnScript: PackratParser[AlterColumnScript] = + alterColumnIfExists ~ ident ~ "SET" ~ script ^^ { case ie ~ name ~ _ ~ ns => + val ctx = PainlessContext(Processor) + val scr = ns.painless(Some(ctx)) + val temp = s"$ctx$scr" + val ret = + temp.split(";") match { + case Array(single) if single.trim.startsWith("return ") => + val stripReturn = single.trim.stripPrefix("return ").trim + s"ctx.$name = $stripReturn" + case multiple => + val last = multiple.last.trim + val temp = multiple.dropRight(1) :+ s" ctx.$name = $last" + temp.mkString(";") + } + AlterColumnScript( + name, + DdlScriptProcessor( + script = ns.sql, + column = name, + dataType = ns.out, + source = ret + ), + ifExists = ie + ) + } + + def dropColumnScript: PackratParser[DropColumnScript] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "SCRIPT") ^^ { case ie ~ name ~ _ => + DropColumnScript(name, ifExists = ie) + } + + def alterColumnDefault: PackratParser[AlterColumnDefault] = alterColumnIfExists ~ ident ~ ("SET" ~ "DEFAULT") ~ value ^^ { case ie ~ name ~ _ ~ dv => AlterColumnDefault(name, dv, ifExists = ie) } @@ -296,7 +350,7 @@ object Parser DropColumnDefault(name, ifExists = ie) } - def setColumnNotNull: PackratParser[AlterColumnNotNull] = + def alterColumnNotNull: PackratParser[AlterColumnNotNull] = alterColumnIfExists ~ ident ~ ("SET" ~ "NOT" ~ "NULL") ^^ { case ie ~ name ~ _ => AlterColumnNotNull(name, ifExists = ie) } @@ -306,17 +360,55 @@ object Parser DropColumnNotNull(name, ifExists = ie) } + def alterColumnComment: PackratParser[AlterColumnComment] = + alterColumnIfExists ~ ident ~ ("SET" ~ "COMMENT") ~ literal ^^ { case ie ~ name ~ _ ~ c => + AlterColumnComment(name, c.value, ifExists = ie) + } + + def dropColumnComment: PackratParser[DropColumnComment] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "COMMENT") ^^ { case ie ~ name ~ _ => + DropColumnComment(name, ifExists = ie) + } + + def alterTableMapping: PackratParser[AlterTableMapping] = + (("SET" | "ADD") ~ "MAPPING") ~ start ~> option <~ end ^^ { opt => + AlterTableMapping(opt._1, opt._2) + } + + def dropTableMapping: PackratParser[DropTableMapping] = + ("DROP" ~ "MAPPING") ~> ident ^^ { m => DropTableMapping(m) } + + def alterTableSetting: PackratParser[AlterTableSetting] = + (("SET" | "ADD") ~ "SETTING") ~ start ~> option <~ end ^^ { opt => + AlterTableSetting(opt._1, opt._2) + } + + def dropTableSetting: PackratParser[DropTableSetting] = + ("DROP" ~ "SETTING") ~> ident ^^ { m => DropTableSetting(m) } + def alterTableStatement: PackratParser[AlterTableStatement] = addColumn | dropColumn | renameColumn | - setColumnOptions | - setColumnType | - setColumnDefault | + alterColumnOptions | + alterColumnOption | + dropColumnOption | + alterColumnType | + alterColumnScript | + dropColumnScript | + alterColumnDefault | dropColumnDefault | - setColumnNotNull | + alterColumnNotNull | dropColumnNotNull | - setColumnFields + alterColumnComment | + dropColumnComment | + alterColumnFields | + alterColumnField | + dropColumnField | + alterTableMapping | + dropTableMapping | + alterTableSetting | + dropTableSetting def alterTable: PackratParser[AlterTable] = ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ start.? ~ repsep( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 7a27acfc..3e807344 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -26,6 +26,7 @@ import app.softnetwork.elastic.sql.schema.{ DdlProcessor, DdlRemoveProcessor, DdlRenameProcessor, + DdlScriptProcessor, DdlTable } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction @@ -434,6 +435,27 @@ package object query { .mkString(", ")})" } } + case class AlterColumnOption( + columnName: String, + optionKey: String, + optionValue: Value[_], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET OPTION ($optionKey = $optionValue)" + } + } + case class DropColumnOption( + columnName: String, + optionKey: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP OPTION $optionKey" + } + } case class AlterColumnType(columnName: String, newType: SQLType, ifExists: Boolean = false) extends AlterTableStatement { override def sql: String = { @@ -441,6 +463,23 @@ package object query { s"ALTER COLUMN$ifExistsClause $columnName SET TYPE $newType" } } + case class AlterColumnScript( + columnName: String, + newScript: DdlScriptProcessor, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET SCRIPT AS (${newScript.script})" + } + } + case class DropColumnScript(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP SCRIPT" + } + } case class AlterColumnDefault( columnName: String, defaultValue: Value[_], @@ -465,13 +504,23 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"ALTER COLUMN$ifExistsClause $columnName DROP DEFAULT" } - override def ddlProcessor: Option[DdlProcessor] = - Some( - DdlRemoveProcessor( - sql, - columnName - ) - ) + } + case class AlterColumnComment( + columnName: String, + comment: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET COMMENT '$comment'" + } + } + case class DropColumnComment(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP COMMENT" + } } case class AlterColumnNotNull(columnName: String, ifExists: Boolean = false) extends AlterTableStatement { @@ -498,6 +547,44 @@ package object query { s"ALTER COLUMN$ifExistsClause $columnName SET FIELDS $fieldsSql" } } + case class AlterColumnField( + columnName: String, + field: DdlColumn, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName ADD FIELD $field" + } + } + case class DropColumnField( + columnName: String, + fieldName: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP FIELD $fieldName" + } + } + case class AlterTableMapping(optionKey: String, optionValue: Value[_]) + extends AlterTableStatement { + override def sql: String = + s"SET MAPPING ($optionKey = $optionValue)" + } + case class DropTableMapping(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP MAPPING $optionKey" + } + case class AlterTableSetting(optionKey: String, optionValue: Value[_]) + extends AlterTableStatement { + override def sql: String = + s"SET SETTING ($optionKey = $optionValue)" + } + case class DropTableSetting(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP SETTING $optionKey" + } case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) extends DdlStatement { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index abf6f903..7bc81198 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -529,6 +529,27 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + case AlterColumnScript(columnName, newScript, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(script = Some(newScript.copy(dataType = col.dataType))) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case DropColumnScript(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(script = None) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnDefault(columnName, newDefault, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -580,6 +601,59 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + case AlterColumnOption( + columnName, + optionKey, + optionValue, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options ++ Map(optionKey -> optionValue)) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case DropColumnOption( + columnName, + optionKey, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options - optionKey) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case AlterColumnComment(columnName, newComment, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = Some(newComment)) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case DropColumnComment(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = None) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) case AlterColumnFields(columnName, newFields, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -591,6 +665,79 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + case AlterColumnField( + columnName, + field, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) { + val updatedFields = if (col.multiFields.exists(_.name == field.name)) { + col.multiFields.map { f => + if (f.name == field.name) field else f + } + } else { + col.multiFields :+ field + } + col.copy(multiFields = updatedFields) + } else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case DropColumnField( + columnName, + fieldName, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + multiFields = col.multiFields.filterNot(_.name == fieldName) + ) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case AlterColumnOption( + columnName, + optionKey, + optionValue, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + options = col.options ++ Map(optionKey -> optionValue) + ) + else col + } + ) + else throw DdlColumnNotFound(columnName, table.name) + case AlterTableMapping(optionKey, optionValue) => + table.copy( + mappings = table.mappings ++ Map(optionKey -> optionValue) + ) + case DropTableMapping(optionKey) => + table.copy( + mappings = table.mappings - optionKey + ) + case AlterTableSetting(optionKey, optionValue) => + table.copy( + settings = table.settings ++ Map(optionKey -> optionValue) + ) + case DropTableSetting(optionKey) => + table.copy( + settings = table.settings - optionKey + ) case _ => table } } From 372a291830a1d3102c404b93a6ea1707129a9d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 17 Dec 2025 15:34:39 +0100 Subject: [PATCH 16/95] compute diff between 2 ddl tables --- .../softnetwork/elastic/schema/package.scala | 88 ++-- .../app/softnetwork/elastic/sql/package.scala | 11 +- .../elastic/sql/query/package.scala | 11 + .../elastic/sql/schema/DdlTableDiff.scala | 132 ++++++ .../elastic/sql/schema/package.scala | 385 +++++++++++++++--- .../elastic/sql/type/SQLTypeUtils.scala | 9 +- .../elastic/sql/parser/ParserSpec.scala | 10 +- 7 files changed, 543 insertions(+), 103 deletions(-) create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 371e8930..05c0fbbe 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -3,14 +3,17 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ + mapper, DdlColumn, DdlDateIndexNameProcessor, DdlDefaultValueProcessor, DdlPartition, DdlPrimaryKeyProcessor, DdlProcessor, + DdlProcessorType, DdlScriptProcessor, - DdlTable + DdlTable, + GenericProcessor } import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode @@ -163,8 +166,8 @@ package object schema { .map { case m: ObjectValue => m.value.get("primary_key") match { - case Some(pk: StringValues) => pk.values.map(_.ddl).toList - case Some(pk: StringValue) => List(pk.ddl) + case Some(pk: StringValues) => pk.values.map(_.ddl.replaceAll("\"", "")).toList + case Some(pk: StringValue) => List(pk.ddl.replaceAll("\"", "")) case _ => List.empty } case _ => List.empty @@ -226,7 +229,7 @@ package object schema { private val ScriptDescRegex = """^\s*([a-zA-Z0-9_]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r - lazy val ddlProcesor: Option[DdlProcessor] = { + lazy val ddlProcesor: DdlProcessor = { val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. val props = processor.get(processorType) @@ -241,26 +244,27 @@ package object schema { // DdlPrimaryKeyProcessor // description: "PRIMARY KEY (id)" val inside = desc.stripPrefix("PRIMARY KEY").trim.stripPrefix("(").stripSuffix(")") - val cols = inside.split(",").map(_.trim).filter(_.nonEmpty).toSet - Some( - DdlPrimaryKeyProcessor( - sql = desc, - column = "_id", - value = cols, - ignoreFailure = ignoreFailure - ) + val cols = + inside.split(",").map(_.replaceAll("\"", "")).map(_.trim).filter(_.nonEmpty).toSet + DdlPrimaryKeyProcessor( + sql = desc, + column = "_id", + value = cols, + ignoreFailure = ignoreFailure ) } else if (desc.startsWith(s"$field DEFAULT")) { - Some( - DdlDefaultValueProcessor( - sql = desc, - column = field, - value = Value(valueNode.asText()), - ignoreFailure = ignoreFailure - ) + DdlDefaultValueProcessor( + sql = desc, + column = field, + value = Value(valueNode.asText()), + ignoreFailure = ignoreFailure ) } else { - None + GenericProcessor( + processorType = DdlProcessorType.Set, + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) } case "script" => @@ -272,17 +276,19 @@ package object schema { desc match { case ScriptDescRegex(col, dataType, script) => - Some( - DdlScriptProcessor( - script = script, - column = col, - dataType = SQLTypes(dataType), - source = source, - ignoreFailure = ignoreFailure - ) + DdlScriptProcessor( + script = script, + column = col, + dataType = SQLTypes(dataType), + source = source, + ignoreFailure = ignoreFailure ) case _ => - None + GenericProcessor( + processorType = DdlProcessorType.Script, + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) } case "date_index_name" => @@ -294,17 +300,21 @@ package object schema { .getOrElse(Nil) val prefix = props.get("index_name_prefix").asText() - Some( - DdlDateIndexNameProcessor( - sql = desc, - column = field, - dateRounding = rounding, - dateFormats = formats, - prefix = prefix - ) + DdlDateIndexNameProcessor( + sql = desc, + column = field, + dateRounding = rounding, + dateFormats = formats, + prefix = prefix + ) + + case other => + GenericProcessor( + processorType = DdlProcessorType(other), + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap ) - case _ => None } } } @@ -315,7 +325,7 @@ package object schema { lazy val processors: Seq[DdlProcessor] = { val processorsNode = pipeline.get("processors") if (processorsNode != null && processorsNode.isArray) { - processorsNode.elements().asScala.toSeq.flatMap(EsProcessor(_).ddlProcesor) + processorsNode.elements().asScala.toSeq.map(EsProcessor(_).ddlProcesor) } else { Seq.empty } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 80159492..d6748926 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -72,6 +72,10 @@ package object sql { def shouldBeScripted: Boolean = false } + trait DdlToken extends Token { + def ddl: String = sql + } + trait TokenValue extends Token { def value: Any } @@ -270,7 +274,7 @@ package object sql { case object Distinct extends Expr("DISTINCT") with TokenRegex abstract class Value[+T](val value: T) - extends Token + extends DdlToken with PainlessScript with FunctionWithValue[T] { override def painless(context: Option[PainlessContext]): String = @@ -288,7 +292,6 @@ package object sql { ) override def nullable: Boolean = false - def ddl: String = sql } object Value { @@ -387,7 +390,7 @@ package object sql { .mkString("(", ", ", ")") override def baseType: SQLType = SQLTypes.Struct override def ddl: String = value - .map { case (k, v) => s"""$k: ${v.ddl}""" } + .map { case (k, v) => s"""$k = ${v.ddl}""" } .mkString("(", ", ", ")") } @@ -419,7 +422,7 @@ package object sql { override def sql: String = s"""'$value'""" override def baseType: SQLType = SQLTypes.Varchar - override def ddl: String = value + override def ddl: String = s""""$value"""" } case object IdValue extends Value[String]("_id") with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 3e807344..026d6ceb 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -585,6 +585,17 @@ package object query { override def sql: String = s"DROP SETTING $optionKey" } + case class AddTableProcessor(processor: DdlProcessor) extends AlterTableStatement { + override def sql: String = s"ADD PROCESSOR ${processor.sql}" + override def ddlProcessor: Option[DdlProcessor] = Some(processor) + } + case class DropTableProcessor(column: String) extends AlterTableStatement { + override def sql: String = s"DROP PROCESSOR $column" + } + case class AlterTableProcessor(processor: DdlProcessor) extends AlterTableStatement { + override def sql: String = s"ALTER PROCESSOR ${processor.sql}" + override def ddlProcessor: Option[DdlProcessor] = Some(processor) + } case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) extends DdlStatement { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala new file mode 100644 index 00000000..0e150d93 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala @@ -0,0 +1,132 @@ +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.sql.Value +import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.query._ + +sealed trait AlterTableStatementDiff { + def stmt: AlterTableStatement +} + +sealed trait ColumnDiff extends AlterTableStatementDiff + +case class ColumnAdded(column: DdlColumn) extends ColumnDiff { + override def stmt: AlterTableStatement = AddColumn(column) +} +case class ColumnRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumn(name) +} +// case class ColumnRenamed(oldName: String, newName: String) extends ColumnDiff + +case class ColumnTypeChanged(name: String, from: SQLType, to: SQLType) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnType(name, to) +} + +case class ColumnDefaultSet(name: String, value: Value[_]) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnDefault(name, value) +} +case class ColumnDefaultRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnDefault(name) +} + +case class ColumnScriptSet(name: String, script: DdlScriptProcessor) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnScript(name, script) +} +case class ColumnScriptRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnScript(name) +} + +case class ColumnCommentSet(name: String, comment: String) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnComment(name, comment) +} +case class ColumnCommentRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnComment(name) +} + +case class ColumnNotNullSet(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnNotNull(name) +} +case class ColumnNotNullRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnNotNull(name) +} + +case class ColumnOptionSet(name: String, key: String, value: Value[_]) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnOption(name, key, value) +} +case class ColumnOptionRemoved(name: String, key: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnOption(name, key) +} + +case class FieldAdded(column: String, field: DdlColumn) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnField(column, field) +} +case class FieldRemoved(column: String, fieldName: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnField(column, fieldName) +} + +sealed trait MappingDiff extends AlterTableStatementDiff + +case class MappingSet(key: String, value: Value[_]) extends MappingDiff { + override def stmt: AlterTableStatement = AlterTableMapping(key, value) +} +case class MappingRemoved(key: String) extends MappingDiff { + override def stmt: AlterTableStatement = DropTableMapping(key) +} + +sealed trait SettingDiff extends AlterTableStatementDiff + +case class SettingSet(key: String, value: Value[_]) extends SettingDiff { + override def stmt: AlterTableStatement = AlterTableSetting(key, value) +} +case class SettingRemoved(key: String) extends SettingDiff { + override def stmt: AlterTableStatement = DropTableSetting(key) +} + +sealed trait PipelineDiff + +case class ProcessorAdded(processor: DdlProcessor) extends PipelineDiff +case class ProcessorRemoved(processor: DdlProcessor) extends PipelineDiff +case class ProcessorTypeChanged( + actual: DdlProcessorType, + desired: DdlProcessorType +) +sealed trait ProcessorPropertyDiff +case class ProcessorPropertyAdded(key: String, value: Any) extends ProcessorPropertyDiff +case class ProcessorPropertyRemoved(key: String) extends ProcessorPropertyDiff +case class ProcessorPropertyChanged(key: String, from: Any, to: Any) extends ProcessorPropertyDiff +case class ProcessorDiff( + typeChanged: Option[ProcessorTypeChanged], + propertyDiffs: List[ProcessorPropertyDiff] +) +case class ProcessorChanged( + from: DdlProcessor, + to: DdlProcessor, + diff: ProcessorDiff +) extends PipelineDiff + +case class DdlTableDiff( + columns: List[ColumnDiff], + mappings: List[MappingDiff], + settings: List[SettingDiff], + pipeline: List[PipelineDiff] +) { + def isEmpty: Boolean = + columns.isEmpty && mappings.isEmpty && settings.isEmpty && pipeline.isEmpty + + def alterTable(tableName: String, ifExists: Boolean): Option[AlterTable] = { + if (isEmpty) { + None + } else { + val statements = columns.map(_.stmt) ++ + mappings.map(_.stmt) ++ + settings.map(_.stmt) + Some( + AlterTable( + tableName, + ifExists, + statements + ) + ) + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 7bc81198..19be7814 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -23,6 +23,7 @@ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode +import java.util.UUID import scala.language.implicitConversions package object schema { @@ -48,9 +49,12 @@ package object schema { case object DateIndexName extends DdlProcessorType { def name: String = "date_index_name" } + def apply(n: String): DdlProcessorType = new DdlProcessorType { + override def name: String = n + } } - sealed trait DdlProcessor extends Token { + sealed trait DdlProcessor extends DdlToken { def column: String def ignoreFailure: Boolean final def node: ObjectNode = { @@ -82,11 +86,88 @@ package object schema { node.set(processorType.name, props) node } - def json: String = node.toString + def json: String = mapper.writeValueAsString(node) def processorType: DdlProcessorType def description: String = sql.trim def name: String = processorType.name def properties: Map[String, Any] + + private def normalizeValue(v: Any): Any = v match { + case s: String => s.trim + case b: Boolean => b + case i: Int => i + case l: Long => l + case d: Double => d + case o: ObjectNode => mapper.writeValueAsString(o) // canonical JSON + case seq: Seq[_] => seq.map(normalizeValue) // recursive + case other => other + } + + private def normalizeProperties(): Map[String, Any] = { + properties + .filterNot { case (k, _) => k == "description" } // we should not compare description + .toSeq + .sortBy(_._1) // order properties by key + .map { case (k, v) => (k, normalizeValue(v)) } + .toMap + } + + def diff(to: DdlProcessor): Option[ProcessorDiff] = { + + val from = this + + // 1. Diff of the type + val typeChanged: Option[ProcessorTypeChanged] = + if (from.processorType != to.processorType) + Some(ProcessorTypeChanged(from.processorType, to.processorType)) + else + None + + // 2. Diff of the properties + val fromProps = from.normalizeProperties() + val toProps = to.normalizeProperties() + + val allKeys = fromProps.keySet ++ toProps.keySet + + val propertyDiffs: List[ProcessorPropertyDiff] = + allKeys.toList.flatMap { key => + (fromProps.get(key), toProps.get(key)) match { + + case (None, Some(v)) => + Some(ProcessorPropertyAdded(key, v)) + + case (Some(_), None) => + Some(ProcessorPropertyRemoved(key)) + + case (Some(v1), Some(v2)) if v1 != v2 => + Some(ProcessorPropertyChanged(key, v1, v2)) + + case _ => + None + } + } + + // 3. If nothing has changed → None + if (typeChanged.isEmpty && propertyDiffs.isEmpty) + None + else + Some(ProcessorDiff(typeChanged, propertyDiffs)) + } + + override def ddl: String = s"$column ${processorType.name.toUpperCase} ${Value(properties).ddl}" + } + + case class GenericProcessor(processorType: DdlProcessorType, properties: Map[String, Any]) + extends DdlProcessor { + override def sql: String = ddl + override def column: String = properties.get("field") match { + case Some(s: String) => s + case _ => UUID.randomUUID().toString + } + override def ignoreFailure: Boolean = properties.get("ignore_failure") match { + case Some(b: Boolean) => b + case _ => false + } } case class DdlScriptProcessor( @@ -96,13 +177,13 @@ package object schema { source: String, ignoreFailure: Boolean = true ) extends DdlProcessor { - override def sql: SQL = s"$column $dataType SCRIPT AS ($script)" + override def sql: String = s"$column $dataType SCRIPT AS ($script)" override def baseType: SQLType = dataType def processorType: DdlProcessorType = DdlProcessorType.Script - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "lang" -> "painless", "source" -> source, @@ -121,7 +202,7 @@ package object schema { def sql: String = s"$column RENAME TO $newName" - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "field" -> column, "target_field" -> newName, @@ -139,7 +220,7 @@ package object schema { ) extends DdlProcessor { def processorType: DdlProcessorType = DdlProcessorType.Remove - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "field" -> column, "ignore_failure" -> ignoreFailure, @@ -158,7 +239,7 @@ package object schema { ) extends DdlProcessor { def processorType: DdlProcessorType = DdlProcessorType.Set - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "field" -> column, "value" -> value.mkString("{{", separator, "}}"), @@ -183,7 +264,7 @@ package object schema { s"""ctx.$column == null""" } - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "field" -> column, "value" -> { @@ -208,7 +289,7 @@ package object schema { ) extends DdlProcessor { def processorType: DdlProcessorType = DdlProcessorType.DateIndexName - override def properties: Map[SQL, Any] = Map( + override def properties: Map[String, Any] = Map( "description" -> description, "field" -> column, "date_rounding" -> dateRounding, @@ -256,9 +337,9 @@ package object schema { name: String, ddlPipelineType: DdlPipelineType, ddlProcessors: Seq[DdlProcessor] - ) extends Token { + ) extends DdlToken { def sql: String = - s"CREATE OR REPLACE ${ddlPipelineType.name} PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql.trim).mkString(", ")})" + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql.trim).mkString(", ")})" def node: ObjectNode = { val node = mapper.createObjectNode() @@ -271,7 +352,47 @@ package object schema { node } - def json: String = node.toString + override def ddl: String = + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.ddl.trim).mkString(", ")})" + + def json: String = mapper.writeValueAsString(node) + + def diff(pipeline: DdlPipeline): List[PipelineDiff] = { + + val actual = this.ddlProcessors + + val desired = pipeline.ddlProcessors + + // 1. Index processors by logical key + def key(p: DdlProcessor) = (p.processorType, p.column) + + val desiredMap = desired.map(p => key(p) -> p).toMap + val actualMap = actual.map(p => key(p) -> p).toMap + + val diffs = scala.collection.mutable.ListBuffer[PipelineDiff]() + + // 2. Added processors + for ((k, p) <- desiredMap if !actualMap.contains(k)) { + diffs += ProcessorAdded(p) + } + + // 3. Removed processors + for ((k, p) <- actualMap if !desiredMap.contains(k)) { + diffs += ProcessorRemoved(p) + } + + // 4. Modified processors + for ((k, from) <- actualMap if desiredMap.contains(k)) { + val to = desiredMap(k) + from.diff(to) match { + case Some(d) => diffs += ProcessorChanged(from, to, d) + case None => // identical → nothing + } + } + + // 5. Optional: detect reordering for processors where order matters + diffs.toList + } } private[schema] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { @@ -323,30 +444,60 @@ package object schema { comment: Option[String] = None, options: Map[String, Value[_]] = Map.empty, struct: Option[DdlColumn] = None - ) extends Token { + ) extends DdlToken { def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) - def level: Int = struct.map(_.level + 1).getOrElse(0) + private def level: Int = struct.map(_.level + 1).getOrElse(0) def update(struct: Option[DdlColumn] = None): DdlColumn = { val updated = this.copy(struct = struct) - updated.copy( - multiFields = multiFields.map { field => - field.update(Some(updated)) - }, - script = script.map { sc => + val updated_script = + script.map { sc => sc.copy( column = updated.path, source = sc.source.replace(s"ctx.$name", s"ctx.${updated.path}") ) } + val sql_script: Option[ObjectValue] = + updated_script match { + case Some(s) => + Some( + ObjectValue( + Map( + "sql" -> StringValue(s.script), + "column" -> StringValue(updated.path), + "painless" -> StringValue(s.source) + ) + ) + ) + case _ => None + } + updated.copy( + multiFields = multiFields.map { field => + field.update(Some(updated)) + }, + script = updated_script, + options = options ++ Map( + "meta" -> ObjectValue( + options.get("meta") match { + case Some(ObjectValue(value)) => + value ++ Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ sql_script.map(st => "script" -> st) + case _ => + Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ sql_script.map(st => "script" -> st) + } + ) + ) ) } def sql: String = { val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")})" + s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.sql}" }.mkString(", ")})" } else { "" } - val defaultOpt = defaultValue.map(v => s" DEFAULT ${v.ddl}").getOrElse("") + val defaultOpt = defaultValue.map(v => s" DEFAULT ${v.sql}").getOrElse("") val notNullOpt = if (notNull) " NOT NULL" else "" val commentOpt = comment.map(c => s" COMMENT '$c'").getOrElse("") val fieldsOpt = if (multiFields.nonEmpty) { @@ -362,7 +513,7 @@ package object schema { def ddlProcessors: Seq[DdlProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => DdlDefaultValueProcessor( - sql = s"$name DEFAULT $dv", + sql = s"$path DEFAULT $dv", column = path, value = dv ) @@ -387,36 +538,84 @@ package object schema { } root.set(name, fieldsNode) } - val sql_script = - script match { - case Some(s) => - Some( - ObjectValue( - Map( - "sql" -> StringValue(s.script), - "column" -> StringValue(path), - "painless" -> StringValue(s.source) - ) - ) - ) - case _ => None - } - val meta = options.get("meta") match { - case Some(ObjectValue(value)) => - ObjectValue( - value ++ Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => - "comment" -> StringValue(ct) - ) ++ sql_script.map(st => "script" -> st) - ) + updateNode(root, options) + root + } + + def diff(desired: DdlColumn, parent: Option[DdlColumn] = None): List[ColumnDiff] = { + val actual = this + val diffs = scala.collection.mutable.ListBuffer[ColumnDiff]() + + // 1. Type + if (SQLTypeUtils.elasticType(actual.dataType) != SQLTypeUtils.elasticType(desired.dataType)) + diffs += ColumnTypeChanged(path, actual.dataType, desired.dataType) + + // 2. Default + (actual.defaultValue, desired.defaultValue) match { + case (None, Some(v)) => diffs += ColumnDefaultSet(path, v) + case (Some(_), None) => diffs += ColumnDefaultRemoved(path) + case (Some(a), Some(b)) if a != b => + diffs += ColumnDefaultSet(path, b) case _ => - ObjectValue( - Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => - "comment" -> StringValue(ct) - ) ++ sql_script.map(st => "script" -> st) - ) } - updateNode(root, options ++ Map("meta" -> meta)) - root + + // 3. Script + (actual.script, desired.script) match { + case (None, Some(s)) => diffs += ColumnScriptSet(path, s) + case (Some(_), None) => diffs += ColumnScriptRemoved(path) + case (Some(a), Some(b)) if a.sql != b.sql => + diffs += ColumnScriptSet(path, b) + case _ => + } + + // 4. Comment + (actual.comment, desired.comment) match { + case (None, Some(c)) => diffs += ColumnCommentSet(path, c) + case (Some(_), None) => diffs += ColumnCommentRemoved(path) + case (Some(a), Some(b)) if a != b => + diffs += ColumnCommentSet(path, b) + case _ => + } + + // 5. Not Null + if (actual.notNull != desired.notNull) { + if (desired.notNull) diffs += ColumnNotNullSet(path) + else diffs += ColumnNotNullRemoved(path) + } + + // 6. Options + val allOptions = actual.options.keySet ++ desired.options.keySet + for (key <- allOptions) { + (actual.options.get(key), desired.options.get(key)) match { + case (None, Some(v)) => diffs += ColumnOptionSet(path, key, v) + case (Some(_), None) => diffs += ColumnOptionRemoved(path, key) + case (Some(a), Some(b)) if a != b => + diffs += ColumnOptionSet(path, key, b) + case _ => + } + } + + // 7. STRUCT / multi-fields + val actualFields = actual.multiFields.map(f => f.name -> f).toMap + val desiredFields = desired.multiFields.map(f => f.name -> f).toMap + + // 7.1. Fields added + for ((name, f) <- desiredFields if !actualFields.contains(name)) { + diffs += FieldAdded(path, f) + } + + // 7.2. Fields removed + for ((name, f) <- actualFields if !desiredFields.contains(name)) { + diffs += FieldRemoved(path, name) + } + + // 7.3. Fields modified + for ((name, a) <- actualFields if desiredFields.contains(name)) { + val d = desiredFields(name) + diffs ++= a.diff(d, Some(desired)) + } + + diffs.toList } } @@ -456,7 +655,8 @@ package object schema { primaryKey: List[String] = Nil, partitionBy: Option[DdlPartition] = None, mappings: Map[String, Value[_]] = Map.empty, - settings: Map[String, Value[_]] = Map.empty + settings: Map[String, Value[_]] = Map.empty, + processors: Seq[DdlProcessor] = Seq.empty ) extends Token { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap @@ -744,7 +944,7 @@ package object schema { } - override def validate(): Either[SQL, Unit] = { + override def validate(): Either[String, Unit] = { var errors = Seq[String]() // check that primary key columns exist primaryKey.foreach { pk => @@ -761,11 +961,15 @@ package object schema { if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) } - lazy val ddlPipeline: DdlPipeline = DdlPipeline( - name = s"${name}_ddl_default_pipeline", - ddlPipelineType = DdlPipelineType.Default, - ddlProcessors = ddlProcessors - ) + lazy val ddlPipeline: DdlPipeline = { + val processorsFromColumns = ddlProcessors.map(p => p.column -> p).toMap + DdlPipeline( + name = s"${name}_ddl_default_pipeline", + ddlPipelineType = DdlPipelineType.Default, + ddlProcessors = + ddlProcessors ++ processors.filterNot(p => processorsFromColumns.contains(p.column)) + ) + } lazy val defaultPipeline: String = settings @@ -827,6 +1031,73 @@ package object schema { lazy val pipeline: ObjectNode = { ddlPipeline.node } + + def diff(desired: DdlTable): DdlTableDiff = { + val actual = this.update() + val desiredUpdated = desired.update() + + val columnDiffs = scala.collection.mutable.ListBuffer[ColumnDiff]() + + // 1. Columns added + val actualCols = actual.cols + val desiredCols = desiredUpdated.cols + + for ((name, col) <- desiredCols if !actualCols.contains(name)) { + columnDiffs += ColumnAdded(col) + } + + // 2. Columns removed + for ((name, col) <- actualCols if !desiredCols.contains(name)) { + columnDiffs += ColumnRemoved(col.name) + } + + // 3. Columns modified + for ((name, a) <- actualCols if desiredCols.contains(name)) { + val d = desiredCols(name) + columnDiffs ++= a.diff(d) + } + + // 4. Mappings + val mappingDiffs = scala.collection.mutable.ListBuffer[MappingDiff]() + val allMappings = actual.mappings.keySet ++ desiredUpdated.mappings.keySet + for (key <- allMappings.filterNot(_ == "_meta") /*FIXME*/ ) { + (actual.mappings.get(key), desiredUpdated.mappings.get(key)) match { + case (None, Some(v)) => mappingDiffs += MappingSet(key, v) + case (Some(_), None) => mappingDiffs += MappingRemoved(key) + case (Some(a), Some(b)) if a != b => + mappingDiffs += MappingSet(key, b) + case _ => + } + } + + // 5. Settings + val settingDiffs = scala.collection.mutable.ListBuffer[SettingDiff]() + val allSettings = actual.settings.keySet ++ desiredUpdated.settings.keySet + for (key <- allSettings) { + (actual.settings.get(key), desiredUpdated.settings.get(key)) match { + case (None, Some(v)) => settingDiffs += SettingSet(key, v) + case (Some(_), None) => settingDiffs += SettingRemoved(key) + case (Some(a), Some(b)) if a != b => + settingDiffs += SettingSet(key, b) + case _ => + } + } + + // 6. Pipeline + val pipelineDiffs = scala.collection.mutable.ListBuffer[PipelineDiff]() + val actualPipeline = actual.ddlPipeline + val desiredPipeline = desiredUpdated.ddlPipeline + actualPipeline.diff(desiredPipeline).foreach { d => + pipelineDiffs += d + } + + DdlTableDiff( + columns = columnDiffs.toList, + mappings = mappingDiffs.toList, + settings = settingDiffs.toList, + pipeline = pipelineDiffs.toList + ) + } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 7f23cdfc..0dca230e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -93,6 +93,10 @@ object SQLTypeUtils { ).contains( out.typeId )) || + (Set(DateTime.typeId, Timestamp.typeId) + .contains(out.typeId) && Set(DateTime.typeId, Timestamp.typeId).contains( + in.typeId + )) || (out.typeId == Numeric.typeId && Set( TinyInt.typeId, SmallInt.typeId, @@ -116,7 +120,10 @@ object SQLTypeUtils { out.typeId )) || (out.isInstanceOf[SQLNumeric] && in.isInstanceOf[SQLNumeric]) || - (out.typeId == Varchar.typeId && in.typeId == Varchar.typeId) || + (Set(Varchar.typeId, Text.typeId, Keyword.typeId) + .contains(out.typeId) && Set(Varchar.typeId, Text.typeId, Keyword.typeId).contains( + in.typeId + )) || (out.typeId == Boolean.typeId && in.typeId == Boolean.typeId) || out.typeId == Any.typeId || in.typeId == Any.typeId || out.typeId == Null.typeId || in.typeId == Null.typeId diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 3f446ad1..734b32f7 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -981,9 +981,10 @@ class ParserSpec extends AnyFlatSpec with Matchers { ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) val sql = ct.ddlTable.sql println(sql) + println(ct.ddlTable.ddlPipeline.ddl) val json = ct.ddlTable.ddlPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings println(indexMappings) indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" @@ -992,7 +993,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { indexSettings.toString shouldBe """{"index":{}}""" val pipeline = ct.ddlTable.pipeline println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE DEFAULT PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) @@ -1009,6 +1010,11 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") println(s"""esIndex pipeline -> ${ddlTable.pipeline.toString}""") + val ddlTableDiff = ddlTable.diff(ct.ddlTable) + ddlTableDiff.columns.isEmpty shouldBe true + ddlTableDiff.mappings.isEmpty shouldBe true + ddlTableDiff.settings.isEmpty shouldBe true + ddlTableDiff.pipeline.isEmpty shouldBe true case _ => fail("Expected CreateTable") } } From 05950c84831470cc8e3777f65fe239ea35d4f3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 17 Dec 2025 16:31:45 +0100 Subject: [PATCH 17/95] fix _meta for mappings --- .../elastic/sql/query/package.scala | 17 +++-- .../elastic/sql/schema/DdlTableDiff.scala | 15 +++-- .../elastic/sql/schema/package.scala | 66 ++++++++++--------- 3 files changed, 58 insertions(+), 40 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 026d6ceb..fb1f0f45 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.sql.schema.{ DdlPipeline, DdlPipelineType, DdlProcessor, + DdlProcessorType, DdlRemoveProcessor, DdlRenameProcessor, DdlScriptProcessor, @@ -585,15 +586,19 @@ package object query { override def sql: String = s"DROP SETTING $optionKey" } - case class AddTableProcessor(processor: DdlProcessor) extends AlterTableStatement { - override def sql: String = s"ADD PROCESSOR ${processor.sql}" + + sealed trait AlterPipelineStatement extends AlterTableStatement + + case class AddPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + override def sql: String = s"ADD PROCESSOR ${processor.ddl}" override def ddlProcessor: Option[DdlProcessor] = Some(processor) } - case class DropTableProcessor(column: String) extends AlterTableStatement { - override def sql: String = s"DROP PROCESSOR $column" + case class DropPipelineProcessor(processorType: DdlProcessorType, column: String) + extends AlterPipelineStatement { + override def sql: String = s"DROP PROCESSOR ${processorType.name.toUpperCase}($column)" } - case class AlterTableProcessor(processor: DdlProcessor) extends AlterTableStatement { - override def sql: String = s"ALTER PROCESSOR ${processor.sql}" + case class AlterPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + override def sql: String = s"ALTER PROCESSOR ${processor.ddl}" override def ddlProcessor: Option[DdlProcessor] = Some(processor) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala index 0e150d93..138016c4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala @@ -82,10 +82,15 @@ case class SettingRemoved(key: String) extends SettingDiff { override def stmt: AlterTableStatement = DropTableSetting(key) } -sealed trait PipelineDiff +sealed trait PipelineDiff extends AlterTableStatementDiff -case class ProcessorAdded(processor: DdlProcessor) extends PipelineDiff -case class ProcessorRemoved(processor: DdlProcessor) extends PipelineDiff +case class ProcessorAdded(processor: DdlProcessor) extends PipelineDiff { + override def stmt: AlterTableStatement = AddPipelineProcessor(processor) +} +case class ProcessorRemoved(processor: DdlProcessor) extends PipelineDiff { + override def stmt: AlterTableStatement = + DropPipelineProcessor(processor.processorType, processor.column) +} case class ProcessorTypeChanged( actual: DdlProcessorType, desired: DdlProcessorType @@ -102,7 +107,9 @@ case class ProcessorChanged( from: DdlProcessor, to: DdlProcessor, diff: ProcessorDiff -) extends PipelineDiff +) extends PipelineDiff { + override def stmt: AlterTableStatement = AlterPipelineProcessor(to) +} case class DdlTableDiff( columns: List[ColumnDiff], diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 19be7814..e208c087 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -154,7 +154,7 @@ package object schema { Some(ProcessorDiff(typeChanged, propertyDiffs)) } - override def ddl: String = s"$column ${processorType.name.toUpperCase} ${Value(properties).ddl}" + override def ddl: String = s"${processorType.name.toUpperCase}${Value(properties).ddl}" } case class GenericProcessor(processorType: DdlProcessorType, properties: Map[String, Any]) @@ -493,7 +493,7 @@ package object schema { } def sql: String = { val opts = if (options.nonEmpty) { - s" OPTIONS (${options.map { case (k, v) => s"$k = ${v.sql}" }.mkString(", ")})" + s" OPTIONS ${ObjectValue(options).ddl}" } else { "" } @@ -619,7 +619,7 @@ package object schema { } } - case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends Token { + case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends DdlToken { def sql: String = s" PARTITION BY $column ($granularity)" val dateRounding: String = granularity.script.get @@ -657,27 +657,51 @@ package object schema { mappings: Map[String, Value[_]] = Map.empty, settings: Map[String, Value[_]] = Map.empty, processors: Seq[DdlProcessor] = Seq.empty - ) extends Token { + ) extends DdlToken { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap - def update(): DdlTable = this.copy(columns = columns.map(_.update())) + private lazy val _meta: Map[String, Value[_]] = Map.empty ++ { + if (primaryKey.nonEmpty) + Option("primary_key" -> StringValues(primaryKey.map(StringValue))) + else + None + }.toMap ++ partitionBy + .map(pb => + Map( + "partition_by" -> ObjectValue( + Map( + "column" -> StringValue(pb.column), + "granularity" -> StringValue(pb.granularity.script.get) + ) + ) + ) + ) + .getOrElse(Map.empty) + + def update(): DdlTable = this.copy( + columns = columns.map(_.update()), + mappings = mappings ++ Map( + "_meta" -> + ObjectValue(mappings.get("_meta") match { + case Some(ObjectValue(value)) => + value ++ _meta + case _ => _meta + }) + ) + ) def sql: String = { val opts = if (mappings.nonEmpty || settings.nonEmpty) { val mappingOpts = if (mappings.nonEmpty) { - s"mappings = (${mappings - .map { case (k, v) => - s"$k = ${v.ddl}" - } - .mkString(", ")})" + s"mappings = ${ObjectValue(mappings).ddl}" } else { "" } val settingsOpts = if (settings.nonEmpty) { - s"settings = (${mappings.map { case (k, v) => s"$k = ${v.ddl}" }.mkString(", ")})" + s"settings = ${ObjectValue(settings).ddl}" } else { "" } @@ -999,24 +1023,6 @@ package object schema { } node.set("properties", fields) updateNode(node, mappings) - if (primaryKey.nonEmpty || partitionBy.nonEmpty) { - val meta = Option(node.get("_meta")).getOrElse(mapper.createObjectNode()) - if (meta != null && meta.isObject) { - val metaObj = meta.asInstanceOf[ObjectNode] - if (primaryKey.nonEmpty) { - val pkArray = mapper.createArrayNode() - primaryKey.foreach(pk => pkArray.add(pk)) - metaObj.replace("primary_key", pkArray) - } - partitionBy.foreach { partition => - val partitionObj = mapper.createObjectNode() - partitionObj.put("column", partition.column) - partitionObj.put("granularity", partition.granularity.script.get) - metaObj.replace("partition_by", partitionObj) - } - node.replace("_meta", metaObj) - } - } node } @@ -1060,7 +1066,7 @@ package object schema { // 4. Mappings val mappingDiffs = scala.collection.mutable.ListBuffer[MappingDiff]() val allMappings = actual.mappings.keySet ++ desiredUpdated.mappings.keySet - for (key <- allMappings.filterNot(_ == "_meta") /*FIXME*/ ) { + for (key <- allMappings) { (actual.mappings.get(key), desiredUpdated.mappings.get(key)) match { case (None, Some(v)) => mappingDiffs += MappingSet(key, v) case (Some(_), None) => mappingDiffs += MappingRemoved(key) From d88f660c4c3cbf6caeb024645c14748ba8696099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 10:55:37 +0100 Subject: [PATCH 18/95] add support for create, alter and drop ddl queries for pipelines --- .../main/resources/softnetwork-elastic.conf | 2 - .../elastic/client/ElasticConfig.scala | 4 +- .../elastic/client/ElasticConfig.scala | 3 +- .../elastic/client/NopeClientApi.scala | 16 ++ sql/build.sbt | 9 +- sql/src/main/resources/softnetwork-sql.conf | 3 + .../elastic/sql/config/ElasticSqlConfig.scala | 24 ++ .../elastic/sql/config/ElasticSqlConfig.scala | 40 ++++ .../softnetwork/elastic/schema/package.scala | 113 ++------- .../app/softnetwork/elastic/sql/package.scala | 16 +- .../elastic/sql/parser/DmlParser.scala | 16 ++ .../elastic/sql/parser/Parser.scala | 85 ++++++- .../elastic/sql/query/package.scala | 49 +++- .../elastic/sql/schema/DdlTableDiff.scala | 43 +++- .../elastic/sql/schema/package.scala | 112 ++++++++- .../elastic/sql/serialization/package.scala | 16 ++ .../elastic/sql/parser/ParserSpec.scala | 215 +++++++++++++++++- 17 files changed, 641 insertions(+), 125 deletions(-) create mode 100644 sql/src/main/resources/softnetwork-sql.conf create mode 100644 sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala create mode 100644 sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala diff --git a/core/src/main/resources/softnetwork-elastic.conf b/core/src/main/resources/softnetwork-elastic.conf index 59b3da13..4b59eec4 100644 --- a/core/src/main/resources/softnetwork-elastic.conf +++ b/core/src/main/resources/softnetwork-elastic.conf @@ -40,6 +40,4 @@ elastic { } } - # Custom settings - composite-key-separator = "|" } \ No newline at end of file diff --git a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala index b49fede5..bcc562b6 100644 --- a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala @@ -44,9 +44,7 @@ case class ElasticConfig( discovery: DiscoveryConfig, connectionTimeout: Duration, socketTimeout: Duration, - metrics: MetricsConfig, - compositeKeySeparator: String = "|" -) + metrics: MetricsConfig) object ElasticConfig extends StrictLogging { def apply(config: Config): ElasticConfig = { diff --git a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala index e0fcd4fd..ddc77f82 100644 --- a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala @@ -44,8 +44,7 @@ case class ElasticConfig( discovery: DiscoveryConfig, connectionTimeout: Duration, socketTimeout: Duration, - metrics: MetricsConfig, - compositeKeySeparator: String + metrics: MetricsConfig ) object ElasticConfig extends StrictLogging { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index 536ec097..0252384e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client import akka.NotUsed diff --git a/sql/build.sbt b/sql/build.sbt index eb0e89c6..cc74c484 100644 --- a/sql/build.sbt +++ b/sql/build.sbt @@ -7,18 +7,23 @@ elasticSearchVersion := Versions.es9 name := s"softclient4es-sql" +val typesafeConfig = Seq( + "com.typesafe" % "config" % Versions.typesafeConfig, + "com.github.kxbmap" %% "configs" % Versions.kxbmap +) + val scalatest = Seq( "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) libraryDependencies ++= jacksonDependencies(elasticSearchVersion.value) ++ // elastic4sDependencies(elasticSearchVersion.value) ++ + typesafeConfig ++ scalatest ++ Seq( "javax.activation" % "activation" % "1.1.1" % Test ) :+ + "com.typesafe.scala-logging" %% "scala-logging" % Versions.scalaLogging :+ // ("app.softnetwork.persistence" %% "persistence-core" % Versions.genericPersistence excludeAll(jacksonExclusions: _*)) :+ "org.scala-lang" % "scala-reflect" % scalaVersion.value :+ "com.google.code.gson" % "gson" % Versions.gson % Test - - diff --git a/sql/src/main/resources/softnetwork-sql.conf b/sql/src/main/resources/softnetwork-sql.conf new file mode 100644 index 00000000..dfb62c17 --- /dev/null +++ b/sql/src/main/resources/softnetwork-sql.conf @@ -0,0 +1,3 @@ +sql { + composite-key-separator = "\\|\\|" # regex separator for composite keys in SQL ddl queries +} diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala new file mode 100644 index 00000000..fbcbefeb --- /dev/null +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -0,0 +1,24 @@ +package app.softnetwork.elastic.sql.config + +import com.typesafe.config.Config +import com.typesafe.scalalogging.StrictLogging +import configs.Configs + +case class ElasticSqlConfig( + compositeKeySeparator: String +) + +object ElasticSqlConfig extends StrictLogging { + def apply(config: Config): ElasticSqlConfig = { + Configs[ElasticSqlConfig] + .get(config.withFallback(ConfigFactory.load("softnetwork-sql.conf"), "sql") + .toEither match { + case Left(configError) => + logger.error(s"Something went wrong with the provided arguments $configError") + throw configError.configException + case Right(r) => r + } + } + + def apply(): ElasticSqlConfig = apply(com.typesafe.config.ConfigFactory.load()) +} diff --git a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala new file mode 100644 index 00000000..d0c9c75e --- /dev/null +++ b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.config + +import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.scalalogging.StrictLogging +import configs.ConfigReader + +case class ElasticSqlConfig( + compositeKeySeparator: String +) + +object ElasticSqlConfig extends StrictLogging { + def apply(config: Config): ElasticSqlConfig = { + ConfigReader[ElasticSqlConfig] + .read(config.withFallback(ConfigFactory.load("softnetwork-sql.conf")), "sql") + .toEither match { + case Left(configError) => + logger.error(s"Something went wrong with the provided arguments $configError") + throw configError.configException + case Right(r) => r + } + } + + def apply(): ElasticSqlConfig = apply(com.typesafe.config.ConfigFactory.load()) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 05c0fbbe..d0019101 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -1,19 +1,32 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ - mapper, DdlColumn, DdlDateIndexNameProcessor, DdlDefaultValueProcessor, DdlPartition, DdlPrimaryKeyProcessor, DdlProcessor, - DdlProcessorType, DdlScriptProcessor, - DdlTable, - GenericProcessor + DdlTable } import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode @@ -226,97 +239,7 @@ package object schema { final case class EsProcessor( processor: JsonNode ) { - private val ScriptDescRegex = - """^\s*([a-zA-Z0-9_]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r - - lazy val ddlProcesor: DdlProcessor = { - val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. - val props = processor.get(processorType) - - processorType match { - case "set" => - val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") - val valueNode = props.get("value") - val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) - - if (field == "_id" && desc.startsWith("PRIMARY KEY")) { - // DdlPrimaryKeyProcessor - // description: "PRIMARY KEY (id)" - val inside = desc.stripPrefix("PRIMARY KEY").trim.stripPrefix("(").stripSuffix(")") - val cols = - inside.split(",").map(_.replaceAll("\"", "")).map(_.trim).filter(_.nonEmpty).toSet - DdlPrimaryKeyProcessor( - sql = desc, - column = "_id", - value = cols, - ignoreFailure = ignoreFailure - ) - } else if (desc.startsWith(s"$field DEFAULT")) { - DdlDefaultValueProcessor( - sql = desc, - column = field, - value = Value(valueNode.asText()), - ignoreFailure = ignoreFailure - ) - } else { - GenericProcessor( - processorType = DdlProcessorType.Set, - properties = - mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap - ) - } - - case "script" => - val desc = props.get("description").asText() - val lang = props.get("lang").asText() - require(lang == "painless", s"Only painless supported, got $lang") - val source = props.get("source").asText() - val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) - - desc match { - case ScriptDescRegex(col, dataType, script) => - DdlScriptProcessor( - script = script, - column = col, - dataType = SQLTypes(dataType), - source = source, - ignoreFailure = ignoreFailure - ) - case _ => - GenericProcessor( - processorType = DdlProcessorType.Script, - properties = - mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap - ) - } - - case "date_index_name" => - val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") - val rounding = props.get("date_rounding").asText() - val formats = Option(props.get("date_formats")) - .map(_.elements().asScala.toList.map(_.asText())) - .getOrElse(Nil) - val prefix = props.get("index_name_prefix").asText() - - DdlDateIndexNameProcessor( - sql = desc, - column = field, - dateRounding = rounding, - dateFormats = formats, - prefix = prefix - ) - - case other => - GenericProcessor( - processorType = DdlProcessorType(other), - properties = - mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap - ) - - } - } + lazy val ddlProcesor: DdlProcessor = DdlProcessor(processor) } final case class EsPipeline( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index d6748926..8eb4e70e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -390,7 +390,12 @@ package object sql { .mkString("(", ", ", ")") override def baseType: SQLType = SQLTypes.Struct override def ddl: String = value - .map { case (k, v) => s"""$k = ${v.ddl}""" } + .map { case (k, v) => + v match { + case IdValue | IngestTimestampValue => s"""$k = "${v.ddl}"""" + case _ => s"""$k = ${v.ddl}""" + } + } .mkString("(", ", ", ")") } @@ -844,9 +849,12 @@ package object sql { s"(doc['$path'].size() == 0 ? $nullValue : doc['$path'].value${painlessMethods.mkString("")})" lazy val processParamName: String = { - if (path.nonEmpty) - s"ctx.$path" - else "" + if (path.nonEmpty) { + if (path.contains(".")) + s"ctx.${path.split("\\.").mkString("?.")}" + else + s"ctx.$path" + } else "" } lazy val processCheckNotNull: Option[String] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala index 18363953..4b0ac6ad 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.query.{Delete, Insert, Update} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index d80a88f0..5daacfba 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -30,7 +30,14 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.{DdlColumn, DdlPartition, DdlScriptProcessor} +import app.softnetwork.elastic.sql.schema.{ + DdlColumn, + DdlPartition, + DdlPipelineType, + DdlProcessor, + DdlProcessorType, + DdlScriptProcessor +} import app.softnetwork.elastic.sql.time.TimeUnit import scala.language.implicitConversions @@ -84,6 +91,72 @@ object Parser opts.toMap } + def processorType: PackratParser[DdlProcessorType] = + ident ^^ { name => + name.toLowerCase match { + case "set" => DdlProcessorType.Set + case "script" => DdlProcessorType.Script + case "rename" => DdlProcessorType.Rename + case "remove" => DdlProcessorType.Remove + case "date_index_name" => DdlProcessorType.DateIndexName + case other => DdlProcessorType(other) + } + } + + def processor: PackratParser[DdlProcessor] = + processorType ~ objectValue ^^ { case pt ~ opts => + DdlProcessor(pt, opts) + } + + def createOrReplacePipeline: PackratParser[CreatePipeline] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "PIPELINE") ~ ident ~ ("WITH" ~ "PROCESSORS") ~ start ~ repsep( + processor, + separator + ) ~ end ^^ { case _ ~ name ~ _ ~ _ ~ proc ~ _ => + CreatePipeline(name, DdlPipelineType.Custom, orReplace = true, processors = proc) + } + + def createPipeline: PackratParser[CreatePipeline] = + ("CREATE" ~ "PIPELINE") ~ ifNotExists ~ ident ~ ("WITH" ~ "PROCESSORS" ~ start) ~ repsep( + processor, + separator + ) <~ end ^^ { case _ ~ ine ~ name ~ _ ~ proc => + CreatePipeline(name, DdlPipelineType.Custom, ifNotExists = ine, processors = proc) + } + + def dropPipeline: PackratParser[DropPipeline] = + ("DROP" ~ "PIPELINE") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropPipeline(name, ifExists = ie) + } + + def addProcessor: PackratParser[AddPipelineProcessor] = + ("ADD" ~ "PROCESSOR") ~ processor ^^ { case _ ~ proc => + AddPipelineProcessor(proc) + } + + def dropProcessor: PackratParser[DropPipelineProcessor] = + ("DROP" ~ "PROCESSOR") ~ processorType ~ start ~ ident ~ end ^^ { case _ ~ pt ~ _ ~ name ~ _ => + DropPipelineProcessor(pt, name) + } + + def alterPipelineStatement: PackratParser[AlterPipelineStatement] = + addProcessor | dropProcessor + + def alterPipeline: PackratParser[AlterPipeline] = + ("ALTER" ~ "PIPELINE") ~ ifExists ~ ident ~ start.? ~ repsep( + alterPipelineStatement, + separator + ) ~ end.? ^^ { case _ ~ ie ~ pipeline ~ s ~ stmts ~ e => + if (s.isDefined && e.isEmpty) { + throw new Exception("Mismatched closing parentheses in ALTER PIPELINE statement") + } else if (s.isEmpty && e.isDefined) { + throw new Exception("Mismatched opening parentheses in ALTER PIPELINE statement") + } else if (s.isEmpty && e.isEmpty && stmts.size > 1) { + throw new Exception("Multiple ALTER PIPELINE statements require parentheses") + } else + AlterPipeline(pipeline, ie, stmts) + } + def multiFields: PackratParser[List[DdlColumn]] = "FIELDS" ~ start ~> repsep(column, separator) <~ end ^^ (cols => cols) | success(Nil) @@ -426,7 +499,15 @@ object Parser } def ddlStatement: PackratParser[DdlStatement] = - createTable | createOrReplaceTable | alterTable | dropTable | truncateTable + createTable | + createPipeline | + createOrReplaceTable | + createOrReplacePipeline | + alterTable | + alterPipeline | + dropTable | + truncateTable | + dropPipeline /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ def insert: PackratParser[Insert] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index fb1f0f45..54c4a461 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -313,6 +313,53 @@ package object query { sealed trait DdlStatement extends Statement + case class CreatePipeline( + name: String, + pipelineType: DdlPipelineType, + ifNotExists: Boolean = false, + orReplace: Boolean = false, + processors: Seq[DdlProcessor] + ) extends DdlStatement { + override def sql: String = { + val processorsDdl = processors.map(_.ddl).mkString(", ") + val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" + s"CREATE$replaceClause PIPELINE$ineClause $name WITH PROCESSORS ($processorsDdl)" + } + + lazy val ddlPipeline: DdlPipeline = + DdlPipeline(name, pipelineType, processors) + } + + case class AlterPipeline( + name: String, + ifExists: Boolean, + statements: List[AlterPipelineStatement] + ) extends DdlStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS " else "" + val parenthesesNeeded = statements.size > 1 + val statementsSql = if (parenthesesNeeded) { + statements.map(_.sql).mkString("(\n\t", ",\n\t", "\n)") + } else { + statements.map(_.sql).mkString("") + } + s"ALTER PIPELINE $name$ifExistsClause $statementsSql" + } + + lazy val ddlProcessors: Seq[DdlProcessor] = statements.flatMap(_.ddlProcessor) + + lazy val pipeline: DdlPipeline = + DdlPipeline(s"alter-pipeline-$name-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) + } + + case class DropPipeline(name: String, ifExists: Boolean = false) extends DdlStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"DROP PIPELINE $ifExistsClause$name" + } + } + case class CreateTable( table: String, ddl: Either[DqlStatement, List[DdlColumn]], @@ -324,8 +371,8 @@ package object query { ) extends DdlStatement { override def sql: String = { - val ineClause = if (ifNotExists) " IF NOT EXISTS" else "" val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" ddl match { case Left(select) => s"CREATE$replaceClause TABLE$ineClause $table AS ${select.sql}" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala index 138016c4..4419aeca 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.sql.schema import app.softnetwork.elastic.sql.Value @@ -82,13 +98,17 @@ case class SettingRemoved(key: String) extends SettingDiff { override def stmt: AlterTableStatement = DropTableSetting(key) } -sealed trait PipelineDiff extends AlterTableStatementDiff +sealed trait AlterPipelineStatementDiff { + def stmt: AlterPipelineStatement +} + +sealed trait PipelineDiff extends AlterPipelineStatementDiff case class ProcessorAdded(processor: DdlProcessor) extends PipelineDiff { - override def stmt: AlterTableStatement = AddPipelineProcessor(processor) + override def stmt: AlterPipelineStatement = AddPipelineProcessor(processor) } case class ProcessorRemoved(processor: DdlProcessor) extends PipelineDiff { - override def stmt: AlterTableStatement = + override def stmt: AlterPipelineStatement = DropPipelineProcessor(processor.processorType, processor.column) } case class ProcessorTypeChanged( @@ -108,7 +128,7 @@ case class ProcessorChanged( to: DdlProcessor, diff: ProcessorDiff ) extends PipelineDiff { - override def stmt: AlterTableStatement = AlterPipelineProcessor(to) + override def stmt: AlterPipelineStatement = AlterPipelineProcessor(to) } case class DdlTableDiff( @@ -136,4 +156,19 @@ case class DdlTableDiff( ) } } + + def alterPipeline(name: String, ifExists: Boolean): Option[AlterPipeline] = { + val pipelineStatements = pipeline.map(_.stmt) + if (pipelineStatements.isEmpty) { + None + } else { + Some( + AlterPipeline( + name, + ifExists, + pipelineStatements + ) + ) + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index e208c087..625321e2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -16,19 +16,23 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization.JacksonConfig import app.softnetwork.elastic.sql.time.TimeUnit -import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.fasterxml.jackson.databind.node.ObjectNode import java.util.UUID +import scala.jdk.CollectionConverters._ import scala.language.implicitConversions package object schema { val mapper: ObjectMapper = JacksonConfig.objectMapper + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + sealed trait DdlProcessorType { def name: String } @@ -157,6 +161,106 @@ package object schema { override def ddl: String = s"${processorType.name.toUpperCase}${Value(properties).ddl}" } + object DdlProcessor { + private val ScriptDescRegex = + """^\s*([a-zA-Z0-9_\\.]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r + + def apply(processorType: DdlProcessorType, properties: ObjectValue): DdlProcessor = { + val node = mapper.createObjectNode() + val props = mapper.createObjectNode() + updateNode(props, properties.value) + node.set(processorType.name, props) + apply(node) + } + + def apply(processor: JsonNode): DdlProcessor = { + val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. + val props = processor.get(processorType) + + processorType match { + case "set" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val valueNode = props.get("value") + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + if (field == "_id") { + val value = + valueNode + .asText() + .trim + .stripPrefix("{{") + .stripSuffix("}}") + .trim + // DdlPrimaryKeyProcessor + val cols = value.split(sqlConfig.compositeKeySeparator).toSet + DdlPrimaryKeyProcessor( + sql = desc, + column = "_id", + value = cols, + ignoreFailure = ignoreFailure + ) + } else { + DdlDefaultValueProcessor( + sql = desc, + column = field, + value = Value(valueNode.asText()), + ignoreFailure = ignoreFailure + ) + } + + case "script" => + val desc = props.get("description").asText() + val lang = props.get("lang").asText() + require(lang == "painless", s"Only painless supported, got $lang") + val source = props.get("source").asText() + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + desc match { + case ScriptDescRegex(col, dataType, script) => + DdlScriptProcessor( + script = script, + column = col, + dataType = SQLTypes(dataType), + source = source, + ignoreFailure = ignoreFailure + ) + case _ => + GenericProcessor( + processorType = DdlProcessorType.Script, + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) + } + + case "date_index_name" => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val rounding = props.get("date_rounding").asText() + val formats = Option(props.get("date_formats")) + .map(_.elements().asScala.toList.map(_.asText())) + .getOrElse(Nil) + val prefix = props.get("index_name_prefix").asText() + + DdlDateIndexNameProcessor( + sql = desc, + column = field, + dateRounding = rounding, + dateFormats = formats, + prefix = prefix + ) + + case other => + GenericProcessor( + processorType = DdlProcessorType(other), + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) + + } + } + } + case class GenericProcessor(processorType: DdlProcessorType, properties: Map[String, Any]) extends DdlProcessor { override def sql: String = ddl @@ -235,7 +339,7 @@ package object schema { value: Set[String], ignoreFailure: Boolean = false, ignoreEmptyValue: Boolean = false, - separator: String = "|" + separator: String = "\\|\\|" ) extends DdlProcessor { def processorType: DdlProcessorType = DdlProcessorType.Set @@ -259,7 +363,7 @@ package object schema { def _if: String = { if (column.contains(".")) - s"""ctx.${column.split(".").mkString("?.")} == null""" + s"""ctx.${column.split("\\.").mkString("?.")} == null""" else s"""ctx.$column == null""" } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 1ab8a9dc..527f9fec 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.sql import com.fasterxml.jackson.annotation.JsonInclude diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 734b32f7..29aac8bb 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,14 +1,24 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.schema.EsIndex -import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.{IngestTimestampValue, ObjectValue, StringValue, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.schema.{mapper, DdlPartition} +import app.softnetwork.elastic.sql.schema.{ + mapper, + DdlDateIndexNameProcessor, + DdlDefaultValueProcessor, + DdlPartition, + DdlPrimaryKeyProcessor, + DdlProcessorType, + DdlScriptProcessor +} import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import scala.collection.Set + object Queries { val numericalEq = "SELECT t.col1, t.col2 FROM Table AS t WHERE t.identifier = 1.0" val numericalLt = "SELECT * FROM Table WHERE identifier < 1" @@ -984,16 +994,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(ct.ddlTable.ddlPipeline.ddl) val json = ct.ddlTable.ddlPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" val indexSettings = ct.ddlTable.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = ct.ddlTable.pipeline println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) @@ -1003,13 +1013,14 @@ class ParserSpec extends AnyFlatSpec with Matchers { name = "users", mappings = mappings, settings = settings, - pipeline = None + pipeline = Some(pipeline) ) val ddlTable = esIndex.ddlTable println(s"""esIndex ddl -> ${ddlTable.sql}""") println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") println(s"""esIndex pipeline -> ${ddlTable.pipeline.toString}""") + println(s"""esIndex ddl pipeline -> ${ddlTable.ddlPipeline.ddl}""") val ddlTableDiff = ddlTable.diff(ct.ddlTable) ddlTableDiff.columns.isEmpty shouldBe true ddlTableDiff.mappings.isEmpty shouldBe true @@ -1271,6 +1282,198 @@ class ParserSpec extends AnyFlatSpec with Matchers { } } + it should "parse CREATE OR REPLACE PIPELINE" in { + val sql = + """CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ), + | SCRIPT ( + | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + | lang = "painless", + | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null = ChronoUnit.YEARS.between(param1, param2)", + | ignore_failure = true + | ), + | SET ( + | field = "ingested_at", + | if = "ctx.ingested_at == null", + | description = "DEFAULT _ingest.timestamp", + | ignore_failure = true, + | value = "_ingest.timestamp" + | ), + | SCRIPT ( + | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", + | lang = "painless", + | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)", + | ignore_failure = true + | ), + | DATE_INDEX_NAME ( + | field = "birthdate", + | index_name_prefix = "users-", + | date_formats = ["yyyy-MM"], + | ignore_failure = true, + | date_rounding = "M", + | description = "PARTITION BY birthdate (MONTH)", + | separator = "-" + | ), + | SET ( + | field = "_id", + | description = "PRIMARY KEY (id)", + | ignore_failure = false, + | ignore_empty_value = false, + | value = "{{id}}" + | ) + |)""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case CreatePipeline( + "user_pipeline", + _, + false, + true, + processors + ) => + processors.size shouldBe 6 + processors.find(_.column == "name") match { + case Some( + DdlDefaultValueProcessor( + "DEFAULT 'anonymous'", + "name", + StringValue("anonymous"), + true + ) + ) => + case other => fail(s"Expected DdlDefaultValueProcessor for name, got $other") + } + processors.find(_.column == "age") match { + case Some( + DdlScriptProcessor( + "DATE_DIFF(birthdate, CURRENT_DATE, YEAR)", + "age", + SQLTypes.Int, + source, + true + ) + ) => + source should include( + "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null = ChronoUnit.YEARS.between(param1, param2)" + ) + case other => fail(s"Expected DdlScriptProcessor for age, got $other") + } + processors.find(_.column == "ingested_at") match { + case Some( + DdlDefaultValueProcessor( + "DEFAULT _ingest.timestamp", + "ingested_at", + IngestTimestampValue, + true + ) + ) => + case other => fail(s"Expected DdlDefaultValueProcessor for ingested_at, got $other") + } + processors.find(_.column == "profile.seniority") match { + case Some( + DdlScriptProcessor( + "DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)", + "profile.seniority", + SQLTypes.Int, + source, + true + ) + ) => + source should include( + "def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)" + ) + case other => fail(s"Expected DdlScriptProcessor for profile.seniority, got $other") + } + processors.find(_.column == "birthdate") match { + case Some( + DdlDateIndexNameProcessor( + "PARTITION BY birthdate (MONTH)", + "birthdate", + "M", + List("yyyy-MM"), + "users-", + "-", + true + ) + ) => + case other => fail(s"Expected DdlDateIndexNameProcessor for birthdate, got $other") + } + processors.find(_.column == "_id") match { + case Some( + DdlPrimaryKeyProcessor( + "PRIMARY KEY (id)", + "_id", + cols, + false, + false, + "\\|\\|" + ) + ) => + cols should contain("id") + case other => fail(s"Expected DdlPrimaryKeyProcessor for _id, got $other") + } + + case _ => fail("Expected CreatePipeline") + } + } + + it should "parse ALTER PIPELINE IF EXISTS" in { + val sql = + """ALTER PIPELINE IF EXISTS user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | if = "ctx.status == null", + | description = "status DEFAULT 'active'", + | ignore_failure = true, + | value = "active" + | ), + | DROP PROCESSOR SET (_id) + |)""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterPipeline( + "user_pipeline", + ie, + statements + ) if ie => + statements.size shouldBe 2 + statements.collect { case AddPipelineProcessor(p) => p } match { + case DdlDefaultValueProcessor( + "status DEFAULT 'active'", + "status", + StringValue("active"), + true + ) :: Nil => + case other => fail(s"Expected AddPipelineProcessor with DdlSetProcessor, got $other") + } + statements.collect { case DropPipelineProcessor(DdlProcessorType.Set, f) => + f + } should contain("_id") + case _ => fail("Expected AlterPipeline") + } + } + + it should "parse DROP PIPELINE if exists" in { + val sql = "DROP PIPELINE IF EXISTS user_pipeline" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case DropPipeline("user_pipeline", ie) if ie => + case _ => fail("Expected DropPipeline") + } + } + // --- DML --- it should "parse INSERT INTO ... VALUES" in { From c1d045740d3152933a81e768a946107f22dff737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 11:51:22 +0100 Subject: [PATCH 19/95] update ddl table merge with field statements --- .../elastic/sql/query/package.scala | 2 +- .../elastic/sql/schema/package.scala | 109 +++++++++++------- .../elastic/sql/parser/ParserSpec.scala | 2 +- 3 files changed, 72 insertions(+), 41 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 54c4a461..53d4e2a0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -602,7 +602,7 @@ package object query { ) extends AlterTableStatement { override def sql: String = { val ifExistsClause = if (ifExists) " IF EXISTS" else "" - s"ALTER COLUMN$ifExistsClause $columnName ADD FIELD $field" + s"ALTER COLUMN$ifExistsClause $columnName SET FIELD $field" } } case class DropColumnField( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 625321e2..b769012d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -551,6 +551,24 @@ package object schema { ) extends DdlToken { def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) private def level: Int = struct.map(_.level + 1).getOrElse(0) + + private def cols: Map[String, DdlColumn] = multiFields.map(field => field.name -> field).toMap + + /* Recursive find */ + def find(path: String): Option[DdlColumn] = { + if (path.contains(".")) { + val parts = path.split("\\.") + cols.get(parts.head).flatMap { col => + col.multiFields + .to(LazyList) + .flatMap(_.find(parts.tail.mkString("."))) + .headOption + } + } else { + cols.get(path) + } + } + def update(struct: Option[DdlColumn] = None): DdlColumn = { val updated = this.copy(struct = struct) val updated_script = @@ -764,6 +782,20 @@ package object schema { ) extends DdlToken { private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap + def find(path: String): Option[DdlColumn] = { + if (path.contains(".")) { + val parts = path.split("\\.") + cols.get(parts.head).flatMap { col => + col.multiFields + .to(LazyList) + .flatMap(_.find(parts.tail.mkString("."))) + .headOption + } + } else { + cols.get(path) + } + } + private lazy val _meta: Map[String, Value[_]] = Map.empty ++ { if (primaryKey.nonEmpty) Option("primary_key" -> StringValues(primaryKey.map(StringValue))) @@ -983,55 +1015,54 @@ package object schema { ) else throw DdlColumnNotFound(columnName, table.name) case AlterColumnFields(columnName, newFields, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(multiFields = newFields.toList) - else col - } - ) - else throw DdlColumnNotFound(columnName, table.name) + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + c.copy( + multiFields = newFields.toList + ) + table + case _ => throw DdlColumnNotFound(columnName, table.name) + } + } case AlterColumnField( columnName, field, ifExists ) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) { - val updatedFields = if (col.multiFields.exists(_.name == field.name)) { - col.multiFields.map { f => - if (f.name == field.name) field else f - } - } else { - col.multiFields :+ field - } - col.copy(multiFields = updatedFields) - } else col - } - ) - else throw DdlColumnNotFound(columnName, table.name) + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + val updatedFields = c.multiFields.filterNot(_.name == field.name) :+ field + c.copy(multiFields = updatedFields) + table + case _ => throw DdlColumnNotFound(columnName, table.name) + } + } case DropColumnField( columnName, fieldName, ifExists ) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy( - multiFields = col.multiFields.filterNot(_.name == fieldName) - ) - else col - } - ) - else throw DdlColumnNotFound(columnName, table.name) + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + c.copy( + multiFields = c.multiFields.filterNot(_.name == fieldName) + ) + table + case _ => throw DdlColumnNotFound(columnName, table.name) + } + } case AlterColumnOption( columnName, optionKey, diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 29aac8bb..caa9321b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1388,7 +1388,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) ) => source should include( - "def param1 = ctx.profile.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)" + "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)" ) case other => fail(s"Expected DdlScriptProcessor for profile.seniority, got $other") } From 07b0ef33565b67991b3ea5d614be13b78f564fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 12:49:07 +0100 Subject: [PATCH 20/95] update ddl table diff for fields, mappings and settings --- .../elastic/sql/schema/DdlTableDiff.scala | 3 + .../elastic/sql/schema/package.scala | 207 ++++++++++++++---- 2 files changed, 170 insertions(+), 40 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala index 4419aeca..12d8b382 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala @@ -79,6 +79,9 @@ case class FieldAdded(column: String, field: DdlColumn) extends ColumnDiff { case class FieldRemoved(column: String, fieldName: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnField(column, fieldName) } +case class FieldAltered(column: String, field: DdlColumn) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnField(column, field) +} sealed trait MappingDiff extends AlterTableStatementDiff diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index b769012d..5861f12f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -33,6 +33,53 @@ package object schema { lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + sealed trait Diff + + case class Altered(name: String, value: Value[_]) extends Diff + + case class Removed(name: String) extends Diff + + private def compareOptions( + actual: Map[String, Value[_]], + desired: Map[String, Value[_]], + path: Option[String] = None + ): List[Diff] = { + val diffs = scala.collection.mutable.ListBuffer[Diff]() + + val allKeys = actual.keySet ++ desired.keySet + + allKeys.foreach { key => + val computedKey = s"${path.map(p => s"$p.").getOrElse("")}$key" + (actual.get(key), desired.get(key)) match { + case (None, Some(v)) => + diffs += Altered(computedKey, v) + case (Some(_), None) => + diffs += Removed(computedKey) + case (Some(a), Some(b)) => + b match { + case ObjectValue(desiredMap) => + a match { + case ObjectValue(actualMap) => + diffs ++= compareOptions( + actualMap, + desiredMap, + Some(computedKey) + ) + case _ => + diffs += Altered(computedKey, b) + } + case _ => + if (a != b) { + diffs += Altered(computedKey, b) + } + } + case _ => + } + } + + diffs.toList + } + sealed trait DdlProcessorType { def name: String } @@ -669,52 +716,144 @@ package object schema { val diffs = scala.collection.mutable.ListBuffer[ColumnDiff]() // 1. Type - if (SQLTypeUtils.elasticType(actual.dataType) != SQLTypeUtils.elasticType(desired.dataType)) - diffs += ColumnTypeChanged(path, actual.dataType, desired.dataType) + if (SQLTypeUtils.elasticType(actual.dataType) != SQLTypeUtils.elasticType(desired.dataType)) { + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnTypeChanged(path, actual.dataType, desired.dataType) + } + } // 2. Default (actual.defaultValue, desired.defaultValue) match { - case (None, Some(v)) => diffs += ColumnDefaultSet(path, v) - case (Some(_), None) => diffs += ColumnDefaultRemoved(path) + case (None, Some(v)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + diffs += ColumnDefaultSet(path, v) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnDefaultRemoved(path) + } case (Some(a), Some(b)) if a != b => - diffs += ColumnDefaultSet(path, b) + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnDefaultSet(path, b) + } case _ => } // 3. Script (actual.script, desired.script) match { - case (None, Some(s)) => diffs += ColumnScriptSet(path, s) - case (Some(_), None) => diffs += ColumnScriptRemoved(path) + case (None, Some(s)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + diffs += ColumnScriptSet(path, s) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnScriptRemoved(path) + } case (Some(a), Some(b)) if a.sql != b.sql => - diffs += ColumnScriptSet(path, b) + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnScriptSet(path, b) + } case _ => } // 4. Comment (actual.comment, desired.comment) match { - case (None, Some(c)) => diffs += ColumnCommentSet(path, c) - case (Some(_), None) => diffs += ColumnCommentRemoved(path) + case (None, Some(c)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentSet(path, c) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentRemoved(path) + } case (Some(a), Some(b)) if a != b => - diffs += ColumnCommentSet(path, b) + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentSet(path, b) + } case _ => } // 5. Not Null if (actual.notNull != desired.notNull) { - if (desired.notNull) diffs += ColumnNotNullSet(path) - else diffs += ColumnNotNullRemoved(path) + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + if (desired.notNull) diffs += ColumnNotNullSet(path) + else diffs += ColumnNotNullRemoved(path) + } } // 6. Options - val allOptions = actual.options.keySet ++ desired.options.keySet - for (key <- allOptions) { - (actual.options.get(key), desired.options.get(key)) match { - case (None, Some(v)) => diffs += ColumnOptionSet(path, key, v) - case (Some(_), None) => diffs += ColumnOptionRemoved(path, key) - case (Some(a), Some(b)) if a != b => - diffs += ColumnOptionSet(path, key, b) - case _ => - } + val optionDiffs = compareOptions(actual.options, desired.options) + parent match { + case Some(p) => + if (optionDiffs.nonEmpty) { + diffs += FieldAltered( + p.path, + desired + ) + } + case None => + diffs ++= optionDiffs.map { + case Altered(name, value) => ColumnOptionSet(path, name, value) + case Removed(name) => ColumnOptionRemoved(path, name) + } } // 7. STRUCT / multi-fields @@ -1200,28 +1339,16 @@ package object schema { // 4. Mappings val mappingDiffs = scala.collection.mutable.ListBuffer[MappingDiff]() - val allMappings = actual.mappings.keySet ++ desiredUpdated.mappings.keySet - for (key <- allMappings) { - (actual.mappings.get(key), desiredUpdated.mappings.get(key)) match { - case (None, Some(v)) => mappingDiffs += MappingSet(key, v) - case (Some(_), None) => mappingDiffs += MappingRemoved(key) - case (Some(a), Some(b)) if a != b => - mappingDiffs += MappingSet(key, b) - case _ => - } + mappingDiffs ++= compareOptions(actual.mappings, desired.mappings).map { + case Altered(name, value) => MappingSet(name, value) + case Removed(name) => MappingRemoved(name) } // 5. Settings val settingDiffs = scala.collection.mutable.ListBuffer[SettingDiff]() - val allSettings = actual.settings.keySet ++ desiredUpdated.settings.keySet - for (key <- allSettings) { - (actual.settings.get(key), desiredUpdated.settings.get(key)) match { - case (None, Some(v)) => settingDiffs += SettingSet(key, v) - case (Some(_), None) => settingDiffs += SettingRemoved(key) - case (Some(a), Some(b)) if a != b => - settingDiffs += SettingSet(key, b) - case _ => - } + settingDiffs ++= compareOptions(actual.settings, desiredUpdated.settings).map { + case Altered(name, value) => SettingSet(name, value) + case Removed(name) => SettingRemoved(name) } // 6. Pipeline From 26777b0c25557393948438e8e6ac2f312aa908c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 14:17:27 +0100 Subject: [PATCH 21/95] update documentation related to ddl support --- documentation/sql/ddl_statements.md | 177 +++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 4 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 3f450e22..a0a2f1c8 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -1,11 +1,11 @@ -# DDL Support +# Data Definition Language (DDL) Support [Back to index](README.md) This document describes the SQL statements supported by the API, focusing on **Data Definition Language (DDL)**. Each section provides syntax, examples, and notes on behavior. --- -## 📐 Data Definition Language (DDL) +## 📐 Table (DDLs) ### CREATE TABLE Create a new table with explicit column definitions or from a `SELECT` query. @@ -349,7 +349,7 @@ PUT _ingest/pipeline/users-composite-id { "set": { "field": "_id", - "value": "{{id}}|{{birthdate}}" + "value": "{{id}}||{{birthdate}}" } } ] @@ -358,7 +358,7 @@ PUT _ingest/pipeline/users-composite-id - `_id` is built from the values of `id` and `birthdate`. - Example: `id = 42`, `birthdate = 2025-12-10` → `_id = "42|2025-12-10"`. -- The separator (`|`) can be customized to avoid collisions (elastic.composite-key-separator). +- The separator (`||`) can be customized to avoid collisions (sql.composite-key-separator). --- @@ -443,4 +443,173 @@ PUT _ingest/pipeline/users-pipeline --- +## 🧩 Pipeline DDLs + +Pipelines define ordered ingestion processors that transform documents before they are indexed. +The SQL‑ES dialect supports three pipeline‑related statements: + +- `CREATE PIPELINE` +- `DROP PIPELINE` +- `ALTER PIPELINE` + +These statements allow users to declare, replace, remove, or modify ingestion pipelines in a declarative SQL‑style syntax. + +--- + +### 🚀 CREATE PIPELINE + +#### **Syntax** + +```sql +CREATE [OR REPLACE] PIPELINE pipeline_name +[IF NOT EXISTS] +WITH PROCESSORS ( + processor_1, + processor_2, + ... +); +``` + +#### **Description** + +- Creates a new ingestion pipeline. +- `OR REPLACE` overwrites an existing pipeline. +- `IF NOT EXISTS` prevents an error if the pipeline already exists. +- Processors are executed **in the order they are declared**. + +#### **Supported Processor Types** + +| SQL Processor | Elasticsearch Equivalent | Purpose | +|--------------|---------------------------|---------| +| `SET` | `set` | Assigns a value to a field in `ctx` | +| `SCRIPT` | `script` | Executes a Painless script | +| `REMOVE` | `remove` | Removes a field | +| `RENAME` | `rename` | Renames a field | +| `DATE_INDEX_NAME` | `date_index_name` | Generates an index name based on a date field | + +#### **Example** + +```sql +CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + SET ( + field = "name", + if = "ctx.name == null", + description = "DEFAULT 'anonymous'", + ignore_failure = true, + value = "anonymous" + ), + SCRIPT ( + description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + lang = "painless", + source = "...", + ignore_failure = true + ), + DATE_INDEX_NAME ( + field = "birthdate", + index_name_prefix = "users-", + date_formats = ["yyyy-MM"], + date_rounding = "M", + separator = "-", + ignore_failure = true + ) +); +``` + +--- + +### 🗑️ DROP PIPELINE + +#### **Syntax** + +```sql +DROP PIPELINE [IF EXISTS] pipeline_name; +``` + +#### **Description** + +- Deletes a pipeline. +- `IF EXISTS` prevents an error if the pipeline does not exist. + +#### **Example** + +```sql +DROP PIPELINE IF EXISTS user_pipeline; +``` + +--- + +### 🔧 ALTER PIPELINE + +#### **Syntax** + +```sql +ALTER PIPELINE [IF EXISTS] pipeline_name ( + alter_action_1, + alter_action_2, + ... +); +``` + +#### **Supported Actions** + +| Action | Description | +|--------|-------------| +| `ADD PROCESSOR ` | Appends a processor to the pipeline | +| `DROP PROCESSOR ()` | Removes a processor identified by its type and field | +| *(Optional future extension)* `ALTER PROCESSOR` | Modify an existing processor | + +#### **Example** + +```sql +ALTER PIPELINE IF EXISTS user_pipeline ( + ADD PROCESSOR SET ( + field = "status", + if = "ctx.status == null", + description = "status DEFAULT 'active'", + ignore_failure = true, + value = "active" + ), + DROP PROCESSOR SET (_id) +); +``` + +--- + +### 📐 Semantic Rules + +#### **Processor Identity** + +A processor is uniquely identified by: + +``` +(processor_type, field) +``` + +This means: + +- Changing the type or field is equivalent to removing the old processor and adding a new one. +- Property changes are treated as processor modifications. + +#### **Processor Ordering** + +Ordering matters for certain processors: + +- `DATE_INDEX_NAME` must appear last. +- `SET _id` must appear last. +- `RENAME` and `REMOVE` should appear before `SCRIPT`. + +Other processors may be reordered without semantic impact. + +--- + +### 📦 Summary of Supported Pipeline DDLs + +| Statement | Purpose | +|----------|----------| +| `CREATE PIPELINE` | Define or replace a pipeline | +| `DROP PIPELINE` | Remove a pipeline | +| `ALTER PIPELINE` | Add or remove processors | + +--- + [Back to index](README.md) From 898db77d67293c6319987b715b7694b2eeb6718d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 15:08:12 +0100 Subject: [PATCH 22/95] update column options, mappings and settings using Object Value set and remove methods - add toJson and fromJson methods --- .../softnetwork/elastic/schema/package.scala | 27 +--- .../app/softnetwork/elastic/sql/package.scala | 116 ++++++++++++++ .../elastic/sql/schema/package.scala | 141 +++-------------- .../elastic/sql/serialization/package.scala | 85 ++++++++++- .../elastic/sql/ObjectValueSpec.scala | 144 ++++++++++++++++++ .../elastic/sql/parser/ParserSpec.scala | 4 +- 6 files changed, 373 insertions(+), 144 deletions(-) create mode 100644 sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index d0019101..3e599387 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -28,6 +28,7 @@ import app.softnetwork.elastic.sql.schema.{ DdlScriptProcessor, DdlTable } +import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.JsonNode @@ -35,26 +36,6 @@ import scala.jdk.CollectionConverters._ package object schema { - private[schema] def extractOptions( - node: JsonNode, - ignoredKeys: Set[String] = Set.empty - ): Map[String, Value[_]] = { - node - .properties() - .asScala - .flatMap { entry => - val key = entry.getKey - val value = entry.getValue - - if (ignoredKeys.contains(key)) { - None - } else { - Value(value).map(key -> Value(_)) - } - } - .toMap - } - final case class EsField( name: String, `type`: String, @@ -97,7 +78,7 @@ package object schema { .getOrElse(Nil) val options = - extractOptions(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) + extractObject(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) val meta = options.get("meta") val comment = meta.flatMap { @@ -173,7 +154,7 @@ package object schema { }.toList) .getOrElse(Nil) - val options = extractOptions(mappings, ignoredKeys = Set("properties", "_doc")) + val options = extractObject(mappings, ignoredKeys = Set("properties", "_doc")) val meta = options.get("_meta") val primaryKey: List[String] = meta .map { @@ -228,7 +209,7 @@ package object schema { def apply(settings: JsonNode): EsSettings = { val index = settings.path("settings").path("index") - val options = extractOptions(index) + val options = extractObject(index) EsSettings( options = options diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 8eb4e70e..411aa992 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.sql.query._ import com.fasterxml.jackson.databind.JsonNode import java.security.MessageDigest +import scala.annotation.tailrec import scala.jdk.CollectionConverters._ import scala.reflect.runtime.universe._ import scala.util.Try @@ -383,6 +384,12 @@ package object sql { } } + sealed trait Diff + + case class Altered(name: String, value: Value[_]) extends Diff + + case class Removed(name: String) extends Diff + case class ObjectValue(override val value: Map[String, Value[_]]) extends Value[Map[String, Value[_]]](value) { override def sql: String = value @@ -397,6 +404,115 @@ package object sql { } } .mkString("(", ", ", ")") + + def set(path: String, newValue: Value[_]): ObjectValue = { + val keys = path.split("\\.") + val updatedValue = { + if (keys.length == 1) { + value + (keys.head -> newValue) + } else { + val parentPath = keys.dropRight(1).mkString(".") + val parentKey = keys.last + val parentObject = find(parentPath) match { + case Some(obj: ObjectValue) => obj + case _ => ObjectValue.empty + } + val updatedParent = parentObject.set(parentKey, newValue) + value + (keys.head -> updatedParent) + } + } + ObjectValue(updatedValue) + } + + def remove(path: String): ObjectValue = { + val keys = path.split("\\.") + val updatedValue = { + if (keys.length == 1) { + value - keys.head + } else { + val parentPath = keys.dropRight(1).mkString(".") + val parentKey = keys.last + val parentObject = find(parentPath) match { + case Some(obj: ObjectValue) => obj + case _ => ObjectValue.empty + } + val updatedParent = parentObject.remove(parentKey) + value + (keys.head -> updatedParent) + } + } + ObjectValue(updatedValue) + } + + def find(path: String): Option[Value[_]] = { + val keys = path.split("\\.") + @tailrec + def loop(current: Map[String, Value[_]], remainingKeys: Seq[String]): Option[Value[_]] = { + remainingKeys match { + case Seq() => None + case Seq(k) => current.get(k) + case k +: ks => + current.get(k) match { + case Some(obj: ObjectValue) => loop(obj.value, ks) + case _ => None + } + } + } + loop(value, keys.toSeq) + } + + def diff(other: ObjectValue, path: Option[String] = None): List[Diff] = { + val diffs = scala.collection.mutable.ListBuffer[Diff]() + + val actual = this.value + val desired = other.value + + val allKeys = actual.keySet ++ desired.keySet + + allKeys.foreach { key => + val computedKey = s"${path.map(p => s"$p.").getOrElse("")}$key" + (actual.get(key), desired.get(key)) match { + case (None, Some(v)) => + diffs += Altered(computedKey, v) + case (Some(_), None) => + diffs += Removed(computedKey) + case (Some(a), Some(b)) => + b match { + case ObjectValue(_) => + a match { + case ObjectValue(_) => + diffs ++= a + .asInstanceOf[ObjectValue] + .diff( + b.asInstanceOf[ObjectValue], + Some(computedKey) + ) + case _ => + diffs += Altered(computedKey, b) + } + case _ => + if (a != b) { + diffs += Altered(computedKey, b) + } + } + case _ => + } + } + + diffs.toList + } + + import app.softnetwork.elastic.sql.serialization._ + + def toJson: JsonNode = this + + } + + object ObjectValue { + import app.softnetwork.elastic.sql.serialization._ + + def empty: ObjectValue = ObjectValue(Map.empty) + + def fromJson(jsonNode: JsonNode): ObjectValue = jsonNode } case object Null extends Value[Null](null) with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 5861f12f..23bf7236 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.config.ElasticSqlConfig import app.softnetwork.elastic.sql.query._ -import app.softnetwork.elastic.sql.serialization.JacksonConfig +import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import com.fasterxml.jackson.databind.node.ObjectNode @@ -33,53 +33,6 @@ package object schema { lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() - sealed trait Diff - - case class Altered(name: String, value: Value[_]) extends Diff - - case class Removed(name: String) extends Diff - - private def compareOptions( - actual: Map[String, Value[_]], - desired: Map[String, Value[_]], - path: Option[String] = None - ): List[Diff] = { - val diffs = scala.collection.mutable.ListBuffer[Diff]() - - val allKeys = actual.keySet ++ desired.keySet - - allKeys.foreach { key => - val computedKey = s"${path.map(p => s"$p.").getOrElse("")}$key" - (actual.get(key), desired.get(key)) match { - case (None, Some(v)) => - diffs += Altered(computedKey, v) - case (Some(_), None) => - diffs += Removed(computedKey) - case (Some(a), Some(b)) => - b match { - case ObjectValue(desiredMap) => - a match { - case ObjectValue(actualMap) => - diffs ++= compareOptions( - actualMap, - desiredMap, - Some(computedKey) - ) - case _ => - diffs += Altered(computedKey, b) - } - case _ => - if (a != b) { - diffs += Altered(computedKey, b) - } - } - case _ => - } - } - - diffs.toList - } - sealed trait DdlProcessorType { def name: String } @@ -214,9 +167,7 @@ package object schema { def apply(processorType: DdlProcessorType, properties: ObjectValue): DdlProcessor = { val node = mapper.createObjectNode() - val props = mapper.createObjectNode() - updateNode(props, properties.value) - node.set(processorType.name, props) + node.set(processorType.name, properties.toJson) apply(node) } @@ -546,45 +497,6 @@ package object schema { } } - private[schema] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { - updates.foreach { case (k, v) => - v match { - case Null => node.putNull(k) - case BooleanValue(b) => node.put(k, b) - case StringValue(s) => node.put(k, s) - case ByteValue(b) => node.put(k, b) - case ShortValue(s) => node.put(k, s) - case IntValue(i) => node.put(k, i) - case LongValue(l) => node.put(k, l) - case DoubleValue(d) => node.put(k, d) - case FloatValue(f) => node.put(k, f) - case IdValue => node.put(k, s"${v.value}") - case IngestTimestampValue => node.put(k, s"${v.value}") - case v: Values[_, _] => - val arrayNode = mapper.createArrayNode() - v.values.foreach { - case Null => arrayNode.addNull() - case BooleanValue(b) => arrayNode.add(b) - case StringValue(s) => arrayNode.add(s) - case ByteValue(b) => arrayNode.add(b) - case ShortValue(s) => arrayNode.add(s) - case IntValue(i) => arrayNode.add(i) - case LongValue(l) => arrayNode.add(l) - case DoubleValue(d) => arrayNode.add(d) - case FloatValue(f) => arrayNode.add(f) - case ObjectValue(o) => arrayNode.add(updateNode(mapper.createObjectNode(), o)) - case _ => // do nothing - } - node.set(k, arrayNode) - case ObjectValue(value) => - if (value.nonEmpty) - node.set(k, updateNode(mapper.createObjectNode(), value)) - case _ => // do nothing - } - } - node - } - case class DdlColumn( name: String, dataType: SQLType, @@ -840,7 +752,7 @@ package object schema { } // 6. Options - val optionDiffs = compareOptions(actual.options, desired.options) + val optionDiffs = ObjectValue(actual.options).diff(ObjectValue(desired.options)) parent match { case Some(p) => if (optionDiffs.nonEmpty) { @@ -1000,6 +912,7 @@ package object schema { def merge(statements: Seq[AlterTableStatement]): DdlTable = { statements.foldLeft(this) { (table, statement) => statement match { + // table columns case AddColumn(column, ifNotExists) => if (ifNotExists && table.cols.contains(column.name)) table else if (!table.cols.contains(column.name)) @@ -1018,6 +931,7 @@ package object schema { } ) else throw DdlColumnNotFound(oldName, table.name) + // column type case AlterColumnType(columnName, newType, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1028,6 +942,7 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + // column script case AlterColumnScript(columnName, newScript, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1049,6 +964,7 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + // column default value case AlterColumnDefault(columnName, newDefault, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1069,6 +985,7 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + // column not null case AlterColumnNotNull(columnName, ifExists) => if (!table.cols.contains(columnName) && ifExists) table else if (table.cols.contains(columnName)) @@ -1089,6 +1006,7 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + // column options case AlterColumnOptions(columnName, newOptions, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1111,7 +1029,9 @@ package object schema { table.copy( columns = table.columns.map { col => if (col.name == columnName) - col.copy(options = col.options ++ Map(optionKey -> optionValue)) + col.copy( + options = ObjectValue(col.options).set(optionKey, optionValue).value + ) else col } ) @@ -1126,11 +1046,14 @@ package object schema { table.copy( columns = table.columns.map { col => if (col.name == columnName) - col.copy(options = col.options - optionKey) + col.copy( + options = ObjectValue(col.options).remove(optionKey).value + ) else col } ) else throw DdlColumnNotFound(columnName, table.name) + // column comments case AlterColumnComment(columnName, newComment, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1153,6 +1076,7 @@ package object schema { } ) else throw DdlColumnNotFound(columnName, table.name) + // multi-fields case AlterColumnFields(columnName, newFields, ifExists) => val col = find(columnName) val exists = col.isDefined @@ -1202,39 +1126,22 @@ package object schema { case _ => throw DdlColumnNotFound(columnName, table.name) } } - case AlterColumnOption( - columnName, - optionKey, - optionValue, - ifExists - ) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy( - options = col.options ++ Map(optionKey -> optionValue) - ) - else col - } - ) - else throw DdlColumnNotFound(columnName, table.name) + // mappings / settings case AlterTableMapping(optionKey, optionValue) => table.copy( - mappings = table.mappings ++ Map(optionKey -> optionValue) + mappings = ObjectValue(table.mappings).set(optionKey, optionValue).value ) case DropTableMapping(optionKey) => table.copy( - mappings = table.mappings - optionKey + mappings = ObjectValue(table.mappings).remove(optionKey).value ) case AlterTableSetting(optionKey, optionValue) => table.copy( - settings = table.settings ++ Map(optionKey -> optionValue) + settings = ObjectValue(table.settings).set(optionKey, optionValue).value ) case DropTableSetting(optionKey) => table.copy( - settings = table.settings - optionKey + settings = ObjectValue(table.settings).remove(optionKey).value ) case _ => table } @@ -1339,14 +1246,14 @@ package object schema { // 4. Mappings val mappingDiffs = scala.collection.mutable.ListBuffer[MappingDiff]() - mappingDiffs ++= compareOptions(actual.mappings, desired.mappings).map { + mappingDiffs ++= ObjectValue(actual.mappings).diff(ObjectValue(desired.mappings)).map { case Altered(name, value) => MappingSet(name, value) case Removed(name) => MappingRemoved(name) } // 5. Settings val settingDiffs = scala.collection.mutable.ListBuffer[SettingDiff]() - settingDiffs ++= compareOptions(actual.settings, desiredUpdated.settings).map { + settingDiffs ++= ObjectValue(actual.settings).diff(ObjectValue(desired.settings)).map { case Altered(name, value) => SettingSet(name, value) case Removed(name) => SettingRemoved(name) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 527f9fec..e1a35284 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -17,10 +17,19 @@ package app.softnetwork.elastic.sql import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature} +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.{ + DeserializationFeature, + JsonNode, + ObjectMapper, + SerializationFeature +} import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.scala.DefaultScalaModule +import scala.language.implicitConversions +import scala.jdk.CollectionConverters._ + package object serialization { /** Jackson ObjectMapper configuration */ @@ -50,4 +59,78 @@ package object serialization { } } + implicit def objectValueToObjectNode(value: ObjectValue): ObjectNode = { + import JacksonConfig.{objectMapper => mapper} + val node = mapper.createObjectNode() + updateNode(node, value.value) + node + } + + private[sql] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { + import JacksonConfig.{objectMapper => mapper} + + updates.foreach { case (k, v) => + v match { + case Null => node.putNull(k) + case BooleanValue(b) => node.put(k, b) + case StringValue(s) => node.put(k, s) + case ByteValue(b) => node.put(k, b) + case ShortValue(s) => node.put(k, s) + case IntValue(i) => node.put(k, i) + case LongValue(l) => node.put(k, l) + case DoubleValue(d) => node.put(k, d) + case FloatValue(f) => node.put(k, f) + case IdValue => node.put(k, s"${v.value}") + case IngestTimestampValue => node.put(k, s"${v.value}") + case v: Values[_, _] => + val arrayNode = mapper.createArrayNode() + v.values.foreach { + case Null => arrayNode.addNull() + case BooleanValue(b) => arrayNode.add(b) + case StringValue(s) => arrayNode.add(s) + case ByteValue(b) => arrayNode.add(b) + case ShortValue(s) => arrayNode.add(s) + case IntValue(i) => arrayNode.add(i) + case LongValue(l) => arrayNode.add(l) + case DoubleValue(d) => arrayNode.add(d) + case FloatValue(f) => arrayNode.add(f) + case ObjectValue(o) => arrayNode.add(updateNode(mapper.createObjectNode(), o)) + case _ => // do nothing + } + node.set(k, arrayNode) + case ObjectValue(value) => + if (value.nonEmpty) + node.set(k, updateNode(mapper.createObjectNode(), value)) + case _ => // do nothing + } + } + node + } + + implicit def jsonNodeToObjectValue( + node: JsonNode + )(implicit ignoredKeys: Set[String] = Set.empty): ObjectValue = { + ObjectValue(extractObject(node, ignoredKeys)) + } + + private[elastic] def extractObject( + node: JsonNode, + ignoredKeys: Set[String] = Set.empty + ): Map[String, Value[_]] = { + node + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val value = entry.getValue + + if (ignoredKeys.contains(key)) { + None + } else { + Value(value).map(key -> Value(_)) + } + } + .toMap + } + } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala new file mode 100644 index 00000000..a6a4043c --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala @@ -0,0 +1,144 @@ +package app.softnetwork.elastic.sql + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class ObjectValueSpec extends AnyFlatSpec with Matchers { + + "ObjectValue" should "correctly store and retrieve nested objects" in { + val nestedObject = Map( + "level1" -> ObjectValue( + Map( + "level2" -> ObjectValue( + Map( + "level3" -> StringValue("value") + ) + ) + ) + ) + ) + + val objectValue = ObjectValue(nestedObject) + + val retrievedObject = objectValue.value + val level1 = retrievedObject("level1").asInstanceOf[ObjectValue].value + val level2 = level1("level2").asInstanceOf[ObjectValue].value + val level3Value = level2("level3").asInstanceOf[StringValue] + + level3Value.value shouldEqual "value" + + objectValue.find("level1.level2.level3").getOrElse(Null).value shouldBe "value" + } + + it should "return Null for non-existing paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + objectValue.find("key2.key4") shouldBe None + objectValue.find("key5") shouldBe None + } + + it should "set values at specified paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.set("key2.key4", StringValue("newValue")) + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + val key4Value = key2("key4").asInstanceOf[StringValue] + + key4Value.value shouldEqual "newValue" + } + + it should "overwrite existing values when setting at a path" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.set("key2.key3", StringValue("updatedValue")) + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + val key3Value = key2("key3").asInstanceOf[StringValue] + + key3Value.value shouldEqual "updatedValue" + } + + it should "remove values at specified paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3"), + "key4" -> StringValue("value4") + ) + ) + ) + ) + + val updatedObject = objectValue.remove("key2.key3") + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + + key2.contains("key3") shouldBe false + key2.contains("key4") shouldBe true + } + + it should "handle removal of non-existing paths gracefully" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.remove("key2.key4") + + updatedObject shouldEqual objectValue + } + + it should "correctly serialize and deserialize to/from JSON" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val jsonString = objectValue.toJson + val deserializedObject = ObjectValue.fromJson(jsonString) + + deserializedObject shouldEqual objectValue + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index caa9321b..7b8472e2 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.schema.EsIndex -import app.softnetwork.elastic.sql.{IngestTimestampValue, ObjectValue, StringValue, Value} +import app.softnetwork.elastic.sql.{IngestTimestampValue, StringValue} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ @@ -17,8 +17,6 @@ import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import scala.collection.Set - object Queries { val numericalEq = "SELECT t.col1, t.col2 FROM Table AS t WHERE t.identifier = 1.0" val numericalLt = "SELECT * FROM Table WHERE identifier < 1" From 91a27b8bfc2f1c787b221e7347570cd3c630de7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 18 Dec 2025 16:29:51 +0100 Subject: [PATCH 23/95] to fix compilation errors with scala 2.12 --- .../elastic/sql/config/ElasticSqlConfig.scala | 4 ++-- .../softnetwork/elastic/schema/package.scala | 2 +- .../app/softnetwork/elastic/sql/package.scala | 2 ++ .../elastic/sql/schema/package.scala | 10 ++++----- .../elastic/sql/serialization/package.scala | 22 +++++++++++++++++++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index fbcbefeb..15ec4b20 100644 --- a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -1,6 +1,6 @@ package app.softnetwork.elastic.sql.config -import com.typesafe.config.Config +import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.StrictLogging import configs.Configs @@ -11,7 +11,7 @@ case class ElasticSqlConfig( object ElasticSqlConfig extends StrictLogging { def apply(config: Config): ElasticSqlConfig = { Configs[ElasticSqlConfig] - .get(config.withFallback(ConfigFactory.load("softnetwork-sql.conf"), "sql") + .get(config.withFallback(ConfigFactory.load("softnetwork-sql.conf")), "sql") .toEither match { case Left(configError) => logger.error(s"Something went wrong with the provided arguments $configError") diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 3e599387..6bc7a493 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -265,7 +265,7 @@ package object schema { } // 3. Enrichment from the pipeline (if provided) - val enrichedCols = scala.collection.mutable.Map.from(initialCols) + val enrichedCols = scala.collection.mutable.Map(initialCols.toSeq: _*) esPipeline.foreach { pipeline => pipeline.processors.foreach { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 411aa992..687ec7aa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -513,6 +513,8 @@ package object sql { def empty: ObjectValue = ObjectValue(Map.empty) def fromJson(jsonNode: JsonNode): ObjectValue = jsonNode + + def parseJson(jsonString: String): ObjectValue = jsonString } case object Null extends Value[Null](null) with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 23bf7236..54cf6d40 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -518,8 +518,7 @@ package object schema { if (path.contains(".")) { val parts = path.split("\\.") cols.get(parts.head).flatMap { col => - col.multiFields - .to(LazyList) + col.multiFields.toStream .flatMap(_.find(parts.tail.mkString("."))) .headOption } @@ -837,8 +836,7 @@ package object schema { if (path.contains(".")) { val parts = path.split("\\.") cols.get(parts.head).flatMap { col => - col.multiFields - .to(LazyList) + col.multiFields.toStream .flatMap(_.find(parts.tail.mkString("."))) .headOption } @@ -907,7 +905,9 @@ package object schema { } def ddlProcessors: Seq[DdlProcessor] = - columns.flatMap(_.ddlProcessors) ++ partitionBy.map(_.ddlProcessor(this)).toSeq ++ primaryKey + columns.flatMap(_.ddlProcessors) ++ partitionBy + .map(_.ddlProcessor(this)) + .toSeq ++ implicitly[Seq[DdlProcessor]](primaryKey) def merge(statements: Seq[AlterTableStatement]): DdlTable = { statements.foldLeft(this) { (table, statement) => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index e1a35284..7faf9070 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -133,4 +133,26 @@ package object serialization { .toMap } + private[elastic] def extractArray( + node: JsonNode + ): Seq[Value[_]] = { + node + .elements() + .asScala + .flatMap { element => + Value(element).map(Value(_)) + } + .toSeq + } + + implicit def toJson(json: String): JsonNode = { + import JacksonConfig.{objectMapper => mapper} + mapper.readTree(json) + } + + implicit def stringToObjectValue(json: String): ObjectValue = { + import JacksonConfig.{objectMapper => mapper} + mapper.readTree(json) + } + } From ebeb1e7129536149c99b0cb1152fe726429fa215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 19 Dec 2025 08:36:05 +0100 Subject: [PATCH 24/95] add pipeline api --- .../elastic/client/ElasticClientApi.scala | 1 + .../client/ElasticClientDelegator.scala | 31 +++ .../elastic/client/ElasticClientHelpers.scala | 48 +++++ .../elastic/client/PipelineApi.scala | 179 ++++++++++++++++++ .../client/metrics/MetricsElasticClient.scala | 17 ++ .../elastic/client/jest/JestClientApi.scala | 1 + .../elastic/client/jest/JestPipelineApi.scala | 67 +++++++ .../client/jest/actions/Pipeline.scala | 75 ++++++++ .../client/rest/RestHighLevelClientApi.scala | 65 +++++++ .../client/rest/RestHighLevelClientApi.scala | 65 +++++++ .../elastic/client/java/JavaClientApi.scala | 68 +++++++ .../elastic/client/java/JavaClientApi.scala | 68 +++++++ .../elastic/sql/query/package.scala | 38 ++-- .../elastic/sql/schema/package.scala | 44 +++++ 14 files changed, 749 insertions(+), 18 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 2ccc0500..6a0ff2e4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -45,6 +45,7 @@ trait ElasticClientApi with FlushApi with VersionApi with SerializationApi + with PipelineApi with ClientCompanion { protected def logger: Logger diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 6346b23b..e6481592 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -1406,4 +1406,35 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { /** Conversion BulkActionType -> BulkItem */ override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = delegate.actionToBulkItem(action.asInstanceOf) + + // ==================== PipelineApi (délégation) ==================== + + override def createPipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = { + delegate.createPipeline(pipelineName, pipelineDefinition) + } + + override def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { + delegate.deletePipeline(pipelineName) + } + + override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { + delegate.getPipeline(pipelineName) + } + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreatePipeline(pipelineName, pipelineDefinition) + + override private[client] def executeDeletePipeline(pipelineName: String): ElasticResult[Boolean] = + delegate.executeDeletePipeline(pipelineName) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = + delegate.executeGetPipeline(pipelineName) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 9a6f4bac..96ea2cec 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -186,6 +186,16 @@ trait ElasticClientHelpers { validateJson("validateJsonMappings", mappings) } + /** Validate the JSON pipeline definition. + * @param pipelineDefinition + * pipeline definition in JSON format + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateJsonPipeline(pipelineDefinition: String): Option[ElasticError] = { + validateJson("validateJsonPipeline", pipelineDefinition) + } + /** Validate the alias name. Aliases follow the same rules as indexes. * @param alias * alias name to validate @@ -205,6 +215,44 @@ trait ElasticClientHelpers { } } + /** Validate the pipeline name. Pipelines do not follow the same rules as indexes. only + * alphanumeric characters, points (.), underscores (_), hyphens (-) and at-signs (@) are allowed + * @param pipelineName + * pipeline name to validate + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validatePipelineName(pipelineName: String): Option[ElasticError] = { + if (pipelineName == null || pipelineName.trim.isEmpty) { + return Some( + ElasticError( + message = "Pipeline name cannot be empty", + cause = None, + statusCode = Some(400), + operation = Some("validatePipelineName") + ) + ) + } + + val trimmed = pipelineName.trim + + val pattern = "^[a-zA-Z0-9._\\-@]+$".r + + if (!pattern.matches(trimmed)) { + return Some( + ElasticError( + message = + "Pipeline name contains invalid characters: only alphanumeric characters, points (.), underscores (_), hyphens (-) and at-signs (@) are allowed", + cause = None, + statusCode = Some(400), + operation = Some("validatePipelineName") + ) + ) + } + + None + } + /** Logger une erreur avec le niveau approprié selon le status code. */ protected def logError( operation: String, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala new file mode 100644 index 00000000..50ca640d --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{AlterPipeline, CreatePipeline, DropPipeline} +import app.softnetwork.elastic.sql.schema.DdlPipeline + +trait PipelineApi extends ElasticClientHelpers { + + def pipeline(sql: String): ElasticResult[Boolean] = { + ElasticResult.attempt(Parser(sql)) match { + case ElasticSuccess(parsedStatement) => + parsedStatement match { + + case Right(statement) => + statement match { + case ddl: CreatePipeline => + createPipeline(ddl.name, ddl.ddlPipeline.json) + case ddl: DropPipeline => + deletePipeline(ddl.name) + case ddl: AlterPipeline => + getPipeline(ddl.name) match { + case ElasticSuccess(Some(existing)) => + val existingPipeline = DdlPipeline(name = ddl.name, json = existing) + val updatingPipeline = existingPipeline.merge(ddl.statements) + updatePipeline(ddl.name, updatingPipeline.json) + case ElasticSuccess(None) => + val error = + ElasticError( + message = s"Pipeline with name '${ddl.name}' not found", + statusCode = Some(404), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + case failure @ ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve pipeline with name '${ddl.name}': ${error.message}" + ) + failure + } + case _ => + val error = + ElasticError( + message = s"Unsupported pipeline DDL statement: $statement", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + case Left(l) => + val error = + ElasticError( + message = s"Error parsing pipeline DDL statement: ${l.msg}", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + case ElasticFailure(elasticError) => + ElasticResult.failure(elasticError) + } + } + + def createPipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { + validatePipelineName(pipelineName) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("createPipeline"), + statusCode = Some(400), + message = s"Invalid pipeline: ${error.message}" + ) + ) + case None => // OK + } + + validateJsonPipeline(pipelineDefinition) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("createPipeline"), + statusCode = Some(400), + message = s"Invalid pipeline: ${error.message}" + ) + ) + case None => // OK + } + + executeCreatePipeline(pipelineName, pipelineDefinition) match { + case success @ ElasticSuccess(created) => + if (created) { + logger.info(s"✅ Successfully created pipeline '$pipelineName'") + } else { + logger.warn(s"⚠️ Pipeline '$pipelineName' not created (it may already exist)") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to create pipeline '$pipelineName': ${error.message}") + failure + } + } + + def updatePipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { + // In Elasticsearch, creating a pipeline with an existing name updates it + createPipeline(pipelineName, pipelineDefinition) match { + case success @ ElasticSuccess(_) => success + case failure @ ElasticFailure(_) => failure + } + } + + def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { + executeDeletePipeline(pipelineName) match { + case success @ ElasticSuccess(deleted) => + if (deleted) { + logger.info(s"✅ Successfully deleted pipeline '$pipelineName'") + } else { + logger.warn(s"⚠️ Pipeline '$pipelineName' not deleted (it may not exist)") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to delete pipeline '$pipelineName': ${error.message}") + failure + } + } + + def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { + executeGetPipeline(pipelineName) match { + case success @ ElasticSuccess(maybePipeline) => + maybePipeline match { + case Some(_) => + logger.info(s"✅ Successfully retrieved pipeline '$pipelineName'") + case None => + logger.warn(s"⚠️ Pipeline '$pipelineName' not found") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve pipeline '$pipelineName': ${error.message}") + failure + } + } + + // ======================================================================== + // METHODS TO IMPLEMENT + // ======================================================================== + + private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeletePipeline(pipelineName: String): ElasticResult[Boolean] + + private[client] def executeGetPipeline(pipelineName: String): ElasticResult[Option[String]] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 43b69045..a86aa7cd 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -1077,4 +1077,21 @@ class MetricsElasticClient( override def resetMetrics(): Unit = { metricsCollector.resetMetrics() } + + // ==================== PipelineApi (délégation) ==================== + + override def createPipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = { + delegate.createPipeline(pipelineName, pipelineDefinition) + } + + override def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { + delegate.deletePipeline(pipelineName) + } + + override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { + delegate.getPipeline(pipelineName) + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 8d34727b..615b9220 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -45,6 +45,7 @@ trait JestClientApi with JestScrollApi with JestBulkApi with JestVersionApi + with JestPipelineApi with JestClientCompanion object JestClientApi extends SerializationApi { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala new file mode 100644 index 00000000..c34b1594 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala @@ -0,0 +1,67 @@ +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.jest.actions.Pipeline +import app.softnetwork.elastic.client.{result, PipelineApi} +import io.searchbox.client.JestResult + +trait JestPipelineApi extends PipelineApi with JestClientHelpers { + _: JestClientCompanion => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = { + // There is no direct API to create a pipeline in Jest. + apply().execute(Pipeline.Create(pipelineName, pipelineDefinition)) match { + case jestResult: JestResult if jestResult.isSucceeded => + result.ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to create pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + + override private[client] def executeDeletePipeline( + pipelineName: String + ): result.ElasticResult[Boolean] = { + // There is no direct API to delete a pipeline in Jest. + apply().execute(Pipeline.Delete(pipelineName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + result.ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to delete pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + + override private[client] def executeGetPipeline( + pipelineName: String + ): result.ElasticResult[Option[String]] = { + // There is no direct API to get a pipeline in Jest. + apply().execute(Pipeline.Get(pipelineName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + result.ElasticSuccess(Some(jsonString)) + } else { + result.ElasticSuccess(None) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to get pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala new file mode 100644 index 00000000..7f3969a2 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala @@ -0,0 +1,75 @@ +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.client.config.ElasticsearchVersion + +object Pipeline { + + import io.searchbox.action.AbstractAction + import io.searchbox.client.JestResult + import com.google.gson.Gson + + case class Create(pipelineId: String, json: String) extends AbstractAction[JestResult] { + + payload = json + + override def getRestMethodName: String = "PUT" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineId" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Get(pipelineId: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineId" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Delete(pipelineName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "DELETE" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 9955aac3..49696257 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -40,6 +40,12 @@ import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + GetPipelineResponse, + PutPipelineRequest +} import org.elasticsearch.action.search.{ ClearScrollRequest, MultiSearchRequest, @@ -61,6 +67,7 @@ import org.elasticsearch.client.indices.{ PutMappingRequest } import org.elasticsearch.common.Strings +import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.xcontent.{DeprecationHandler, XContentType} import org.elasticsearch.rest.RestStatus @@ -93,6 +100,7 @@ trait RestHighLevelClientApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion with RestHighLevelClientVersion + with RestHighLevelClientPipelineApi /** Version API implementation for RestHighLevelClient * @see @@ -1391,3 +1399,60 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } } + +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = + executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( + operation = "createPipeline", + retryable = false + )( + request = new PutPipelineRequest( + pipelineName, + new BytesArray(pipelineDefinition), + XContentType.JSON + ) + )( + executor = req => apply().ingest().putPipeline(req, RequestOptions.DEFAULT) + ) + + override private[client] def executeDeletePipeline( + pipelineName: String + ): result.ElasticResult[Boolean] = { + executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( + operation = "deletePipeline", + retryable = false + )( + request = new DeletePipelineRequest(pipelineName) + )( + executor = req => apply().ingest().deletePipeline(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetPipeline( + pipelineName: JSONQuery + ): result.ElasticResult[Option[JSONQuery]] = { + executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( + operation = "getPipeline", + retryable = true + )( + request = new GetPipelineRequest(pipelineName) + )( + executor = req => apply().ingest().getPipeline(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val pipelines = resp.pipelines().asScala + if (pipelines.nonEmpty) { + val pipeline = pipelines.head + Some(pipeline.toString) + } else { + None + } + } + ) + } +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 5a581a6c..fd11cd2b 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -40,6 +40,12 @@ import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + GetPipelineResponse, + PutPipelineRequest +} import org.elasticsearch.action.search.{ ClearScrollRequest, ClosePointInTimeRequest, @@ -64,6 +70,7 @@ import org.elasticsearch.client.indices.{ PutMappingRequest } import org.elasticsearch.common.Strings +import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue import org.elasticsearch.xcontent.{DeprecationHandler, XContentType} import org.elasticsearch.rest.RestStatus @@ -96,6 +103,7 @@ trait RestHighLevelClientApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion with RestHighLevelClientVersion + with RestHighLevelClientPipelineApi /** Version API implementation for RestHighLevelClient * @see @@ -1565,3 +1573,60 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } } + +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = + executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( + operation = "createPipeline", + retryable = false + )( + request = new PutPipelineRequest( + pipelineName, + new BytesArray(pipelineDefinition), + XContentType.JSON + ) + )( + executor = req => apply().ingest().putPipeline(req, RequestOptions.DEFAULT) + ) + + override private[client] def executeDeletePipeline( + pipelineName: String + ): result.ElasticResult[Boolean] = { + executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( + operation = "deletePipeline", + retryable = false + )( + request = new DeletePipelineRequest(pipelineName) + )( + executor = req => apply().ingest().deletePipeline(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetPipeline( + pipelineName: JSONQuery + ): result.ElasticResult[Option[JSONQuery]] = { + executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( + operation = "getPipeline", + retryable = true + )( + request = new GetPipelineRequest(pipelineName) + )( + executor = req => apply().ingest().getPipeline(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val pipelines = resp.pipelines().asScala + if (pipelines.nonEmpty) { + val pipeline = pipelines.head + Some(pipeline.toString) + } else { + None + } + } + ) + } +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index c79b9167..928f6d34 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -53,6 +53,11 @@ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESS import co.elastic.clients.elasticsearch.core.search.PointInTimeReference import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} +import co.elastic.clients.elasticsearch.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + PutPipelineRequest +} import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -80,6 +85,7 @@ trait JavaClientApi with JavaClientScrollApi with JavaClientCompanion with JavaClientVersionApi + with JavaClientPipelineApi /** Elasticsearch client implementation using the Java Client * @see @@ -1511,3 +1517,65 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } } + +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { + _: JavaClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createPipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .putPipeline( + new PutPipelineRequest.Builder() + .id(pipelineName) + .withJson(new StringReader(pipelineDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeletePipeline( + pipelineName: String + ): result.ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deletePipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .deletePipeline( + new DeletePipelineRequest.Builder() + .id(pipelineName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getPipeline", + index = None, + retryable = true + )( + apply() + .ingest() + .getPipeline( + new GetPipelineRequest.Builder() + .id(pipelineName) + .build() + ) + ) { resp => + resp.result().asScala.get(pipelineName).map { pipeline => + pipeline.toString + } + } + } +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index a458e39b..13be79fb 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -49,6 +49,11 @@ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESS import co.elastic.clients.elasticsearch.core.search.{PointInTimeReference, SearchRequestBody} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} +import co.elastic.clients.elasticsearch.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + PutPipelineRequest +} import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -76,6 +81,7 @@ trait JavaClientApi with JavaClientScrollApi with JavaClientCompanion with JavaClientVersionApi + with JavaClientPipelineApi /** Elasticsearch client implementation using the Java Client * @see @@ -1505,3 +1511,65 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } } + +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { + _: JavaClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createPipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .putPipeline( + new PutPipelineRequest.Builder() + .id(pipelineName) + .withJson(new StringReader(pipelineDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeletePipeline( + pipelineName: String + ): result.ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deletePipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .deletePipeline( + new DeletePipelineRequest.Builder() + .id(pipelineName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getPipeline", + index = None, + retryable = true + )( + apply() + .ingest() + .getPipeline( + new GetPipelineRequest.Builder() + .id(pipelineName) + .build() + ) + ) { resp => + resp.pipelines().asScala.get(pipelineName).map { pipeline => + pipeline.toString + } + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 53d4e2a0..b6453391 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -313,13 +313,15 @@ package object query { sealed trait DdlStatement extends Statement + sealed trait PipelineDdlStatement extends DdlStatement + case class CreatePipeline( name: String, pipelineType: DdlPipelineType, ifNotExists: Boolean = false, orReplace: Boolean = false, processors: Seq[DdlProcessor] - ) extends DdlStatement { + ) extends PipelineDdlStatement { override def sql: String = { val processorsDdl = processors.map(_.ddl).mkString(", ") val replaceClause = if (orReplace) " OR REPLACE" else "" @@ -331,11 +333,26 @@ package object query { DdlPipeline(name, pipelineType, processors) } + sealed trait AlterPipelineStatement extends AlterTableStatement + + case class AddPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + override def sql: String = s"ADD PROCESSOR ${processor.ddl}" + override def ddlProcessor: Option[DdlProcessor] = Some(processor) + } + case class DropPipelineProcessor(processorType: DdlProcessorType, column: String) + extends AlterPipelineStatement { + override def sql: String = s"DROP PROCESSOR ${processorType.name.toUpperCase}($column)" + } + case class AlterPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + override def sql: String = s"ALTER PROCESSOR ${processor.ddl}" + override def ddlProcessor: Option[DdlProcessor] = Some(processor) + } + case class AlterPipeline( name: String, ifExists: Boolean, statements: List[AlterPipelineStatement] - ) extends DdlStatement { + ) extends PipelineDdlStatement { override def sql: String = { val ifExistsClause = if (ifExists) " IF EXISTS " else "" val parenthesesNeeded = statements.size > 1 @@ -353,7 +370,7 @@ package object query { DdlPipeline(s"alter-pipeline-$name-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) } - case class DropPipeline(name: String, ifExists: Boolean = false) extends DdlStatement { + case class DropPipeline(name: String, ifExists: Boolean = false) extends PipelineDdlStatement { override def sql: String = { val ifExistsClause = if (ifExists) "IF EXISTS " else "" s"DROP PIPELINE $ifExistsClause$name" @@ -634,21 +651,6 @@ package object query { s"DROP SETTING $optionKey" } - sealed trait AlterPipelineStatement extends AlterTableStatement - - case class AddPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { - override def sql: String = s"ADD PROCESSOR ${processor.ddl}" - override def ddlProcessor: Option[DdlProcessor] = Some(processor) - } - case class DropPipelineProcessor(processorType: DdlProcessorType, column: String) - extends AlterPipelineStatement { - override def sql: String = s"DROP PROCESSOR ${processorType.name.toUpperCase}($column)" - } - case class AlterPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { - override def sql: String = s"ALTER PROCESSOR ${processor.ddl}" - override def ddlProcessor: Option[DdlProcessor] = Some(processor) - } - case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) extends DdlStatement { override def sql: String = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 54cf6d40..95302936 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -495,8 +495,52 @@ package object schema { // 5. Optional: detect reordering for processors where order matters diffs.toList } + + def merge(statements: Seq[AlterPipelineStatement]): DdlPipeline = { + statements.foldLeft(this) { (current, alter) => + alter match { + case AddPipelineProcessor(processor) => + current.copy(ddlProcessors = + current.ddlProcessors.filterNot(p => + p.processorType == processor.processorType && p.column == processor.column + ) :+ processor + ) + case DropPipelineProcessor(processorType, column) => + current.copy(ddlProcessors = + current.ddlProcessors.filterNot(p => + p.processorType == processorType && p.column == column + ) + ) + case AlterPipelineProcessor(processor) => + current.copy(ddlProcessors = current.ddlProcessors.map { p => + if (p.processorType == processor.processorType && p.column == processor.column) { + processor + } else { + p + } + }) + } + } + } } + object DdlPipeline { + def apply(name: String, json: String): DdlPipeline = { + val node = mapper.readTree(json) + val processorsNode = node.get("processors") + val processors = processorsNode.elements().asScala.toSeq.map { p => + DdlProcessor(p) + } + DdlPipeline( + name = name, + ddlPipelineType = + if (name.startsWith("default-")) DdlPipelineType.Default + else if (name.startsWith("final-")) DdlPipelineType.Final + else DdlPipelineType.Custom, + ddlProcessors = processors + ) + } + } case class DdlColumn( name: String, dataType: SQLType, From 64a65dfdc8f562cbfece47e3dd984ac7261e01a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 21 Dec 2025 11:52:31 +0100 Subject: [PATCH 25/95] finalize pipeline api - fix painless script with current date time --- .../sql/bridge/ElasticAggregation.scala | 16 +- .../elastic/sql/bridge/ElasticBridge.scala | 2 +- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/package.scala | 73 +- .../elastic/sql/SQLCriteriaSpec.scala | 4 + .../elastic/sql/SQLQuerySpec.scala | 86 ++- .../client/ElasticClientDelegator.scala | 13 +- .../elastic/client/ElasticsearchVersion.scala | 11 +- .../elastic/client/NopeClientApi.scala | 2 +- .../elastic/client/PipelineApi.scala | 124 +++- .../elastic/client/ScrollApi.scala | 6 +- .../elastic/client/SearchApi.scala | 13 +- .../client/metrics/MetricsElasticClient.scala | 4 +- .../sql/bridge/ElasticAggregation.scala | 16 +- .../elastic/sql/bridge/ElasticBridge.scala | 2 +- .../elastic/sql/bridge/ElasticCriteria.scala | 4 +- .../elastic/sql/bridge/package.scala | 77 ++- .../elastic/sql/SQLCriteriaSpec.scala | 4 + .../elastic/sql/SQLQuerySpec.scala | 178 +++-- .../elastic/client/jest/JestClientApi.scala | 14 +- .../elastic/client/jest/JestIndicesApi.scala | 12 +- .../elastic/client/jest/JestPipelineApi.scala | 21 +- .../elastic/client/jest/JestSearchApi.scala | 2 +- .../client/jest/actions/Pipeline.scala | 7 +- .../client/JestClientPipelineApiSpec.scala | 8 + .../client/rest/RestHighLevelClientApi.scala | 17 +- .../RestHighLevelClientPipelineApiSpec.scala | 8 + .../client/rest/RestHighLevelClientApi.scala | 21 +- .../RestHighLevelClientPipelineApiSpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 11 +- .../client/JavaClientPipelineApiSpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 11 +- .../client/JavaClientPipelineApiSpec.scala | 8 + .../elastic/sql/function/time/package.scala | 18 +- .../app/softnetwork/elastic/sql/package.scala | 7 + .../elastic/sql/schema/package.scala | 26 +- .../elastic/sql/serialization/package.scala | 10 + .../elastic/sql/parser/ParserSpec.scala | 8 +- .../elastic/client/ElasticClientSpec.scala | 4 +- .../elastic/client/MockElasticClientApi.scala | 2 +- .../elastic/client/PipelineApiSpec.scala | 638 ++++++++++++++++++ 41 files changed, 1257 insertions(+), 251 deletions(-) create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d6c11fd8..52405a90 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -101,7 +101,7 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - ): ElasticAggregation = { + )(implicit timestamp: Long): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -156,14 +156,14 @@ object ElasticAggregation { if (transformFuncs.nonEmpty) { val context = PainlessContext() val scriptSrc = identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => val context = PainlessContext() val scriptSrc = th.identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) case _ => buildField(aggName, sourceField) } @@ -231,7 +231,7 @@ object ElasticAggregation { .filter(_.isScriptField) .groupBy(_.sourceField) .map(_._2.head) - .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) + .map(f => f.sourceField -> now(Script(f.painless(None)).lang("painless"))) .toMap, size = limit, sorts = th.orderBy @@ -269,7 +269,7 @@ object ElasticAggregation { val painless = script.identifier.painless(None) bucketScriptAggregation( aggName, - Script(s"$painless").lang("painless"), + now(Script(s"$painless").lang("painless")), params.toMap ) case _ => @@ -349,7 +349,7 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Seq[Aggregation] = { + )(implicit timestamp: Long): Seq[Aggregation] = { val trees = BucketTree(buckets.flatMap(_.headOption)) println( s"[DEBUG] buildBuckets called with buckets: \n$trees" @@ -371,7 +371,7 @@ object ElasticAggregation { if (!bucket.isBucketScript && bucket.shouldBeScripted) { val context = PainlessContext() val painless = bucket.painless(Some(context)) - Some(Script(s"$context$painless").lang("painless")) + Some(now(Script(s"$context$painless").lang("painless"))) } else { None } @@ -520,7 +520,7 @@ object ElasticAggregation { val bucketSelector = bucketSelectorAggregation( "having_filter", - Script(script), + now(Script(script)), extractMetricsPathForBucket( criteria, nested, diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 124326dc..394aecaf 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -45,7 +45,7 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - ): Query = { + )(implicit timestamp: Long): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 0473a2eb..32305e18 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -21,7 +21,9 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit + timestamp: Long + ): Query = { val query = criteria.boolQuery.copy(group = group) query .filter(criteria.asFilter(Option(query))) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 2cc10739..36cacfc2 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -51,10 +51,17 @@ import scala.language.implicitConversions package object bridge { + def now(script: Script)(implicit timestamp: Long): Script = { + if (!script.script.contains("params.__now__")) { + return script + } + script.param("__now__", timestamp) + } + implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - ): Option[FilterAggregation] = { + )(implicit timestamp: Long): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -130,7 +137,7 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - ): Option[FilterAggregation] = + )(implicit timestamp: Long): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -148,7 +155,7 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[AbstractAggregation] = { + )(implicit timestamp: Long): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -200,7 +207,7 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[NestedAggregation] = { + )(implicit timestamp: Long): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -410,7 +417,9 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SingleSearch): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit + timestamp: Long + ): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -425,7 +434,9 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SingleSearch): SearchRequest = { + implicit def requestToSearchRequest( + request: SingleSearch + )(implicit timestamp: Long): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -485,13 +496,15 @@ package object bridge { val script = field.painless(Some(context)) scriptField( field.scriptName, - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") - .params(field.identifier.functions.headOption match { - case Some(f: PainlessParams) => f.params - case _ => Map.empty[String, Any] - }) + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + .params(field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + }) + ) ) } } @@ -523,9 +536,11 @@ package object bridge { } val scriptSort = ScriptSort( - script = Script(script = script) - .lang("painless") - .scriptType(Source), + script = now( + Script(script = script) + .lang("painless") + .scriptType(Source) + ), scriptSortType = sort.field.out match { case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number case _ => ScriptSortType.String @@ -557,7 +572,7 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - ): MultiSearchRequest = { + )(implicit timestamp: Long): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -568,7 +583,7 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -581,7 +596,7 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) return scriptQuery( - Script(script = s"$context$script").lang("painless").scriptType("source") + now(Script(script = s"$context$script").lang("painless").scriptType("source")) ) } // Geo distance special case @@ -799,18 +814,22 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => matchAllQuery() @@ -866,7 +885,7 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - ): Query = { + )(implicit timestamp: Long): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -997,7 +1016,7 @@ package object bridge { implicit def sqlQueryToAggregations( query: SelectStatement - ): Seq[ElasticAggregation] = { + )(implicit timestamp: Long): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 9f3bdc46..4749d58a 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -7,6 +7,8 @@ import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { @@ -17,6 +19,8 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { def asQuery(sql: String): String = { import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 58c071c2..e947fd4b 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -6,12 +6,16 @@ import app.softnetwork.elastic.sql.query._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { sqlQuery.statement match { case Some(value: SingleSearch) => @@ -998,9 +1002,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle having with date functions" in { val select: ElasticSearchRequest = SelectStatement("""SELECT userId, MAX(createdAt) as lastSeen - |FROM table - |GROUP BY userId - |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) + |FROM table + |GROUP BY userId + |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) val query = select.query println(query) query shouldBe @@ -1028,7 +1032,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1041,6 +1048,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by with having and date time functions" in { @@ -1090,7 +1098,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1106,6 +1117,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by index" in { @@ -1157,7 +1169,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1173,6 +1188,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle date_parse function" in { @@ -1918,7 +1934,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -1953,6 +1972,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle nullif function as script field" in { @@ -1969,7 +1989,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2012,6 +2035,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle cast function as script field" in { @@ -2028,19 +2052,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c2": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toInstant().toEpochMilli()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c3": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c4": { @@ -2097,6 +2130,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case function as script field" in { // 40 @@ -2113,7 +2147,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2150,6 +2187,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case with expression function as script field" in { @@ -2166,7 +2204,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2205,6 +2246,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle extract function as script field" in { @@ -2345,7 +2387,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2418,6 +2463,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("-", " - ") .replaceAll("==", " == ") .replaceAll("\\|\\|", " || ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle mathematic function as script field and condition" in { @@ -2877,7 +2923,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2928,6 +2977,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle all extractors" in { @@ -3509,7 +3559,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -3594,6 +3647,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") .replaceAll("false:", "false : ") .replaceAll("DateTimeFormatter", " DateTimeFormatter") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "determine the aggregation context" in { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index e6481592..8af090a3 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -1133,7 +1133,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] implicit def sqlSearchRequestToJsonQuery( sqlSearch: SingleSearch - ): String = + )(implicit timestamp: Long): String = delegate.sqlSearchRequestToJsonQuery(sqlSearch) override private[client] def executeSingleSearch( @@ -1416,8 +1416,8 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.createPipeline(pipelineName, pipelineDefinition) } - override def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { - delegate.deletePipeline(pipelineName) + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { + delegate.deletePipeline(pipelineName, ifExists = ifExists) } override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { @@ -1430,8 +1430,11 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.executeCreatePipeline(pipelineName, pipelineDefinition) - override private[client] def executeDeletePipeline(pipelineName: String): ElasticResult[Boolean] = - delegate.executeDeletePipeline(pipelineName) + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeletePipeline(pipelineName, ifExists = ifExists) override private[client] def executeGetPipeline( pipelineName: String diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 49fb2a41..c7f2eee0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -67,6 +67,15 @@ object ElasticsearchVersion { /** Check if version is ES 8+ */ def isEs8OrHigher(version: String): Boolean = { - isAtLeast(version, 8, 0) + isAtLeast(version, 8) + } + + def isEs7OrHigher(version: String): Boolean = { + isAtLeast(version, 7) + } + + def isEs6(version: String): Boolean = { + val (major, _, _) = parse(version) + major == 6 } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index 0252384e..ffafb00d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -209,7 +209,7 @@ trait NopeClientApi extends ElasticClientApi { */ override private[client] implicit def sqlSearchRequestToJsonQuery( sqlSearch: query.SingleSearch - ): String = "{\"query\": {\"match_all\": {}}}" + )(implicit timestamp: Long): String = "{\"query\": {\"match_all\": {}}}" override private[client] def executeUpdateSettings( index: String, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index 50ca640d..65f0aa37 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -24,10 +24,21 @@ import app.softnetwork.elastic.client.result.{ } import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{AlterPipeline, CreatePipeline, DropPipeline} -import app.softnetwork.elastic.sql.schema.DdlPipeline +import app.softnetwork.elastic.sql.schema.{DdlPipeline, GenericProcessor} -trait PipelineApi extends ElasticClientHelpers { +trait PipelineApi extends ElasticClientHelpers { _: VersionApi => + // ======================================================================== + // PIPELINE API + // ======================================================================== + + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ def pipeline(sql: String): ElasticResult[Boolean] = { ElasticResult.attempt(Parser(sql)) match { case ElasticSuccess(parsedStatement) => @@ -36,16 +47,56 @@ trait PipelineApi extends ElasticClientHelpers { case Right(statement) => statement match { case ddl: CreatePipeline => - createPipeline(ddl.name, ddl.ddlPipeline.json) + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = ddl.ddlPipeline.copy( + ddlProcessors = ddl.ddlPipeline.ddlProcessors.map { processor => + GenericProcessor( + processor.processorType, + processor.properties.filterNot(_._1 == "description") + ) + } + ) + createPipeline(ddl.name, pipeline.json) + } else + createPipeline(ddl.name, ddl.ddlPipeline.json) case ddl: DropPipeline => - deletePipeline(ddl.name) + deletePipeline(ddl.name, ifExists = ddl.ifExists) case ddl: AlterPipeline => getPipeline(ddl.name) match { case ElasticSuccess(Some(existing)) => val existingPipeline = DdlPipeline(name = ddl.name, json = existing) val updatingPipeline = existingPipeline.merge(ddl.statements) - updatePipeline(ddl.name, updatingPipeline.json) - case ElasticSuccess(None) => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve Elasticsearch version: ${error.message}" + ) + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = updatingPipeline.copy( + ddlProcessors = updatingPipeline.ddlProcessors.map { processor => + GenericProcessor( + processor.processorType, + processor.properties.filterNot(_._1 == "description") + ) + } + ) + updatePipeline(ddl.name, pipeline.json) + } else + updatePipeline(ddl.name, updatingPipeline.json) + case ElasticSuccess(None) if !ddl.ifExists => val error = ElasticError( message = s"Pipeline with name '${ddl.name}' not found", @@ -54,6 +105,11 @@ trait PipelineApi extends ElasticClientHelpers { ) logger.error(s"❌ ${error.message}") ElasticResult.failure(error) + case ElasticSuccess(None) if ddl.ifExists => + logger.info( + s"ℹ️ Pipeline with name '${ddl.name}' not found, skipping update as 'ifExists' is true" + ) + ElasticSuccess(false) case failure @ ElasticFailure(error) => logger.error( s"❌ Failed to retrieve pipeline with name '${ddl.name}': ${error.message}" @@ -85,6 +141,15 @@ trait PipelineApi extends ElasticClientHelpers { } } + /** Create a new ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ def createPipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { validatePipelineName(pipelineName) match { case Some(error) => @@ -124,6 +189,15 @@ trait PipelineApi extends ElasticClientHelpers { } } + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ def updatePipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { // In Elasticsearch, creating a pipeline with an existing name updates it createPipeline(pipelineName, pipelineDefinition) match { @@ -132,8 +206,30 @@ trait PipelineApi extends ElasticClientHelpers { } } - def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { - executeDeletePipeline(pipelineName) match { + /** Delete an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param ifExists + * flag indicating whether to ignore if the pipeline does not exist + * @return + * ElasticResult[Boolean] indicating success or failure + */ + def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { + if (ifExists) { + getPipeline(pipelineName) match { + case ElasticSuccess(Some(_)) => // Pipeline exists, proceed to delete + case ElasticSuccess(None) => + logger.info( + s"ℹ️ Pipeline '$pipelineName' does not exist, skipping deletion as 'ifExists' is true" + ) + return ElasticSuccess(false) // Indicate that nothing was deleted + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to check existence of pipeline '$pipelineName': ${error.message}") + return failure + } + } + executeDeletePipeline(pipelineName, ifExists = ifExists) match { case success @ ElasticSuccess(deleted) => if (deleted) { logger.info(s"✅ Successfully deleted pipeline '$pipelineName'") @@ -147,6 +243,13 @@ trait PipelineApi extends ElasticClientHelpers { } } + /** Retrieve an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @return + * ElasticResult[Option[String]\] containing the pipeline definition in JSON format if found + */ def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { executeGetPipeline(pipelineName) match { case success @ ElasticSuccess(maybePipeline) => @@ -172,7 +275,10 @@ trait PipelineApi extends ElasticClientHelpers { pipelineDefinition: String ): ElasticResult[Boolean] - private[client] def executeDeletePipeline(pipelineName: String): ElasticResult[Boolean] + private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] private[client] def executeGetPipeline(pipelineName: String): ElasticResult[Option[String]] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 17415e87..19680764 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -120,6 +120,7 @@ trait ScrollApi extends ElasticClientHelpers { sql: SelectStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { + implicit def timestamp: Long = System.currentTimeMillis() sql.statement match { case Some(single: SingleSearch) => if (single.windowFunctions.nonEmpty) @@ -379,7 +380,10 @@ trait ScrollApi extends ElasticClientHelpers { sql: SelectStatement, request: SingleSearch, config: ScrollConfig - )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { + )(implicit + system: ActorSystem, + timestamp: Long + ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { implicit val ec: ExecutionContext = system.dispatcher diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 315f8cd9..297dfe90 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -66,6 +66,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * the Elasticsearch response */ def search(sql: SelectStatement): ElasticResult[ElasticResponse] = { + implicit def timestamp: Long = System.currentTimeMillis() sql.statement match { case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( @@ -304,6 +305,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { + implicit def timestamp: Long = System.currentTimeMillis() sqlQuery.statement match { case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( @@ -749,6 +751,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { )(implicit formats: Formats ): ElasticResult[Seq[(U, Seq[I])]] = { + implicit def timestamp: Long = System.currentTimeMillis() sql.statement match { case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( @@ -986,7 +989,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String + private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String private def parseInnerHits[M: Manifest: ClassTag, I: Manifest: ClassTag]( searchResult: JsonObject, @@ -1102,7 +1107,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { private def searchWithWindowEnrichment( sql: SelectStatement, request: SingleSearch - ): ElasticResult[ElasticResponse] = { + )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { logger.info(s"🪟 Detected ${request.windowFunctions.size} window functions") @@ -1128,7 +1133,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { */ protected def executeWindowAggregations( request: SingleSearch - ): ElasticResult[WindowCache] = { + )(implicit timestamp: Long): ElasticResult[WindowCache] = { // Build aggregation request val aggRequest = buildWindowAggregationRequest(request) @@ -1216,7 +1221,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { private def executeBaseQuery( sql: SelectStatement, request: SingleSearch - ): ElasticResult[ElasticResponse] = { + )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { val baseQuery = createBaseQuery(sql, request) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index a86aa7cd..04ee3c8a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -1087,8 +1087,8 @@ class MetricsElasticClient( delegate.createPipeline(pipelineName, pipelineDefinition) } - override def deletePipeline(pipelineName: String): ElasticResult[Boolean] = { - delegate.deletePipeline(pipelineName) + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { + delegate.deletePipeline(pipelineName, ifExists = ifExists) } override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d76b14a5..c54c4451 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -101,7 +101,7 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - ): ElasticAggregation = { + )(implicit timestamp: Long): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -156,14 +156,14 @@ object ElasticAggregation { if (transformFuncs.nonEmpty) { val context = PainlessContext() val scriptSrc = identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => val context = PainlessContext() val scriptSrc = th.identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) case _ => buildField(aggName, sourceField) } @@ -231,7 +231,7 @@ object ElasticAggregation { .filter(_.isScriptField) .groupBy(_.sourceField) .map(_._2.head) - .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) + .map(f => f.sourceField -> now(Script(f.painless(None)).lang("painless"))) .toMap, size = limit, sorts = th.orderBy @@ -269,7 +269,7 @@ object ElasticAggregation { val painless = script.identifier.painless(None) bucketScriptAggregation( aggName, - Script(s"$painless").lang("painless"), + now(Script(s"$painless").lang("painless")), params.toMap ) case _ => @@ -349,7 +349,7 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Seq[Aggregation] = { + )(implicit timestamp: Long): Seq[Aggregation] = { val trees = BucketTree(buckets.flatMap(_.headOption)) println( s"[DEBUG] buildBuckets called with buckets: \n$trees" @@ -371,7 +371,7 @@ object ElasticAggregation { if (!bucket.isBucketScript && bucket.shouldBeScripted) { val context = PainlessContext() val painless = bucket.painless(Some(context)) - Some(Script(s"$context$painless").lang("painless")) + Some(now(Script(s"$context$painless").lang("painless"))) } else { None } @@ -520,7 +520,7 @@ object ElasticAggregation { val bucketSelector = bucketSelectorAggregation( "having_filter", - Script(script), + now(Script(script)), extractMetricsPathForBucket( criteria, nested, diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 187bf278..799a3509 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -45,7 +45,7 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - ): Query = { + )(implicit timestamp: Long): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index caa12dd0..f745889a 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -21,7 +21,9 @@ import com.sksamuel.elastic4s.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit + timestamp: Long + ): Query = { val query = criteria.boolQuery.copy(group = group) query .filter(criteria.asFilter(Option(query))) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 87a77d4f..ebed4abe 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -47,10 +47,17 @@ import scala.language.implicitConversions package object bridge { + def now(script: Script)(implicit timestamp: Long): Script = { + if (!script.script.contains("params.__now__")) { + return script + } + script.param("__now__", timestamp) + } + implicit def requestToNestedFilterAggregation( request: SingleSearch, innerHitsName: String - ): Option[FilterAggregation] = { + )(implicit timestamp: Long): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -126,7 +133,7 @@ package object bridge { implicit def requestToFilterAggregation( request: SingleSearch - ): Option[FilterAggregation] = + )(implicit timestamp: Long): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -144,7 +151,7 @@ package object bridge { implicit def requestToRootAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[AbstractAggregation] = { + )(implicit timestamp: Long): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -194,7 +201,7 @@ package object bridge { implicit def requestToScopedAggregations( request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[NestedAggregation] = { + )(implicit timestamp: Long): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -404,7 +411,9 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SingleSearch): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit + timestamp: Long + ): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -419,7 +428,9 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SingleSearch): SearchRequest = { + implicit def requestToSearchRequest( + request: SingleSearch + )(implicit timestamp: Long): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -479,13 +490,15 @@ package object bridge { val script = field.painless(Some(context)) scriptField( field.scriptName, - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") - .params(field.identifier.functions.headOption match { - case Some(f: PainlessParams) => f.params - case _ => Map.empty[String, Any] - }) + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + .params((field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + })) + ) ) } } @@ -517,9 +530,11 @@ package object bridge { } val scriptSort = ScriptSort( - script = Script(script = script) - .lang("painless") - .scriptType(Source), + script = now( + Script(script = script) + .lang("painless") + .scriptType(Source) + ), scriptSortType = sort.field.out match { case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number case _ => ScriptSortType.String @@ -551,7 +566,7 @@ package object bridge { implicit def requestToMultiSearchRequest( request: MultiSearch - ): MultiSearchRequest = { + )(implicit timestamp: Long): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -562,7 +577,7 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -575,7 +590,11 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) return scriptQuery( - Script(script = s"$context$script").lang("painless").scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } // Geo distance special case @@ -793,18 +812,22 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => matchAllQuery() @@ -860,7 +883,7 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - ): Query = { + )(implicit timestamp: Long): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -992,7 +1015,7 @@ package object bridge { @deprecated implicit def sqlQueryToAggregations( query: SelectStatement - ): Seq[ElasticAggregation] = { + )(implicit timestamp: Long): Seq[ElasticAggregation] = { import query._ statement .map { diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index dc7cd685..a612cb81 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -8,6 +8,8 @@ import com.sksamuel.elastic4s.searches.SearchRequest import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { @@ -18,6 +20,8 @@ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { def asQuery(sql: String): String = { import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 856e7bca..0526b085 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -6,12 +6,16 @@ import app.softnetwork.elastic.sql.query.{SelectStatement, SingleSearch} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { sqlQuery.statement match { case Some(value: SingleSearch) => @@ -1028,7 +1032,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1041,6 +1048,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by with having and date time functions" in { @@ -1090,7 +1098,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1106,6 +1117,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by index" in { @@ -1117,62 +1129,66 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { println(query) query shouldBe """{ - | "query": { - | "match_all": {} - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "Country": { - | "terms": { - | "field": "Country", - | "exclude": "USA", - | "min_doc_count": 1, - | "order": { - | "_key": "asc" - | } - | }, - | "aggs": { - | "City": { - | "terms": { - | "field": "City", - | "exclude": "Berlin", - | "min_doc_count": 1 - | }, - | "aggs": { - | "cnt": { - | "value_count": { - | "field": "CustomerID" - | } - | }, - | "lastSeen": { - | "max": { - | "field": "createdAt" - | } - | }, - | "having_filter": { - | "bucket_selector": { - | "buckets_path": { - | "cnt": "cnt", - | "lastSeen": "lastSeen" - | }, - | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" - | } - | } - | } - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "Country": { + | "terms": { + | "field": "Country", + | "exclude": "USA", + | "min_doc_count": 1, + | "order": { + | "_key": "asc" + | } + | }, + | "aggs": { + | "City": { + | "terms": { + | "field": "City", + | "exclude": "Berlin", + | "min_doc_count": 1 + | }, + | "aggs": { + | "cnt": { + | "value_count": { + | "field": "CustomerID" + | } + | }, + | "lastSeen": { + | "max": { + | "field": "createdAt" + | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "cnt": "cnt", + | "lastSeen": "lastSeen" + | }, + | "script": { + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } + | } + | } + | } + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle date_parse function" in { @@ -1918,7 +1934,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -1953,6 +1972,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle nullif function as script field" in { @@ -1969,7 +1989,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2012,6 +2035,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle cast function as script field" in { @@ -2028,19 +2052,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c2": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toInstant().toEpochMilli()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c3": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c4": { @@ -2097,6 +2130,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case function as script field" in { // 40 @@ -2113,7 +2147,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2150,6 +2187,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case with expression function as script field" in { @@ -2166,7 +2204,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2205,6 +2246,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle extract function as script field" in { @@ -2345,7 +2387,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2418,6 +2463,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("-", " - ") .replaceAll("==", " == ") .replaceAll("\\|\\|", " || ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle mathematic function as script field and condition" in { @@ -2877,7 +2923,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2928,6 +2977,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle all extractors" in { @@ -3509,7 +3559,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -3594,6 +3647,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") .replaceAll("false:", "false : ") .replaceAll("DateTimeFormatter", " DateTimeFormatter") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "determine the aggregation context" in { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index 615b9220..a92d4072 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -17,7 +17,6 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client._ -import app.softnetwork.elastic.sql.query.{SelectStatement, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import io.searchbox.action.BulkableAction import io.searchbox.core._ @@ -57,19 +56,8 @@ object JestClientApi extends SerializationApi { search.build() } - implicit class SearchSQLQuery(sqlQuery: SelectStatement) { - def jestSearch: Option[Search] = { - sqlQuery.statement match { - case Some(value: SingleSearch) => - val request: ElasticSearchRequest = value - Some(request) - case _ => None - } - } - } - implicit class SearchElasticQuery(elasticQuery: ElasticQuery) { - def search: (Search, JSONQuery) = { + def search: (Search, String) = { import elasticQuery._ val _search = new Search.Builder(query) for (indice <- indices) _search.addIndex(indice) diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index ae90f749..cd0c17e4 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -46,11 +46,15 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe index = Some(index), retryable = false // Creation can not be retried ) { - new CreateIndex.Builder(index) + val builder = new CreateIndex.Builder(index) .settings(settings) - .aliases(aliases.map(alias => s"""{"$alias":{}}""").mkString(",")) - .mappings(mappings.getOrElse("{}")) - .build() + if (aliases.nonEmpty) { + builder.aliases(aliases.map(alias => s"""{"$alias":{}}""").mkString(",")) + } + mappings.foreach { mapping => + builder.mappings(mapping) + } + builder.build() } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala index c34b1594..c684110f 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala @@ -1,11 +1,14 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.jest.actions.Pipeline -import app.softnetwork.elastic.client.{result, PipelineApi} +import app.softnetwork.elastic.client.{result, PipelineApi, SerializationApi} +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult -trait JestPipelineApi extends PipelineApi with JestClientHelpers { - _: JestClientCompanion => +trait JestPipelineApi extends PipelineApi with JestClientHelpers with JestVersionApi { + _: SerializationApi with JestClientCompanion => override private[client] def executeCreatePipeline( pipelineName: String, @@ -26,7 +29,8 @@ trait JestPipelineApi extends PipelineApi with JestClientHelpers { } override private[client] def executeDeletePipeline( - pipelineName: String + pipelineName: String, + ifExists: Boolean ): result.ElasticResult[Boolean] = { // There is no direct API to delete a pipeline in Jest. apply().execute(Pipeline.Delete(pipelineName)) match { @@ -50,7 +54,14 @@ trait JestPipelineApi extends PipelineApi with JestClientHelpers { case jestResult: JestResult if jestResult.isSucceeded => val jsonString = jestResult.getJsonString if (jsonString != null && jsonString.nonEmpty) { - result.ElasticSuccess(Some(jsonString)) + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode if objectNode.has(pipelineName) => + val pipelineNode = objectNode.get(pipelineName) + result.ElasticSuccess(Some(pipelineNode)) + case _ => + result.ElasticSuccess(None) + } } else { result.ElasticSuccess(None) } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index cf92984a..21809931 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -31,7 +31,7 @@ trait JestSearchApi extends SearchApi with JestClientHelpers { private[client] implicit def sqlSearchRequestToJsonQuery( sqlSearch: SingleSearch - ): String = + )(implicit timestamp: Long): String = implicitly[ElasticSearchRequest](sqlSearch).query import JestClientApi._ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala index 7f3969a2..733f1549 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala @@ -25,6 +25,7 @@ object Pipeline { ): JestResult = { val result = new JestResult(gson) result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) Option(json).foreach(result.setJsonString) Option(reasonPhrase).foreach(result.setErrorMessage) result @@ -46,7 +47,10 @@ object Pipeline { ): JestResult = { val result = new JestResult(gson) result.setResponseCode(statusCode) - Option(json).foreach(result.setJsonString) + result.setSucceeded(statusCode == 200 || statusCode == 201 || statusCode == 404) + if (statusCode != 404) { + Option(json).foreach(result.setJsonString) + } Option(reasonPhrase).foreach(result.setErrorMessage) result } @@ -67,6 +71,7 @@ object Pipeline { ): JestResult = { val result = new JestResult(gson) result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) Option(json).foreach(result.setJsonString) Option(reasonPhrase).foreach(result.setErrorMessage) result diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala new file mode 100644 index 00000000..aa59f134 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class JestClientPipelineApiSpec extends PipelineApiSpec with EmbeddedElasticTestKit { + override def client: PipelineApi = new JestClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 49696257..2c2baa68 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.{Alias, IndicesAliasesRequest} @@ -751,7 +752,9 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1400,7 +1403,10 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } -trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { +trait RestHighLevelClientPipelineApi + extends PipelineApi + with RestHighLevelClientHelpers + with RestHighLevelClientVersion { _: RestHighLevelClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( @@ -1421,7 +1427,8 @@ trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClien ) override private[client] def executeDeletePipeline( - pipelineName: String + pipelineName: String, + ifExists: Boolean ): result.ElasticResult[Boolean] = { executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( operation = "deletePipeline", @@ -1448,7 +1455,9 @@ trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClien val pipelines = resp.pipelines().asScala if (pipelines.nonEmpty) { val pipeline = pipelines.head - Some(pipeline.toString) + val config = pipeline.getConfigAsMap + val mapper = JacksonConfig.objectMapper + Some(mapper.writeValueAsString(config)) } else { None } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala new file mode 100644 index 00000000..2c38f9ce --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with EmbeddedElasticTestKit { + override lazy val client: PipelineApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index fd11cd2b..be3a2ff0 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.Alias @@ -224,8 +225,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH case statusLine if statusLine.getStatusCode >= 400 => (false, None) case _ => - val json = new JsonParser() - .parse( + val json = JsonParser + .parseString( scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString ) .getAsJsonObject @@ -756,7 +757,9 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1574,7 +1577,10 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } -trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { +trait RestHighLevelClientPipelineApi + extends PipelineApi + with RestHighLevelClientHelpers + with RestHighLevelClientVersion { _: RestHighLevelClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( @@ -1595,7 +1601,8 @@ trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClien ) override private[client] def executeDeletePipeline( - pipelineName: String + pipelineName: String, + ifExists: Boolean ): result.ElasticResult[Boolean] = { executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( operation = "deletePipeline", @@ -1622,7 +1629,9 @@ trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClien val pipelines = resp.pipelines().asScala if (pipelines.nonEmpty) { val pipeline = pipelines.head - Some(pipeline.toString) + val config = pipeline.getConfigAsMap + val mapper = JacksonConfig.objectMapper + Some(mapper.writeValueAsString(config)) } else { None } diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala new file mode 100644 index 00000000..b10763b7 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 928f6d34..ac31e3f3 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -774,7 +774,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1518,7 +1520,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } -trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with JavaClientVersionApi { _: JavaClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( @@ -1541,7 +1543,8 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { )(resp => resp.acknowledged()) override private[client] def executeDeletePipeline( - pipelineName: String + pipelineName: String, + ifExists: Boolean ): result.ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deletePipeline", @@ -1574,7 +1577,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { ) ) { resp => resp.result().asScala.get(pipelineName).map { pipeline => - pipeline.toString + convertToJson(pipeline) } } } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala new file mode 100644 index 00000000..46efe3bd --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 13be79fb..78190da2 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -768,7 +768,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( @@ -1512,7 +1514,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } -trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with JavaClientVersionApi { _: JavaClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( @@ -1535,7 +1537,8 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { )(resp => resp.acknowledged()) override private[client] def executeDeletePipeline( - pipelineName: String + pipelineName: String, + ifExists: Boolean ): result.ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deletePipeline", @@ -1568,7 +1571,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { ) ) { resp => resp.pipelines().asScala.get(pipelineName).map { pipeline => - pipeline.toString + convertToJson(pipeline) } } } diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala new file mode 100644 index 00000000..46efe3bd --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 6ddb905d..acc58ea7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -38,6 +38,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes, SQLVarchar } +import app.softnetwork.elastic.sql.function.time.CurrentFunction.queryTimestamp import app.softnetwork.elastic.sql.time.{IsoField, TimeField, TimeInterval, TimeUnit} package object time { @@ -116,7 +117,7 @@ package object time { } sealed trait DateTimeFunction extends Function { - def now: String = "ZonedDateTime.now(ZoneId.of('Z'))" + def now: String = "ZonedDateTime.ofInstant(Instant.ofEpochMilli({{__now__}}), ZoneId.of('Z'))" override def baseType: SQLType = SQLTypes.DateTime } @@ -140,17 +141,28 @@ package object time { override def painless(context: Option[PainlessContext]): String = { context match { case Some(ctx) => - ctx.addParam(LiteralParam(param)) match { + ctx.addParam(LiteralParam(param.replaceAll("\\{\\{__now__}}", ctx.timestamp))) match { case Some(p) => return SQLTypeUtils.coerce(p, this.baseType, this.out, nullable = false, context) case _ => } case _ => } - SQLTypeUtils.coerce(param, this.baseType, this.out, nullable = false, context) + SQLTypeUtils.coerce( + param.replaceAll("\\{\\{__now__}}", queryTimestamp), + this.baseType, + this.out, + nullable = false, + context + ) } } + object CurrentFunction { + val processorTimestamp: String = "ctx['_ingest']['timestamp']" + val queryTimestamp: String = "params.__now__" + } + sealed trait CurrentDateTimeFunction extends DateTimeFunction with CurrentFunction { override def param: String = now } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 687ec7aa..6b8c41a2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -153,6 +153,13 @@ package object sql { def isProcessor: Boolean = context == PainlessContextType.Processor + lazy val timestamp: String = { + context match { + case PainlessContextType.Processor => CurrentFunction.processorTimestamp + case PainlessContextType.Query => CurrentFunction.queryTimestamp + } + } + /** Add a token parameter to the context if not already present * * @param token diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 95302936..08213451 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -63,31 +63,7 @@ package object schema { def ignoreFailure: Boolean final def node: ObjectNode = { val node = mapper.createObjectNode() - val props = mapper.createObjectNode() - for ((key, value) <- properties) { - value match { - case v: String => props.put(key, v) - case v: Boolean => props.put(key, v) - case v: Int => props.put(key, v) - case v: Long => props.put(key, v) - case v: Double => props.put(key, v) - case v: ObjectNode => props.set(key, v) - case v: Seq[_] => - val arrayNode = mapper.createArrayNode() - v.foreach { - case s: String => arrayNode.add(s) - case b: Boolean => arrayNode.add(b) - case i: Int => arrayNode.add(i) - case l: Long => arrayNode.add(l) - case d: Double => arrayNode.add(d) - case o: ObjectNode => arrayNode.add(o) - case _ => - } - props.set(key, arrayNode) - case _ => - } - } - node.set(processorType.name, props) + node.set(processorType.name, properties) node } def json: String = mapper.writeValueAsString(node) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 7faf9070..40771006 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -59,6 +59,16 @@ package object serialization { } } + implicit def mapToJsonNode(value: Map[String, Any]): JsonNode = { + import JacksonConfig.{objectMapper => mapper} + mapper.valueToTree[JsonNode](value.asJava) + } + + implicit def jsonNodeToString(node: JsonNode): String = { + import JacksonConfig.{objectMapper => mapper} + mapper.writeValueAsString(node) + } + implicit def objectValueToObjectNode(value: ObjectValue): ObjectNode = { import JacksonConfig.{objectMapper => mapper} val node = mapper.createObjectNode() diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 7b8472e2..e850809e 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -979,7 +979,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { .map(p => p.source) .getOrElse("") should include( """def param1 = ctx.birthdate; - |def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); + |def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); |ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin .replaceAll("\n", " ") ) @@ -992,16 +992,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(ct.ddlTable.ddlPipeline.ddl) val json = ct.ddlTable.ddlPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" val indexSettings = ct.ddlTable.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = ct.ddlTable.pipeline println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index bdfc660e..958219a2 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -41,7 +41,7 @@ import _root_.java.time.format.DateTimeFormatter import _root_.java.util.UUID import _root_.java.util.concurrent.TimeUnit import java.time.temporal.Temporal -import java.time.{LocalDate, LocalDateTime, ZoneOffset} +import java.time.{LocalDate, LocalDateTime, ZoneOffset, ZonedDateTime} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContextExecutor, Future} import scala.util.{Failure, Success, Try} @@ -58,6 +58,8 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M import ElasticProviders._ + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + lazy val pClient: ElasticProvider[Person] = new PersonProvider( elasticConfig ) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 9aee0e72..829c2db3 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -292,7 +292,7 @@ trait MockElasticClientApi extends ElasticClientApi { override private[client] implicit def sqlSearchRequestToJsonQuery( sqlSearch: SingleSearch - ): String = + )(implicit timestamp: Long): String = """{ | "query": { | "match_all": {} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala new file mode 100644 index 00000000..c91d7081 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala @@ -0,0 +1,638 @@ +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.slf4j.{Logger, LoggerFactory} + +import java.util.concurrent.TimeUnit +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContextExecutor} + +trait PipelineApiSpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: PipelineApi + + override def beforeAll(): Unit = { + self.beforeAll() + log.info("🚀 Starting PipelineApiSpec test suite") + } + + override def afterAll(): Unit = { + log.info("🏁 Finishing PipelineApiSpec test suite") + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + self.afterAll() + } + + override def beforeEach(): Unit = { + super.beforeEach() + // Cleanup: delete test pipelines before each test + cleanupTestPipelines() + } + + override def afterEach(): Unit = { + // Cleanup: delete test pipelines after each test + cleanupTestPipelines() + super.afterEach() + } + + private def cleanupTestPipelines(): Unit = { + val testPipelines = List( + "test_pipeline", + "user_pipeline", + "simple_pipeline", + "complex_pipeline", + "pipeline_to_update", + "pipeline_to_delete", + "pipeline_with_script", + "invalid_pipeline" + ) + + testPipelines.foreach { pipelineName => + client.deletePipeline(pipelineName, ifExists = false) match { + case _ => // Ignore result, just cleanup + } + } + } + + // ======================================================================== + // TEST: CREATE PIPELINE + // ======================================================================== + + "PipelineApi" should "create a simple pipeline with SET processor" in { + val pipelineDefinition = + """ + |{ + | "description": "Simple test pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "active" + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("simple_pipeline", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify pipeline was created + val getResult = client.getPipeline("simple_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + } + + it should "create a complex pipeline with multiple processors" in { + val pipelineDefinition = + """ + |{ + | "description": "Complex test pipeline", + | "processors": [ + | { + | "set": { + | "field": "name", + | "value": "anonymous", + | "if": "ctx.name == null" + | } + | }, + | { + | "lowercase": { + | "field": "status" + | } + | }, + | { + | "remove": { + | "field": "temp_field", + | "ignore_failure": true + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("complex_pipeline", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "create a pipeline with script processor" in { + val pipelineDefinition = + """ + |{ + | "description": "Pipeline with script", + | "processors": [ + | { + | "script": { + | "lang": "painless", + | "source": "ctx.age = ChronoUnit.YEARS.between(ZonedDateTime.parse(ctx.birthdate), ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')))" + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("pipeline_with_script", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "fail to create pipeline with invalid JSON" in { + val invalidPipelineDefinition = """{ "invalid": json }""" + + val result = client.createPipeline("invalid_pipeline", invalidPipelineDefinition) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "fail to create pipeline with empty name" in { + val pipelineDefinition = + """ + |{ + | "description": "Test", + | "processors": [] + |} + |""".stripMargin + + val result = client.createPipeline("", pipelineDefinition) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "fail to create pipeline with invalid name characters" in { + val pipelineDefinition = + """ + |{ + | "description": "Test", + | "processors": [] + |} + |""".stripMargin + + val result = client.createPipeline("invalid/pipeline*name", pipelineDefinition) + + result shouldBe a[ElasticFailure] + } + + // ======================================================================== + // TEST: UPDATE PIPELINE + // ======================================================================== + + it should "update an existing pipeline" in { + // First create a pipeline + val initialDefinition = + """ + |{ + | "description": "Initial pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "pending" + | } + | } + | ] + |} + |""".stripMargin + + client.createPipeline("pipeline_to_update", initialDefinition) + + // Now update it + val updatedDefinition = + """ + |{ + | "description": "Updated pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "active" + | } + | }, + | { + | "set": { + | "field": "updated", + | "value": true + | } + | } + | ] + |} + |""".stripMargin + + val result = client.updatePipeline("pipeline_to_update", updatedDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify the update + val getResult = client.getPipeline("pipeline_to_update") + getResult shouldBe a[ElasticSuccess[_]] + val pipelineJson = getResult.toOption.get.get + pipelineJson should include("Updated pipeline") + } + + // ======================================================================== + // TEST: GET PIPELINE + // ======================================================================== + + it should "retrieve an existing pipeline" in { + val pipelineDefinition = + """ + |{ + | "description": "Test retrieval", + | "processors": [ + | { + | "set": { + | "field": "test", + | "value": "value" + | } + | } + | ] + |} + |""".stripMargin + + client.createPipeline("test_pipeline", pipelineDefinition) + + val result = client.getPipeline("test_pipeline") + + result shouldBe a[ElasticSuccess[_]] + val maybePipeline = result.toOption.get + maybePipeline shouldBe defined + maybePipeline.get should include("Test retrieval") + } + + it should "return None when getting a non-existent pipeline" in { + val result = client.getPipeline("non_existent_pipeline") + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe None + } + + // ======================================================================== + // TEST: DELETE PIPELINE + // ======================================================================== + + it should "delete an existing pipeline" in { + val pipelineDefinition = + """ + |{ + | "description": "Pipeline to delete", + | "processors": [] + |} + |""".stripMargin + + client.createPipeline("pipeline_to_delete", pipelineDefinition) + + val deleteResult = client.deletePipeline("pipeline_to_delete", ifExists = false) + + deleteResult shouldBe a[ElasticSuccess[_]] + deleteResult.toOption.get shouldBe true + + // Verify deletion + val getResult = client.getPipeline("pipeline_to_delete") + getResult.toOption.get shouldBe None + } + + it should "handle deletion of non-existent pipeline gracefully" in { + val result = client.deletePipeline("non_existent_pipeline", ifExists = true) + + // Should succeed but return false or handle gracefully + result shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - CREATE PIPELINE + // ======================================================================== + + it should "execute CREATE PIPELINE DDL statement" in { + val sql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ) + |) + |""".stripMargin + + val result = client.pipeline(sql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify pipeline was created + val getResult = client.getPipeline("user_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + } + + it should "execute CREATE OR REPLACE PIPELINE DDL statement" in { + // First create + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "status", value = "pending") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then replace + val replaceSql = + """ + |CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "status", value = "active") + |) + |""".stripMargin + + val result = client.pipeline(replaceSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "execute CREATE PIPELINE IF NOT EXISTS DDL statement" in { + val sql = + """ + |CREATE PIPELINE IF NOT EXISTS user_pipeline WITH PROCESSORS ( + | SET (field = "test", value = "value") + |) + |""".stripMargin + + // First execution should create + val result1 = client.pipeline(sql) + result1 shouldBe a[ElasticSuccess[_]] + + // Second execution should not fail + val result2 = client.pipeline(sql) + result2 shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - DROP PIPELINE + // ======================================================================== + + it should "execute DROP PIPELINE DDL statement" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE test_pipeline WITH PROCESSORS ( + | SET (field = "test", value = "value") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then drop it + val dropSql = "DROP PIPELINE test_pipeline" + + val result = client.pipeline(dropSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify deletion + val getResult = client.getPipeline("test_pipeline") + getResult.toOption.get shouldBe None + } + + it should "execute DROP PIPELINE IF EXISTS DDL statement" in { + val sql = "DROP PIPELINE IF EXISTS non_existent_pipeline" + + val result = client.pipeline(sql) + + // Should not fail even if pipeline doesn't exist + result shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - ALTER PIPELINE + // ======================================================================== + + it should "execute ALTER PIPELINE with ADD PROCESSOR" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then alter it + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | value = "active" + | ) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify the pipeline was updated + val getResult = client.getPipeline("user_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + val pipelineJson = getResult.toOption.get.get + pipelineJson should include("status") + } + + it should "execute ALTER PIPELINE with DROP PROCESSOR" in { + // First create a pipeline with multiple processors + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous"), + | SET (field = "status", value = "active") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then drop one processor + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | DROP PROCESSOR SET (status) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "execute ALTER PIPELINE with multiple actions" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then alter with multiple actions + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | ADD PROCESSOR SET (field = "status", value = "active"), + | ADD PROCESSOR REMOVE (field = "temp_field", ignore_failure = true) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "fail to ALTER non-existent pipeline" in { + val alterSql = + """ + |ALTER PIPELINE non_existent_pipeline ( + | ADD PROCESSOR SET (field = "test", value = "value") + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(404) + error.message should include("not found") + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "execute ALTER PIPELINE IF EXISTS without error" in { + val alterSql = + """ + |ALTER PIPELINE IF EXISTS non_existent_pipeline ( + | ADD PROCESSOR SET (field = "test", value = "value") + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + // Should handle gracefully with IF EXISTS + result match { + case ElasticSuccess(_) => succeed + case ElasticFailure(error) => + error.statusCode shouldBe Some(404) + } + } + + // ======================================================================== + // TEST: ERROR HANDLING + // ======================================================================== + + it should "fail with invalid SQL syntax" in { + val invalidSql = "CREATE PIPELINE INVALID SYNTAX" + + val result = client.pipeline(invalidSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for invalid pipeline ddl statement") + } + } + + it should "fail with unsupported DDL statement" in { + val unsupportedSql = "SELECT * FROM pipelines" + + val result = client.pipeline(unsupportedSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for unsupported pipeline ddl statement") + } + } + + // ======================================================================== + // TEST: INTEGRATION WITH INDEX + // ======================================================================== + + it should "create pipeline and use it with index" in { + // Create an index with pipeline + createIndex("test_index_with_pipeline") + + // Create a pipeline + val pipelineDefinition = + """ + |{ + | "description": "Pipeline for test index", + | "processors": [ + | { + | "set": { + | "field": "indexed_at", + | "value": "{{_ingest.timestamp}}" + | } + | } + | ] + |} + |""".stripMargin + + val createResult = client.createPipeline("test_index_pipeline", pipelineDefinition) + createResult shouldBe a[ElasticSuccess[_]] + + // Verify pipeline exists + val getResult = client.getPipeline("test_index_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + + // Cleanup + deleteIndex("test_index_with_pipeline") + client.deletePipeline("test_index_pipeline", ifExists = false) + } +} From 4558dbf648e4697effb44bf6e61ba91cabb1aafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Dec 2025 07:56:25 +0100 Subject: [PATCH 26/95] add support for template api --- .../elastic/client/ElasticClientApi.scala | 5 + .../client/ElasticClientDelegator.scala | 123 +++++- .../elastic/client/ElasticClientHelpers.scala | 223 +++++++++- .../elastic/client/ElasticsearchVersion.scala | 24 ++ .../elastic/client/TemplateApi.scala | 273 ++++++++++++ .../elastic/client/TemplateConverter.scala | 215 ++++++++++ .../client/metrics/MetricsElasticClient.scala | 103 ++++- .../client/TemplateConverterSpec.scala | 79 ++++ .../elastic/client/jest/JestClientApi.scala | 1 + .../elastic/client/jest/JestTemplateApi.scala | 163 +++++++ .../client/jest/actions/Template.scala | 124 ++++++ .../client/JestClientTemplateApiSpec.scala | 8 + .../client/rest/RestHighLevelClientApi.scala | 275 ++++++++++-- .../RestHighLevelClientTemplateApiSpec.scala | 9 + .../client/rest/RestHighLevelClientApi.scala | 401 ++++++++++++++++-- .../RestHighLevelClientTemplateApiSpec.scala | 9 + .../elastic/client/java/JavaClientApi.scala | 337 ++++++++++++--- .../client/java/JavaClientCompanion.scala | 4 - .../client/java/JavaClientConversion.scala | 3 +- .../client/java/JavaClientHelpers.scala | 7 +- .../client/JavaClientTemplateApiSpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 339 ++++++++++++--- .../client/java/JavaClientCompanion.scala | 4 - .../client/java/JavaClientConversion.scala | 3 +- .../client/java/JavaClientHelpers.scala | 7 +- .../client/JavaClientTemplateApiSpec.scala | 8 + .../elastic/sql/query/package.scala | 8 +- .../elastic/sql/serialization/package.scala | 4 +- .../elastic/client/PipelineApiSpec.scala | 2 +- .../elastic/client/TemplateApiSpec.scala | 134 ++++++ 30 files changed, 2698 insertions(+), 205 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala create mode 100644 core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 6a0ff2e4..c2aad4f5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -17,6 +17,8 @@ package app.softnetwork.elastic.client import app.softnetwork.common.ClientCompanion +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper import com.typesafe.config.{Config, ConfigFactory} import org.json4s.jackson import org.json4s.jackson.Serialization @@ -46,6 +48,7 @@ trait ElasticClientApi with VersionApi with SerializationApi with PipelineApi + with TemplateApi with ClientCompanion { protected def logger: Logger @@ -57,4 +60,6 @@ trait ElasticClientApi trait SerializationApi { implicit val serialization: Serialization.type = jackson.Serialization + val mapper: ObjectMapper = JacksonConfig.objectMapper + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 8af090a3..668bce63 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -1407,7 +1407,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = delegate.actionToBulkItem(action.asInstanceOf) - // ==================== PipelineApi (délégation) ==================== + // ==================== PipelineApi (delegate) ==================== override def createPipeline( pipelineName: String, @@ -1440,4 +1440,125 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { pipelineName: String ): ElasticResult[Option[String]] = delegate.executeGetPipeline(pipelineName) + + // ==================== TemplateApi (delegate) ==================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + override def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.createTemplate(templateName, templateDefinition) + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + override def deleteTemplate(templateName: String, ifExists: Boolean): ElasticResult[Boolean] = + delegate.deleteTemplate(templateName, ifExists) + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + override def getTemplate(templateName: String): ElasticResult[Option[String]] = + delegate.getTemplate(templateName) + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + override def listTemplates(): ElasticResult[Map[String, String]] = + delegate.listTemplates() + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + override def templateExists(templateName: String): ElasticResult[Boolean] = + delegate.templateExists(templateName) + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreateComposableTemplate(templateName, templateDefinition) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteComposableTemplate(templateName, ifExists) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + delegate.executeGetComposableTemplate(templateName) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + delegate.executeListComposableTemplates() + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + delegate.executeComposableTemplateExists(templateName) + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreateLegacyTemplate(templateName, templateDefinition) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteLegacyTemplate(templateName, ifExists) + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = + delegate.executeGetLegacyTemplate(templateName) + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = + delegate.executeListLegacyTemplates() + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + delegate.executeLegacyTemplateExists(templateName) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 96ea2cec..4d28b8f6 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -33,6 +33,7 @@ trait ElasticClientHelpers { * - No colon (:) except for system indexes * - Does not start with -, _, + * - Is not . or .. + * - Max length 255 characters * @param index * name of the index to validate * @return @@ -208,7 +209,8 @@ trait ElasticClientHelpers { case Some(error) => Some( error.copy( - operation = Some("validateAliasName") + operation = Some("validateAliasName"), + message = error.message.replaceAll("Index", "Alias") ) ) case None => None @@ -253,6 +255,225 @@ trait ElasticClientHelpers { None } + /** Validate the template name. Index templates follow the same rules as indexes: + * - Not empty + * - Lowercase only + * - No characters: \, /, *, ?, ", <, >, |, space, comma, # + * - No colon (:) + * - Does not start with -, _, + + * - Is not . or .. + * - Max length 255 characters + * - Can contain wildcards (* and ?) in index_patterns but not in the template name itself + * + * @param templateName + * name of the template to validate + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateTemplateName(templateName: String): Option[ElasticError] = { + validateIndexName(templateName) match { + case Some(error) => + Some( + error.copy( + operation = Some("validateTemplateName"), + message = error.message.replaceAll("Index", "Template") + ) + ) + case None => None + } + } + + /** Validate composable template JSON definition (ES 7.8+) */ + protected def validateJsonComposableTemplate( + templateDefinition: String + ): Option[result.ElasticError] = { + if (templateDefinition == null || templateDefinition.trim.isEmpty) { + return Some( + result.ElasticError( + message = "Template definition cannot be null or empty", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + try { + import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + import com.fasterxml.jackson.databind.node.ObjectNode + + val mapper = new ObjectMapper() + val rootNode: JsonNode = mapper.readTree(templateDefinition) + + if (!rootNode.isObject) { + return Some( + result.ElasticError( + message = "Template definition must be a JSON object", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + val objectNode = rootNode.asInstanceOf[ObjectNode] + + // Composable templates require 'index_patterns' + if (!objectNode.has("index_patterns")) { + return Some( + result.ElasticError( + message = "Composable template must contain 'index_patterns' field", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + val indexPatternsNode = objectNode.get("index_patterns") + if (!indexPatternsNode.isArray || indexPatternsNode.size() == 0) { + return Some( + result.ElasticError( + message = "'index_patterns' must be a non-empty array", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + // Valid fields for composable templates + val validFields = Set( + "index_patterns", + "template", // Contains settings, mappings, aliases + "priority", // Replaces 'order' + "version", + "composed_of", // Component templates + "_meta", + "data_stream", // For data streams + "allow_auto_create" + ) + + import scala.jdk.CollectionConverters._ + val templateFields = objectNode.fieldNames().asScala.toSet + val invalidFields = templateFields -- validFields + + if (invalidFields.nonEmpty) { + logger.warn( + s"⚠️ Composable template contains potentially invalid fields: ${invalidFields.mkString(", ")}" + ) + } + + None + + } catch { + case e: com.fasterxml.jackson.core.JsonParseException => + Some( + result.ElasticError( + message = s"Invalid JSON syntax: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate"), + cause = Some(e) + ) + ) + case e: Exception => + Some( + result.ElasticError( + message = s"Invalid composable template definition: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate"), + cause = Some(e) + ) + ) + } + } + + /** Validate legacy template JSON definition (ES < 7.8) */ + protected def validateJsonLegacyTemplate( + templateDefinition: String + ): Option[result.ElasticError] = { + if (templateDefinition == null || templateDefinition.trim.isEmpty) { + return Some( + result.ElasticError( + message = "Template definition cannot be null or empty", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + try { + import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + import com.fasterxml.jackson.databind.node.ObjectNode + + val mapper = new ObjectMapper() + val rootNode: JsonNode = mapper.readTree(templateDefinition) + + if (!rootNode.isObject) { + return Some( + result.ElasticError( + message = "Template definition must be a JSON object", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + val objectNode = rootNode.asInstanceOf[ObjectNode] + + // Legacy templates require 'index_patterns' or 'template' + if (!objectNode.has("index_patterns") && !objectNode.has("template")) { + return Some( + result.ElasticError( + message = "Legacy template must contain 'index_patterns' or 'template' field", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + // Valid fields for legacy templates + val validFields = Set( + "index_patterns", + "template", // Old name for index_patterns + "settings", + "mappings", + "aliases", + "order", + "version", + "_meta" + ) + + import scala.jdk.CollectionConverters._ + val templateFields = objectNode.fieldNames().asScala.toSet + val invalidFields = templateFields -- validFields + + if (invalidFields.nonEmpty) { + logger.warn( + s"⚠️ Legacy template contains potentially invalid fields: ${invalidFields.mkString(", ")}" + ) + } + + None + + } catch { + case e: com.fasterxml.jackson.core.JsonParseException => + Some( + result.ElasticError( + message = s"Invalid JSON syntax: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate"), + cause = Some(e) + ) + ) + case e: Exception => + Some( + result.ElasticError( + message = s"Invalid legacy template definition: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate"), + cause = Some(e) + ) + ) + } + } + /** Logger une erreur avec le niveau approprié selon le status code. */ protected def logError( operation: String, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index c7f2eee0..64ef818d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -21,6 +21,7 @@ package app.softnetwork.elastic.client object ElasticsearchVersion { /** Parse Elasticsearch version string (e.g., "7.10.2", "8.11.0") + * * @return * (major, minor, patch) */ @@ -70,12 +71,35 @@ object ElasticsearchVersion { isAtLeast(version, 8) } + /** Check if version is ES 7+ + */ def isEs7OrHigher(version: String): Boolean = { isAtLeast(version, 7) } + /** Check if version is ES 6 + */ def isEs6(version: String): Boolean = { val (major, _, _) = parse(version) major == 6 } + + /** Check if Data Streams are supported (ES >= 7.9) + */ + def supportsDataStreams(version: String): Boolean = { + isAtLeast(version, 7, 9) + } + + /** Check if Composable Templates are supported (ES >= 7.8) + */ + def supportsComposableTemplates(version: String): Boolean = { + isAtLeast(version, 7, 8) + } + + /** Check if Searchable Snapshots are supported (ES >= 7.10)} + */ + def supportsSearchableSnapshots(version: String): Boolean = { + isAtLeast(version, 7, 10) + } + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala new file mode 100644 index 00000000..5ff51483 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala @@ -0,0 +1,273 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} + +import scala.util.{Failure, Success} + +/** API for managing Elasticsearch index templates. + * + * Automatically handles format conversion: + * - Legacy format → Composable format (ES 7.8+) + * - Composable format → Legacy format (ES < 7.8) + */ +trait TemplateApi extends ElasticClientHelpers { _: VersionApi => + + // ======================================================================== + // TEMPLATE API + // ======================================================================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + // Normalize template format based on ES version + val normalizedTemplate = TemplateConverter.normalizeTemplate( + templateDefinition, + elasticVersion + ) match { + case Success(json) => json + case Failure(ex) => + logger.error(s"❌ Failed to normalize template format: ${ex.getMessage}", ex) + return ElasticFailure( + ElasticError( + message = s"Invalid template format: ${ex.getMessage}", + statusCode = Some(400), + operation = Some("createTemplate"), + cause = Some(ex) + ) + ) + } + + // Determine which API to use + val isLegacyFormat = TemplateConverter.isLegacyFormat(templateDefinition) + val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(elasticVersion) + + if (supportsComposable) { + logger.info( + s"✅ Using composable template API (ES $elasticVersion)" + + (if (isLegacyFormat) " [converted from legacy format]" else "") + ) + + validateJsonComposableTemplate(normalizedTemplate) match { + case Some(error) => ElasticFailure(error) + case None => executeCreateComposableTemplate(templateName, normalizedTemplate) + } + } else { + logger.info(s"⚠️ Using legacy template API (ES $elasticVersion < 7.8)") + + validateJsonLegacyTemplate(normalizedTemplate) match { + case Some(error) => ElasticFailure(error) + case None => executeCreateLegacyTemplate(templateName, normalizedTemplate) + } + } + } + } + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + def deleteTemplate( + templateName: String, + ifExists: Boolean = false + ): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + logger.debug(s"Using composable template API for deletion (ES $elasticVersion)") + executeDeleteComposableTemplate(templateName, ifExists) + } else { + logger.debug(s"Using legacy template API for deletion (ES $elasticVersion < 7.8)") + executeDeleteLegacyTemplate(templateName, ifExists) + } + } + } + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + def getTemplate(templateName: String): ElasticResult[Option[String]] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeGetComposableTemplate(templateName) + } else { + executeGetLegacyTemplate(templateName) + } + } + } + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + def listTemplates(): ElasticResult[Map[String, String]] = { + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeListComposableTemplates() + } else { + executeListLegacyTemplates() + } + } + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + def templateExists(templateName: String): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeComposableTemplateExists(templateName) + } else { + executeLegacyTemplateExists(templateName) + } + } + } + + // ==================== ABSTRACT METHODS - COMPOSABLE (ES 7.8+) ==================== + + private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] + + private[client] def executeListComposableTemplates(): ElasticResult[Map[String, String]] + + private[client] def executeComposableTemplateExists(templateName: String): ElasticResult[Boolean] + + // ==================== ABSTRACT METHODS - LEGACY (ES < 7.8) ==================== + + private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetLegacyTemplate(templateName: String): ElasticResult[Option[String]] + + private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] + + private[client] def executeLegacyTemplateExists(templateName: String): ElasticResult[Boolean] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala new file mode 100644 index 00000000..422837e7 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala @@ -0,0 +1,215 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.node.ObjectNode + +import scala.util.{Success, Try} + +object TemplateConverter { + + private lazy val mapper = JacksonConfig.objectMapper + + /** Detect if a template JSON is in legacy format + * + * Legacy format has settings/mappings/aliases at root level Composable format has them nested + * under 'template' key + * + * @param templateJson + * the template JSON string + * @return + * true if legacy format + */ + def isLegacyFormat(templateJson: String): Boolean = { + Try { + val root = mapper.readTree(templateJson) + + // Legacy format: has settings/mappings/aliases at root + val hasRootSettings = root.has("settings") + val hasRootMappings = root.has("mappings") + val hasRootAliases = root.has("aliases") + + // Composable format: has 'template' key containing settings/mappings/aliases + val hasTemplateKey = root.has("template") + + // Legacy if has root-level settings/mappings/aliases AND no 'template' key + (hasRootSettings || hasRootMappings || hasRootAliases) && !hasTemplateKey + + }.getOrElse(false) + } + + /** Convert legacy template format to composable template format + * + * Legacy format: + * {{{ + * { + * "index_patterns": ["logs-*"], + * "order": 1, + * "settings": {...}, + * "mappings": {...}, + * "aliases": {...} + * } + * }}} + * + * Composable format: + * {{{ + * { + * "index_patterns": ["logs-*"], + * "priority": 1, + * "template": { + * "settings": {...}, + * "mappings": {...}, + * "aliases": {...} + * } + * } + * }}} + * + * @param legacyJson + * the legacy template JSON + * @return + * composable template JSON + */ + def convertLegacyToComposable(legacyJson: String): Try[String] = { + Try { + val root = mapper.readTree(legacyJson).asInstanceOf[ObjectNode] + val composable = mapper.createObjectNode() + + // 1. Copy index_patterns (required) + if (root.has("index_patterns")) { + composable.set("index_patterns", root.get("index_patterns")) + } else if (root.has("template")) { + // Old legacy format used 'template' instead of 'index_patterns' + val patterns = mapper.createArrayNode() + patterns.add(root.get("template").asText()) + composable.set("index_patterns", patterns) + } + + // 2. Convert 'order' to 'priority' + if (root.has("order")) { + composable.put("priority", root.get("order").asInt()) + } + + // 3. Copy version if present + if (root.has("version")) { + composable.set("version", root.get("version")) + } + + // 4. Copy _meta if present + if (root.has("_meta")) { + composable.set("_meta", root.get("_meta")) + } + + // 5. Create nested 'template' object with settings/mappings/aliases + val templateNode = mapper.createObjectNode() + + if (root.has("settings")) { + templateNode.set("settings", root.get("settings")) + } + + if (root.has("mappings")) { + templateNode.set("mappings", root.get("mappings")) + } + + if (root.has("aliases")) { + templateNode.set("aliases", root.get("aliases")) + } + + // Only add 'template' key if it has content + if (templateNode.size() > 0) { + composable.set("template", templateNode) + } + + mapper.writeValueAsString(composable) + } + } + + /** Convert composable template format to legacy template format + * + * Used for backward compatibility when ES version < 7.8 + * + * @param composableJson + * the composable template JSON + * @return + * legacy template JSON + */ + def convertComposableToLegacy(composableJson: String): Try[String] = { + Try { + val root = mapper.readTree(composableJson).asInstanceOf[ObjectNode] + val legacy = mapper.createObjectNode() + + // 1. Copy index_patterns (required) + if (root.has("index_patterns")) { + legacy.set("index_patterns", root.get("index_patterns")) + } + + // 2. Convert 'priority' to 'order' + if (root.has("priority")) { + legacy.put("order", root.get("priority").asInt()) + } + + // 3. Copy version if present + if (root.has("version")) { + legacy.set("version", root.get("version")) + } + + // 4. Copy _meta if present + if (root.has("_meta")) { + legacy.set("_meta", root.get("_meta")) + } + + // 5. Flatten 'template' object to root level + if (root.has("template")) { + val templateNode = root.get("template") + + if (templateNode.has("settings")) { + legacy.set("settings", templateNode.get("settings")) + } + + if (templateNode.has("mappings")) { + legacy.set("mappings", templateNode.get("mappings")) + } + + if (templateNode.has("aliases")) { + legacy.set("aliases", templateNode.get("aliases")) + } + } + + // Note: 'composed_of' and 'data_stream' are not supported in legacy format + // They will be silently ignored + + mapper.writeValueAsString(legacy) + } + } + + /** Validate and normalize template JSON based on ES version + * + * @param templateJson + * the template JSON (legacy or composable format) + * @param esVersion + * the Elasticsearch version + * @return + * normalized template JSON appropriate for the ES version + */ + def normalizeTemplate(templateJson: String, esVersion: String): Try[String] = { + val isLegacy = isLegacyFormat(templateJson) + val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(esVersion) + + (isLegacy, supportsComposable) match { + case (true, true) => + // Legacy format + ES 7.8+ → Convert to composable + convertLegacyToComposable(templateJson) + + case (false, false) => + // Composable format + ES < 7.8 → Convert to legacy + convertComposableToLegacy(templateJson) + + case (true, false) => + // Legacy format + ES < 7.8 → Keep as is + Success(templateJson) + + case (false, true) => + // Composable format + ES 7.8+ → Keep as is + Success(templateJson) + } + } + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 04ee3c8a..4d7eeacb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -1078,20 +1078,103 @@ class MetricsElasticClient( metricsCollector.resetMetrics() } - // ==================== PipelineApi (délégation) ==================== + // ==================== PipelineApi (delegate) ==================== override def createPipeline( pipelineName: String, pipelineDefinition: String - ): ElasticResult[Boolean] = { - delegate.createPipeline(pipelineName, pipelineDefinition) - } + ): ElasticResult[Boolean] = + measureResult("createPipeline") { + delegate.createPipeline(pipelineName, pipelineDefinition) + } - override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { - delegate.deletePipeline(pipelineName, ifExists = ifExists) - } + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = + measureResult("deletePipeline") { + delegate.deletePipeline(pipelineName, ifExists = ifExists) + } - override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { - delegate.getPipeline(pipelineName) - } + override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = + measureResult("getPipeline") { + delegate.getPipeline(pipelineName) + } + + // ==================== TemplateApi (delegate) ==================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + override def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + measureResult("createTemplate") { + delegate.createTemplate(templateName, templateDefinition) + } + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + override def deleteTemplate(templateName: String, ifExists: Boolean): ElasticResult[Boolean] = + measureResult("deleteTemplate") { + delegate.deleteTemplate(templateName, ifExists) + } + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + override def getTemplate(templateName: String): ElasticResult[Option[String]] = + measureResult("getTemplate") { + delegate.getTemplate(templateName) + } + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + override def listTemplates(): ElasticResult[Map[String, String]] = + measureResult("listTemplates") { + delegate.listTemplates() + } + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + override def templateExists(templateName: String): ElasticResult[Boolean] = + measureResult("templateExists") { + delegate.templateExists(templateName) + } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala new file mode 100644 index 00000000..f86f9223 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala @@ -0,0 +1,79 @@ +package app.softnetwork.elastic.client + +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TemplateConverterSpec extends AnyFlatSpec with Matchers { + + "isLegacyFormat" should "detect legacy format" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + TemplateConverter.isLegacyFormat(legacy) shouldBe true + } + + it should "detect composable format" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + TemplateConverter.isLegacyFormat(composable) shouldBe false + } + + "convertLegacyToComposable" should "convert legacy to composable" in { + val legacy = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": {"number_of_shards": 1}, + | "mappings": {"properties": {"timestamp": {"type": "date"}}} + |} + |""".stripMargin + + val result = TemplateConverter.convertLegacyToComposable(legacy) + result.isSuccess shouldBe true + + val composable = new ObjectMapper().readTree(result.get) + composable.has("priority") shouldBe true + composable.get("priority").asInt() shouldBe 1 + composable.has("template") shouldBe true + composable.get("template").has("settings") shouldBe true + composable.get("template").has("mappings") shouldBe true + } + + "normalizeTemplate" should "convert legacy to composable for ES 7.8+" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + val result = TemplateConverter.normalizeTemplate(legacy, "7.10.0") + + result.isSuccess shouldBe true + val normalized = new ObjectMapper().readTree(result.get) + normalized.has("priority") shouldBe true + normalized.has("template") shouldBe true + } + + it should "convert composable to legacy for ES < 7.8" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + val result = TemplateConverter.normalizeTemplate(composable, "7.5.0") + + result.isSuccess shouldBe true + val normalized = new ObjectMapper().readTree(result.get) + normalized.has("order") shouldBe true + normalized.has("settings") shouldBe true + normalized.has("template") shouldBe false + } + + it should "keep legacy format for ES < 7.8" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + val result = TemplateConverter.normalizeTemplate(legacy, "7.5.0") + + result.isSuccess shouldBe true + result.get should include("order") + result.get should not include "priority" + } + + it should "keep composable format for ES 7.8+" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + val result = TemplateConverter.normalizeTemplate(composable, "7.10.0") + + result.isSuccess shouldBe true + result.get should include("priority") + result.get should include("template") + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index a92d4072..9a31eaa1 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -45,6 +45,7 @@ trait JestClientApi with JestBulkApi with JestVersionApi with JestPipelineApi + with JestTemplateApi with JestClientCompanion object JestClientApi extends SerializationApi { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala new file mode 100644 index 00000000..9855c941 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala @@ -0,0 +1,163 @@ +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.jest.actions.Template +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.client.{SerializationApi, TemplateApi} +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import io.searchbox.client.JestResult + +import scala.jdk.CollectionConverters._ + +trait JestTemplateApi extends TemplateApi with JestClientHelpers with JestVersionApi { + _: SerializationApi with JestClientCompanion => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = ElasticSuccess(None) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = ElasticSuccess(Map.empty[String, String]) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + apply().execute(Template.Create(templateName, templateDefinition)) match { + case jestResult: JestResult if jestResult.isSucceeded => + ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to create template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + apply().execute(Template.Delete(templateName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to delete template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + apply().execute(Template.Get(templateName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode if objectNode.has(templateName) => + val templateNode = objectNode.get(templateName) + ElasticSuccess(Some(templateNode)) + case _ => + ElasticSuccess(None) + } + } else { + ElasticSuccess(None) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to get template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + apply().execute(Template.GetAll()) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode => + val templates = objectNode + .fields() + .asScala + .map { entry => + entry.getKey -> entry.getValue.toString + } + .toMap + ElasticSuccess(templates) + case _ => + ElasticSuccess(Map.empty[String, String]) + } + } else { + ElasticSuccess(Map.empty[String, String]) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to list templates: $errorMessage" + ) + ) + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + apply().execute(Template.Exists(templateName)) match { + case jestResult: JestResult => + val statusCode = jestResult.getResponseCode + ElasticSuccess(statusCode == 200) + case _ => + ElasticSuccess(false) + } + } + +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala new file mode 100644 index 00000000..24036909 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala @@ -0,0 +1,124 @@ +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.client.config.ElasticsearchVersion + +object Template { + + import io.searchbox.action.AbstractAction + import io.searchbox.client.JestResult + import com.google.gson.Gson + + case class Create(templateName: String, json: String) extends AbstractAction[JestResult] { + + payload = json + + override def getRestMethodName: String = "PUT" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Get(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201 || statusCode == 404) + if (statusCode != 404) { + Option(json).foreach(result.setJsonString) + } + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class GetAll() extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + "/_template" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Delete(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "DELETE" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Exists(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "HEAD" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 404) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala new file mode 100644 index 00000000..c8e34ff2 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class JestClientTemplateApiSpec extends TemplateApiSpec with EmbeddedElasticTestKit { + override def client: TemplateApi with VersionApi = new JestClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 2c2baa68..2fa1741f 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,10 +21,10 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.{Alias, IndicesAliasesRequest} @@ -37,6 +37,7 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.admin.indices.refresh.{RefreshRequest, RefreshResponse} import org.elasticsearch.action.admin.indices.settings.get.{GetSettingsRequest, GetSettingsResponse} import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} @@ -64,7 +65,12 @@ import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CreateIndexRequest, GetIndexRequest, + GetIndexTemplatesRequest, + GetIndexTemplatesResponse, GetMappingsRequest, + IndexTemplateMetaData, + IndexTemplatesExistRequest, + PutIndexTemplateRequest, PutMappingRequest } import org.elasticsearch.common.Strings @@ -102,6 +108,7 @@ trait RestHighLevelClientApi with RestHighLevelClientCompanion with RestHighLevelClientVersion with RestHighLevelClientPipelineApi + with RestHighLevelClientTemplateApi /** Version API implementation for RestHighLevelClient * @see @@ -109,7 +116,7 @@ trait RestHighLevelClientApi */ trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeRestLowLevelAction[String]( operation = "version", index = None, @@ -137,7 +144,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH settings: String, mappings: Option[String], aliases: Seq[String] - ): result.ElasticResult[Boolean] = { + ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), @@ -152,7 +159,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH ) } - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", index = Some(index), @@ -163,7 +170,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().delete(req, RequestOptions.DEFAULT) ) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[CloseIndexRequest, AcknowledgedResponse]( operation = "closeIndex", index = Some(index), @@ -174,7 +181,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().close(req, RequestOptions.DEFAULT) ) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[OpenIndexRequest, AcknowledgedResponse]( operation = "openIndex", index = Some(index), @@ -190,7 +197,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH targetIndex: String, refresh: Boolean, pipeline: Option[String] - ): result.ElasticResult[(Boolean, Option[Long])] = + ): ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", index = Some(s"$sourceIndex->$targetIndex"), @@ -219,8 +226,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH case statusLine if statusLine.getStatusCode >= 400 => (false, None) case _ => - val json = new JsonParser() - .parse( + val json = JsonParser + .parseString( scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString ) .getAsJsonObject @@ -232,7 +239,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } }) - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeRestAction[GetIndexRequest, Boolean, Boolean]( operation = "indexExists", index = Some(index), @@ -257,7 +264,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeAddAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", index = Some(index), @@ -276,7 +283,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "removeAlias", index = Some(index), @@ -292,7 +299,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeRestAction[GetAliasesRequest, GetAliasesResponse, Boolean]( operation = "aliasExists", index = Some(alias), @@ -303,7 +310,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().getAlias(req, RequestOptions.DEFAULT) )(response => !response.getAliases.isEmpty) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeRestAction[GetAliasesRequest, GetAliasesResponse, String]( operation = "getAliases", index = Some(index), @@ -318,7 +325,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex -> $newIndex"), @@ -351,7 +358,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "updateSettings", index = Some(index), @@ -363,7 +370,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien executor = req => apply().indices().putSettings(req, RequestOptions.DEFAULT) ) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeRestAction[GetSettingsRequest, GetSettingsResponse, String]( operation = "loadSettings", index = Some(index), @@ -385,7 +392,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "setMapping", index = Some(index), @@ -397,7 +404,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH executor = req => apply().indices().putMapping(req, RequestOptions.DEFAULT) ) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeRestAction[ GetMappingsRequest, org.elasticsearch.client.indices.GetMappingsResponse, @@ -426,7 +433,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH */ trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeRestAction[RefreshRequest, RefreshResponse, Boolean]( operation = "refresh", index = Some(index), @@ -449,7 +456,7 @@ trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientHelpe index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[FlushRequest, FlushResponse, Boolean]( operation = "flush", index = Some(index), @@ -470,7 +477,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe _: RestHighLevelClientCompanion => override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeRestAction[CountRequest, CountResponse, Option[Double]]( operation = "count", index = Some(query.indices.mkString(",")), @@ -483,7 +490,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = { + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = { executeAsyncRestAction[CountRequest, CountResponse, Option[Double]]( operation = "countAsync", index = Some(query.indices.mkString(",")), @@ -508,7 +515,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[IndexRequest, IndexResponse, Boolean]( operation = "index", index = Some(index), @@ -540,7 +547,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[IndexRequest, IndexResponse, Boolean]( operation = "indexAsync", index = Some(index), @@ -579,7 +586,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "update", index = Some(index), @@ -613,7 +620,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = executeAsyncRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "updateAsync", index = Some(index), @@ -654,7 +661,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "delete", index = Some(index), @@ -676,7 +683,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "deleteAsync", index = Some(index), @@ -707,7 +714,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[GetRequest, GetResponse, Option[String]]( operation = "get", index = Some(index), @@ -726,7 +733,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[GetRequest, GetResponse, Option[String]]( operation = "getAsync", index = Some(index), @@ -759,7 +766,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -789,7 +796,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "multiSearch", index = Some( @@ -825,7 +832,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "executeSingleSearchAsync", index = Some(elasticQuery.indices.mkString(",")), @@ -855,7 +862,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "executeMultiSearchAsync", index = Some( @@ -1412,7 +1419,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeCreatePipeline( pipelineName: String, pipelineDefinition: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( operation = "createPipeline", retryable = false @@ -1429,7 +1436,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeDeletePipeline( pipelineName: String, ifExists: Boolean - ): result.ElasticResult[Boolean] = { + ): ElasticResult[Boolean] = { executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( operation = "deletePipeline", retryable = false @@ -1442,7 +1449,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeGetPipeline( pipelineName: JSONQuery - ): result.ElasticResult[Option[JSONQuery]] = { + ): ElasticResult[Option[JSONQuery]] = { executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( operation = "getPipeline", retryable = true @@ -1456,7 +1463,6 @@ trait RestHighLevelClientPipelineApi if (pipelines.nonEmpty) { val pipeline = pipelines.head val config = pipeline.getConfigAsMap - val mapper = JacksonConfig.objectMapper Some(mapper.writeValueAsString(config)) } else { None @@ -1465,3 +1471,194 @@ trait RestHighLevelClientPipelineApi ) } } + +trait RestHighLevelClientTemplateApi + extends TemplateApi + with RestHighLevelClientHelpers + with RestHighLevelClientVersion { + _: RestHighLevelClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = ElasticSuccess(None) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = ElasticSuccess(Map.empty[String, String]) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Option[String]]( + operation = "getTemplate", + retryable = true + )( + request = new GetIndexTemplatesRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates.asScala + if (templates.nonEmpty) { + val template = templates.head + Some(legacyTemplateToJson(template)) + } else { + None + } + } + ) + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Map[String, String]]( + operation = "listTemplates", + retryable = true + )( + request = new GetIndexTemplatesRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + resp.getIndexTemplates.asScala.map { template => + template.name() -> legacyTemplateToJson(template) + }.toMap + } + ) + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[IndexTemplatesExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new IndexTemplatesExistRequest(templateName) + )( + executor = req => apply().indices().existsTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== CONVERSION HELPERS ==================== + + private def legacyTemplateToJson(template: IndexTemplateMetaData): String = { + + val root = mapper.createObjectNode() + + // index_patterns + val patternsArray = mapper.createArrayNode() + template.patterns().asScala.foreach(patternsArray.add) + root.set("index_patterns", patternsArray) + + // order + root.put("order", template.order()) + + // version + if (template.version() != null) { + root.put("version", template.version()) + } + + // settings + if (template.settings() != null && !template.settings().isEmpty) { + val settingsNode = mapper.createObjectNode() + template.settings().keySet().asScala.foreach { key => + val value = template.settings().get(key) + if (value != null) { + settingsNode.put(key, value) + } + } + root.set("settings", settingsNode) + } + + // mappings + if (template.mappings() != null && template.mappings().source().string().nonEmpty) { + try { + val mappingsNode = mapper.readTree(template.mappings().source().string()) + root.set("mappings", mappingsNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse mappings: ${e.getMessage}") + } + } + + // aliases + if (template.aliases() != null && !template.aliases().isEmpty) { + val aliasesNode = mapper.createObjectNode() + template.aliases().asScala.foreach { alias => + val aliasName = alias.key + try { + val aliasNode = mapper.readTree(alias.value.toString) + aliasesNode.set(alias.key, aliasNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse alias '$aliasName': ${e.getMessage}") + aliasesNode.set(aliasName, mapper.createObjectNode()) + } + } + root.set("aliases", aliasesNode) + } + + mapper.writeValueAsString(root) + } + +} diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala new file mode 100644 index 00000000..c2a90a24 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with EmbeddedElasticTestKit { + override lazy val client: TemplateApi with VersionApi = + new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index be3a2ff0..1a57564c 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,6 +21,12 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} @@ -37,6 +43,7 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.admin.indices.refresh.{RefreshRequest, RefreshResponse} import org.elasticsearch.action.admin.indices.settings.get.{GetSettingsRequest, GetSettingsResponse} import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} @@ -65,15 +72,25 @@ import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CloseIndexRequest, + ComposableIndexTemplateExistRequest, CreateIndexRequest, + DeleteComposableIndexTemplateRequest, + GetComposableIndexTemplateRequest, GetIndexRequest, + GetIndexTemplatesRequest, + GetIndexTemplatesResponse, GetMappingsRequest, + IndexTemplateMetadata, + IndexTemplatesExistRequest, + PutComposableIndexTemplateRequest, + PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate import org.elasticsearch.common.Strings import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue -import org.elasticsearch.xcontent.{DeprecationHandler, XContentType} +import org.elasticsearch.xcontent.{DeprecationHandler, ToXContent, XContentFactory, XContentType} import org.elasticsearch.rest.RestStatus import org.elasticsearch.search.builder.{PointInTimeBuilder, SearchSourceBuilder} import org.elasticsearch.search.sort.{FieldSortBuilder, SortOrder} @@ -105,6 +122,7 @@ trait RestHighLevelClientApi with RestHighLevelClientCompanion with RestHighLevelClientVersion with RestHighLevelClientPipelineApi + with RestHighLevelClientTemplateApi /** Version API implementation for RestHighLevelClient * @see @@ -113,7 +131,7 @@ trait RestHighLevelClientApi trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeRestLowLevelAction[String]( operation = "version", index = None, @@ -143,7 +161,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH settings: String, mappings: Option[String], aliases: Seq[String] - ): result.ElasticResult[Boolean] = { + ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), @@ -158,7 +176,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH ) } - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", index = Some(index), @@ -169,7 +187,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().delete(req, RequestOptions.DEFAULT) ) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[CloseIndexRequest, AcknowledgedResponse]( operation = "closeIndex", index = Some(index), @@ -180,7 +198,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().close(req, RequestOptions.DEFAULT) ) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[OpenIndexRequest, AcknowledgedResponse]( operation = "openIndex", index = Some(index), @@ -196,7 +214,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH targetIndex: String, refresh: Boolean, pipeline: Option[String] - ): result.ElasticResult[(Boolean, Option[Long])] = + ): ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", index = Some(s"$sourceIndex->$targetIndex"), @@ -238,7 +256,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } }) - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeRestAction[GetIndexRequest, Boolean, Boolean]( operation = "indexExists", index = Some(index), @@ -263,7 +281,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeAddAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", index = Some(index), @@ -282,7 +300,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "removeAlias", index = Some(index), @@ -298,7 +316,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeRestAction[GetAliasesRequest, GetAliasesResponse, Boolean]( operation = "aliasExists", index = Some(alias), @@ -309,7 +327,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().getAlias(req, RequestOptions.DEFAULT) )(response => !response.getAliases.isEmpty) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeRestAction[GetAliasesRequest, GetAliasesResponse, String]( operation = "getAliases", index = Some(index), @@ -324,7 +342,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex -> $newIndex"), @@ -356,7 +374,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "updateSettings", index = Some(index), @@ -368,7 +386,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien executor = req => apply().indices().putSettings(req, RequestOptions.DEFAULT) ) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeRestAction[GetSettingsRequest, GetSettingsResponse, String]( operation = "loadSettings", index = Some(index), @@ -391,7 +409,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "setMapping", index = Some(index), @@ -403,7 +421,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH executor = req => apply().indices().putMapping(req, RequestOptions.DEFAULT) ) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeRestAction[ GetMappingsRequest, org.elasticsearch.client.indices.GetMappingsResponse, @@ -433,7 +451,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeRestAction[RefreshRequest, RefreshResponse, Boolean]( operation = "refresh", index = Some(index), @@ -456,7 +474,7 @@ trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientHelpe index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[FlushRequest, FlushResponse, Boolean]( operation = "flush", index = Some(index), @@ -477,7 +495,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe _: RestHighLevelClientCompanion => override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeRestAction[CountRequest, CountResponse, Option[Double]]( operation = "count", index = Some(query.indices.mkString(",")), @@ -490,7 +508,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = { + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = { executeAsyncRestAction[CountRequest, CountResponse, Option[Double]]( operation = "countAsync", index = Some(query.indices.mkString(",")), @@ -515,7 +533,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[IndexRequest, IndexResponse, Boolean]( operation = "index", index = Some(index), @@ -546,7 +564,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[IndexRequest, IndexResponse, Boolean]( operation = "indexAsync", index = Some(index), @@ -584,7 +602,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "update", index = Some(index), @@ -618,7 +636,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = executeAsyncRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "updateAsync", index = Some(index), @@ -659,7 +677,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "delete", index = Some(index), @@ -681,7 +699,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "deleteAsync", index = Some(index), @@ -712,7 +730,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[GetRequest, GetResponse, Option[String]]( operation = "get", index = Some(index), @@ -731,7 +749,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[GetRequest, GetResponse, Option[String]]( operation = "getAsync", index = Some(index), @@ -764,7 +782,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -794,7 +812,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "multiSearch", index = Some( @@ -830,7 +848,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "executeSingleSearchAsync", index = Some(elasticQuery.indices.mkString(",")), @@ -860,7 +878,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "executeMultiSearchAsync", index = Some( @@ -1586,7 +1604,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeCreatePipeline( pipelineName: String, pipelineDefinition: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( operation = "createPipeline", retryable = false @@ -1603,7 +1621,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeDeletePipeline( pipelineName: String, ifExists: Boolean - ): result.ElasticResult[Boolean] = { + ): ElasticResult[Boolean] = { executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( operation = "deletePipeline", retryable = false @@ -1616,7 +1634,7 @@ trait RestHighLevelClientPipelineApi override private[client] def executeGetPipeline( pipelineName: JSONQuery - ): result.ElasticResult[Option[JSONQuery]] = { + ): ElasticResult[Option[JSONQuery]] = { executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( operation = "getPipeline", retryable = true @@ -1639,3 +1657,314 @@ trait RestHighLevelClientPipelineApi ) } } + +trait RestHighLevelClientTemplateApi + extends TemplateApi + with RestHighLevelClientHelpers + with RestHighLevelClientVersion { + _: RestHighLevelClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutComposableIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutComposableIndexTemplateRequest() + req.name(templateName) + req.indexTemplate( + ComposableIndexTemplate.parse( + org.elasticsearch.common.xcontent.XContentHelper.createParser( + org.elasticsearch.xcontent.NamedXContentRegistry.EMPTY, + org.elasticsearch.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(templateDefinition), + XContentType.JSON + ) + ) + ) + req + } + )( + executor = req => apply().indices().putIndexTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteComposableIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteComposableIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteIndexTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + try { + val request = new GetComposableIndexTemplateRequest(templateName) + val response = apply().indices().getIndexTemplate(request, RequestOptions.DEFAULT) + + val templates = response.getIndexTemplates + if (templates != null && !templates.isEmpty) { + val template = templates.get(templateName) + if (template != null) { + ElasticSuccess(Some(composableTemplateToJson(template))) + } else { + ElasticSuccess(None) + } + } else { + ElasticSuccess(None) + } + } catch { + case _: org.elasticsearch.index.IndexNotFoundException => + ElasticSuccess(None) + case e: Exception => + ElasticFailure( + ElasticError( + message = s"Failed to get composable template: ${e.getMessage}", + operation = Some("getTemplate"), + cause = Some(e) + ) + ) + } + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = { + try { + val request = new GetComposableIndexTemplateRequest("*") + val response = apply().indices().getIndexTemplate(request, RequestOptions.DEFAULT) + + val templates = response.getIndexTemplates + if (templates != null) { + ElasticSuccess( + templates.asScala.map { case (name, template) => + name -> composableTemplateToJson(template) + }.toMap + ) + } else { + ElasticSuccess(Map.empty) + } + } catch { + case e: Exception => + ElasticFailure( + ElasticError( + message = s"Failed to list composable templates: ${e.getMessage}", + operation = Some("listTemplates"), + cause = Some(e) + ) + ) + } + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[ComposableIndexTemplateExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new ComposableIndexTemplateExistRequest(templateName) + )( + executor = req => apply().indices().existsIndexTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Option[String]]( + operation = "getTemplate", + retryable = true + )( + request = new GetIndexTemplatesRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates.asScala + if (templates.nonEmpty) { + val template = templates.head + Some(legacyTemplateToJson(template)) + } else { + None + } + } + ) + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Map[String, String]]( + operation = "listTemplates", + retryable = true + )( + request = new GetIndexTemplatesRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val mapper = JacksonConfig.objectMapper + resp.getIndexTemplates.asScala.map { template => + template.name() -> legacyTemplateToJson(template) + }.toMap + } + ) + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[IndexTemplatesExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new IndexTemplatesExistRequest(templateName) + )( + executor = req => apply().indices().existsTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== CONVERSION HELPERS ==================== + + private def composableTemplateToJson(template: ComposableIndexTemplate): String = { + try { + val builder = XContentFactory.jsonBuilder() + template.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.close() + org.elasticsearch.common.Strings.toString(builder) + } catch { + case e: Exception => + logger.error(s"Failed to convert composable template to JSON: ${e.getMessage}", e) + "{}" + } + } + + private def legacyTemplateToJson(template: IndexTemplateMetadata): String = { + import com.fasterxml.jackson.databind.ObjectMapper + + val mapper = new ObjectMapper() + val root = mapper.createObjectNode() + + // index_patterns + val patternsArray = mapper.createArrayNode() + template.patterns().asScala.foreach(patternsArray.add) + root.set("index_patterns", patternsArray) + + // order + root.put("order", template.order()) + + // version + if (template.version() != null) { + root.put("version", template.version()) + } + + // settings + if (template.settings() != null && !template.settings().isEmpty) { + val settingsNode = mapper.createObjectNode() + template.settings().keySet().asScala.foreach { key => + val value = template.settings().get(key) + if (value != null) { + settingsNode.put(key, value) + } + } + root.set("settings", settingsNode) + } + + // mappings + if (template.mappings() != null && template.mappings().source().string().nonEmpty) { + try { + val mappingsNode = mapper.readTree(template.mappings().source().string()) + root.set("mappings", mappingsNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse mappings: ${e.getMessage}") + } + } + + // aliases + if (template.aliases() != null && !template.aliases().isEmpty) { + val aliasesNode = mapper.createObjectNode() + template.aliases().asScala.foreach { alias => + val aliasName = alias.key + try { + val aliasNode = mapper.readTree(alias.value.toString) + aliasesNode.set(alias.key, aliasNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse alias '$aliasName': ${e.getMessage}") + aliasesNode.set(aliasName, mapper.createObjectNode()) + } + } + root.set("aliases", aliasesNode) + } + + mapper.writeValueAsString(root) + } +} diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala new file mode 100644 index 00000000..67b1622c --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = + new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index ac31e3f3..da65820a 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -25,7 +25,12 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -86,6 +91,7 @@ trait JavaClientApi with JavaClientCompanion with JavaClientVersionApi with JavaClientPipelineApi + with JavaClientTemplateApi /** Elasticsearch client implementation using the Java Client * @see @@ -93,7 +99,7 @@ trait JavaClientApi */ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { _: SerializationApi with JavaClientCompanion => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeJavaAction( operation = "version", index = None, @@ -116,7 +122,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel settings: String, mappings: Option[String], aliases: Seq[String] - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", index = Some(index), @@ -140,7 +146,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.acknowledged()) - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", index = Some(index), @@ -151,7 +157,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .delete(new DeleteIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "closeIndex", index = Some(index), @@ -162,7 +168,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .close(new CloseIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "openIndex", index = Some(index), @@ -178,7 +184,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel targetIndex: String, refresh: Boolean, pipeline: Option[String] - ): result.ElasticResult[(Boolean, Option[Long])] = + ): ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", index = Some(s"$sourceIndex -> $targetIndex"), @@ -202,7 +208,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel (failures.isEmpty, Option(response.total())) } - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "indexExists", index = Some(index), @@ -227,7 +233,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { override private[client] def executeAddAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", index = Some(index), @@ -249,7 +255,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "removeAlias", index = Some(index), @@ -268,7 +274,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "aliasExists", index = None, @@ -281,7 +287,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.value()) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeJavaAction( operation = "getAliases", index = Some(index), @@ -298,7 +304,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex <-> $newIndex"), @@ -334,7 +340,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "updateSettings", index = Some(index), @@ -350,7 +356,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeJavaAction( operation = "loadSettings", index = Some(index), @@ -375,7 +381,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "setMapping", index = Some(index), @@ -388,7 +394,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeJavaAction( operation = "getMapping", index = Some(index), @@ -439,7 +445,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { trait JavaClientRefreshApi extends RefreshApi with JavaClientHelpers { _: JavaClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "refresh", index = Some(index), @@ -469,7 +475,7 @@ trait JavaClientFlushApi extends FlushApi with JavaClientHelpers { index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "flush", index = Some(index), @@ -497,7 +503,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeJavaAction( operation = "count", index = Some(query.indices.mkString(",")), @@ -513,14 +519,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = fromCompletableFuture( async() .count( new CountRequest.Builder().index(query.indices.asJava).build() ) ).map { response => - result.ElasticSuccess(Option(response.count().toDouble)) + ElasticSuccess(Option(response.count().toDouble)) } } @@ -537,7 +543,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "index", index = Some(index), @@ -566,7 +572,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .index( @@ -579,8 +585,8 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Created | Result.Updated | Result.NoOp => result.ElasticSuccess(true) - case _ => result.ElasticSuccess(false) + case Result.Created | Result.Updated | Result.NoOp => ElasticSuccess(true) + case _ => ElasticSuccess(false) } } } @@ -598,7 +604,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "update", index = Some(index), @@ -632,7 +638,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .update( @@ -647,14 +653,14 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Created | Result.Updated | Result.NoOp => result.ElasticSuccess(true) + case Result.Created | Result.Updated | Result.NoOp => ElasticSuccess(true) case Result.NotFound => - result.ElasticFailure( - result.ElasticError( + ElasticFailure( + ElasticError( s"Document with id: $id not found in index: $index" ) // if upsert is false ) - case _ => result.ElasticSuccess(false) + case _ => ElasticSuccess(false) } } @@ -671,7 +677,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "delete", index = Some(index), @@ -694,7 +700,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .delete( @@ -706,8 +712,8 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Deleted | Result.NoOp => result.ElasticSuccess(true) - case _ => result.ElasticSuccess(false) + case Result.Deleted | Result.NoOp => ElasticSuccess(true) + case _ => ElasticSuccess(false) } } @@ -723,7 +729,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "get", index = Some(index), @@ -747,7 +753,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .get( @@ -759,9 +765,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { ) ).map { response => if (response.found()) { - result.ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) + ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) } else { - result.ElasticSuccess(None) + ElasticSuccess(None) } } @@ -781,7 +787,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -801,7 +807,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "multiSearch", index = Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")), @@ -820,7 +826,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .search( @@ -831,12 +837,12 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture { val items = elasticQueries.queries.map { q => new RequestItem.Builder() @@ -849,7 +855,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } } @@ -1526,7 +1532,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java override private[client] def executeCreatePipeline( pipelineName: String, pipelineDefinition: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createPipeline", index = None, @@ -1545,7 +1551,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java override private[client] def executeDeletePipeline( pipelineName: String, ifExists: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deletePipeline", index = None, @@ -1582,3 +1588,238 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java } } } + +trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with JavaClientVersionApi { + _: JavaClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createComposableTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteComposableTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteIndexTemplate( + DeleteIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + ) { resp => + resp.indexTemplates().asScala.headOption.map { template => + convertToJson(template) + } + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name("*") + } + ) + ) { resp => + resp + .indexTemplates() + .asScala + .map { template => + template.name -> convertToJson(template) + } + .toMap + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsIndexTemplate( + ExistsIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.value()) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .putTemplate( + new PutTemplateRequest.Builder() + .name(templateName) + .withJson(new StringReader(templateDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteTemplate( + new DeleteTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name(templateName) + .build() + ) + ) { resp => + resp.result().asScala.get(templateName).map { template => + convertToJson(template) + } + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name("*") + .build() + ) + ) { resp => + resp + .result() + .asScala + .map { case (name, template) => + name -> convertToJson(template) + } + .toMap + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsTemplate( + new ExistsTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.value()) + } + +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index e1841ff2..4ecb88b4 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -20,8 +20,6 @@ import app.softnetwork.elastic.client.ElasticClientCompanion import co.elastic.clients.elasticsearch.{ElasticsearchAsyncClient, ElasticsearchClient} import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.ClassTagExtensions import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -38,8 +36,6 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { private val asyncRef = new AtomicReference[Option[ElasticsearchAsyncClient]](None) - lazy val mapper: ObjectMapper with ClassTagExtensions = new ObjectMapper() with ClassTagExtensions - def async(): ElasticsearchAsyncClient = { asyncRef.get() match { case Some(c) => c diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 20729e6b..4295d333 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -16,13 +16,14 @@ package app.softnetwork.elastic.client.java +import app.softnetwork.elastic.client.SerializationApi import co.elastic.clients.json.JsonpSerializable import co.elastic.clients.json.jackson.JacksonJsonpMapper import java.io.{IOException, StringWriter} import scala.util.Try -trait JavaClientConversion { _: JavaClientCompanion => +trait JavaClientConversion { _: JavaClientCompanion with SerializationApi => private[this] val jsonpMapper = new JacksonJsonpMapper(mapper) /** Convert any Elasticsearch response to JSON string */ diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 940aa01e..00e4d600 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -16,12 +16,15 @@ package app.softnetwork.elastic.client.java -import app.softnetwork.elastic.client.ElasticClientHelpers +import app.softnetwork.elastic.client.{ElasticClientHelpers, SerializationApi} import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} import scala.util.{Failure, Success, Try} -trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { +trait JavaClientHelpers + extends ElasticClientHelpers + with JavaClientConversion + with SerializationApi { _: JavaClientCompanion => // ======================================================================== diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala new file mode 100644 index 00000000..3dbe1131 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 78190da2..8b16650f 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -26,7 +26,12 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.client -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -82,6 +87,7 @@ trait JavaClientApi with JavaClientCompanion with JavaClientVersionApi with JavaClientPipelineApi + with JavaClientTemplateApi /** Elasticsearch client implementation using the Java Client * @see @@ -89,7 +95,7 @@ trait JavaClientApi */ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { _: SerializationApi with JavaClientCompanion => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeJavaAction( operation = "version", index = None, @@ -112,7 +118,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel settings: String, mappings: Option[String], aliases: Seq[String] - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", index = Some(index), @@ -136,7 +142,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.acknowledged()) - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", index = Some(index), @@ -147,7 +153,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .delete(new DeleteIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "closeIndex", index = Some(index), @@ -158,7 +164,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .close(new CloseIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "openIndex", index = Some(index), @@ -174,7 +180,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel targetIndex: String, refresh: Boolean, pipeline: Option[String] - ): result.ElasticResult[(Boolean, Option[Long])] = + ): ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", index = Some(s"$sourceIndex -> $targetIndex"), @@ -198,7 +204,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel (failures.isEmpty, Option(response.total())) } - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "indexExists", index = Some(index), @@ -223,7 +229,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { override private[client] def executeAddAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", index = Some(index), @@ -245,7 +251,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "removeAlias", index = Some(index), @@ -264,7 +270,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "aliasExists", index = None, @@ -277,7 +283,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.value()) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeJavaAction( operation = "getAliases", index = Some(index), @@ -294,7 +300,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex <-> $newIndex"), @@ -330,7 +336,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "updateSettings", index = Some(index), @@ -346,7 +352,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeJavaAction( operation = "loadSettings", index = Some(index), @@ -371,7 +377,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "setMapping", index = Some(index), @@ -384,7 +390,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeJavaAction( operation = "getMapping", index = Some(index), @@ -436,7 +442,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { trait JavaClientRefreshApi extends RefreshApi with JavaClientHelpers { _: JavaClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "refresh", index = Some(index), @@ -466,7 +472,7 @@ trait JavaClientFlushApi extends FlushApi with JavaClientHelpers { index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "flush", index = Some(index), @@ -494,7 +500,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeJavaAction( operation = "count", index = Some(query.indices.mkString(",")), @@ -510,14 +516,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = fromCompletableFuture( async() .count( new CountRequest.Builder().index(query.indices.asJava).build() ) ).map { response => - result.ElasticSuccess(Option(response.count().toDouble)) + ElasticSuccess(Option(response.count().toDouble)) } } @@ -534,7 +540,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "index", index = Some(index), @@ -562,7 +568,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .index( @@ -575,10 +581,10 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to index document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to index document with id: $id in index: $index") ) } } @@ -598,7 +604,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "update", index = Some(index), @@ -627,7 +633,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .update( @@ -642,10 +648,10 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to update document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to update document with id: $id in index: $index") ) } } @@ -663,7 +669,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "delete", index = Some(index), @@ -685,7 +691,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .delete( @@ -697,10 +703,10 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to delete document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to delete document with id: $id in index: $index") ) } } @@ -717,7 +723,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "get", index = Some(index), @@ -741,7 +747,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .get( @@ -753,9 +759,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { ) ).map { response => if (response.found()) { - result.ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) + ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) } else { - result.ElasticSuccess(None) + ElasticSuccess(None) } } @@ -775,7 +781,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -795,7 +801,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "multiSearch", index = Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")), @@ -814,7 +820,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .search( @@ -825,12 +831,12 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture { val items = elasticQueries.queries.map { q => new RequestItem.Builder() @@ -843,7 +849,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } } @@ -1520,7 +1526,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java override private[client] def executeCreatePipeline( pipelineName: String, pipelineDefinition: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createPipeline", index = None, @@ -1539,7 +1545,7 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java override private[client] def executeDeletePipeline( pipelineName: String, ifExists: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deletePipeline", index = None, @@ -1576,3 +1582,238 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java } } } + +trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with JavaClientVersionApi { + _: JavaClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createComposableTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteComposableTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteIndexTemplate( + DeleteIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + ) { resp => + resp.indexTemplates().asScala.headOption.map { template => + convertToJson(template) + } + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name("*") + } + ) + ) { resp => + resp + .indexTemplates() + .asScala + .map { template => + template.name -> convertToJson(template) + } + .toMap + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsIndexTemplate( + ExistsIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.value()) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .putTemplate( + new PutTemplateRequest.Builder() + .name(templateName) + .withJson(new StringReader(templateDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteTemplate( + new DeleteTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name(templateName) + .build() + ) + ) { resp => + resp.templates().asScala.get(templateName).map { template => + convertToJson(template) + } + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name("*") + .build() + ) + ) { resp => + resp + .templates() + .asScala + .map { case (name, template) => + name -> convertToJson(template) + } + .toMap + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsTemplate( + new ExistsTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.value()) + } + +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index e1841ff2..4ecb88b4 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -20,8 +20,6 @@ import app.softnetwork.elastic.client.ElasticClientCompanion import co.elastic.clients.elasticsearch.{ElasticsearchAsyncClient, ElasticsearchClient} import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.ClassTagExtensions import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -38,8 +36,6 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { private val asyncRef = new AtomicReference[Option[ElasticsearchAsyncClient]](None) - lazy val mapper: ObjectMapper with ClassTagExtensions = new ObjectMapper() with ClassTagExtensions - def async(): ElasticsearchAsyncClient = { asyncRef.get() match { case Some(c) => c diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 20729e6b..4295d333 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -16,13 +16,14 @@ package app.softnetwork.elastic.client.java +import app.softnetwork.elastic.client.SerializationApi import co.elastic.clients.json.JsonpSerializable import co.elastic.clients.json.jackson.JacksonJsonpMapper import java.io.{IOException, StringWriter} import scala.util.Try -trait JavaClientConversion { _: JavaClientCompanion => +trait JavaClientConversion { _: JavaClientCompanion with SerializationApi => private[this] val jsonpMapper = new JacksonJsonpMapper(mapper) /** Convert any Elasticsearch response to JSON string */ diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 940aa01e..00e4d600 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -16,12 +16,15 @@ package app.softnetwork.elastic.client.java -import app.softnetwork.elastic.client.ElasticClientHelpers +import app.softnetwork.elastic.client.{ElasticClientHelpers, SerializationApi} import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} import scala.util.{Failure, Success, Try} -trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { +trait JavaClientHelpers + extends ElasticClientHelpers + with JavaClientConversion + with SerializationApi { _: JavaClientCompanion => // ======================================================================== diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala new file mode 100644 index 00000000..3dbe1131 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index b6453391..e9be244f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -313,7 +313,7 @@ package object query { sealed trait DdlStatement extends Statement - sealed trait PipelineDdlStatement extends DdlStatement + sealed trait PipelineStatement extends DdlStatement case class CreatePipeline( name: String, @@ -321,7 +321,7 @@ package object query { ifNotExists: Boolean = false, orReplace: Boolean = false, processors: Seq[DdlProcessor] - ) extends PipelineDdlStatement { + ) extends PipelineStatement { override def sql: String = { val processorsDdl = processors.map(_.ddl).mkString(", ") val replaceClause = if (orReplace) " OR REPLACE" else "" @@ -352,7 +352,7 @@ package object query { name: String, ifExists: Boolean, statements: List[AlterPipelineStatement] - ) extends PipelineDdlStatement { + ) extends PipelineStatement { override def sql: String = { val ifExistsClause = if (ifExists) " IF EXISTS " else "" val parenthesesNeeded = statements.size > 1 @@ -370,7 +370,7 @@ package object query { DdlPipeline(s"alter-pipeline-$name-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) } - case class DropPipeline(name: String, ifExists: Boolean = false) extends PipelineDdlStatement { + case class DropPipeline(name: String, ifExists: Boolean = false) extends PipelineStatement { override def sql: String = { val ifExistsClause = if (ifExists) "IF EXISTS " else "" s"DROP PIPELINE $ifExistsClause$name" diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index 40771006..e5b6fc34 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.{ SerializationFeature } import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} import scala.language.implicitConversions import scala.jdk.CollectionConverters._ @@ -35,7 +35,7 @@ package object serialization { /** Jackson ObjectMapper configuration */ object JacksonConfig { lazy val objectMapper: ObjectMapper = { - val mapper = new ObjectMapper() + val mapper = new ObjectMapper() with ClassTagExtensions // Scala module for native support of Scala types mapper.registerModule(DefaultScalaModule) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala index c91d7081..9164f64f 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala @@ -69,7 +69,7 @@ trait PipelineApiSpec ) testPipelines.foreach { pipelineName => - client.deletePipeline(pipelineName, ifExists = false) match { + client.deletePipeline(pipelineName, ifExists = true) match { case _ => // Ignore result, just cleanup } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala new file mode 100644 index 00000000..a756996b --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala @@ -0,0 +1,134 @@ +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.ExecutionContextExecutor + +trait TemplateApiSpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: TemplateApi with VersionApi + + def supportComposableTemplates: Boolean = { + // Get Elasticsearch version + val elasticVersion = { + client.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + log.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return false + } + } + ElasticsearchVersion.supportsComposableTemplates(elasticVersion) + } + + "createTemplate" should "create and retrieve template" in { + val template = + """ + |{ + | "index_patterns": ["test-*"], + | "priority": 100, + | "template": { + | "settings": { + | "number_of_shards": 1 + | } + | } + |} + |""".stripMargin + + // Create + val createResult = client.createTemplate("test-template", template) + createResult shouldBe a[ElasticSuccess[_]] + + // Verify existence + val existsResult = client.templateExists("test-template") + existsResult shouldBe ElasticSuccess(true) + + // Get + val getResult = client.getTemplate("test-template") + getResult match { + case ElasticSuccess(Some(json)) => + json should include("test-*") + if (supportComposableTemplates) + json should include("priority") + case _ => fail("Failed to get template") + } + + // Cleanup + client.deleteTemplate("test-template", ifExists = true) + } + + "createTemplate" should "convert legacy format automatically" in { + val legacyTemplate = + """ + |{ + | "index_patterns": ["legacy-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + + val result = client.createTemplate("legacy-test", legacyTemplate) + result shouldBe a[ElasticSuccess[_]] + + // Verify conversion + client.getTemplate("legacy-test") match { + case ElasticSuccess(Some(json)) if supportComposableTemplates => + json should include("priority") + json should include("template") + case ElasticSuccess(Some(json)) => + json should include("legacy-*") + case _ => fail("Failed to get template") + } + + // Cleanup + client.deleteTemplate("legacy-test", ifExists = true) + } + + "deleteTemplate" should "handle non-existing template with ifExists=true" in { + val result = client.deleteTemplate("non-existing", ifExists = true) + result shouldBe ElasticSuccess(false) + } + + "listTemplates" should "return all templates" in { + // Create test templates + client.createTemplate("list-1", """{"index_patterns":["test1-*"],"priority":1,"template":{}}""") + client.createTemplate("list-2", """{"index_patterns":["test2-*"],"priority":2,"template":{}}""") + + // List + val result = client.listTemplates() + result match { + case ElasticSuccess(templates) => + templates.keys should contain allOf ("list-1", "list-2") + case _ => fail("Failed to list templates") + } + + // Cleanup + client.deleteTemplate("list-1", ifExists = true) + client.deleteTemplate("list-2", ifExists = true) + } +} From a477789ed9c2dbd48929bce03313a07a3892a3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Dec 2025 13:32:27 +0100 Subject: [PATCH 27/95] finalize template api --- .../elastic/client/ElasticsearchVersion.scala | 34 + .../elastic/client/TemplateApi.scala | 18 +- .../elastic/client/TemplateConverter.scala | 188 +++-- .../elastic/client/jest/JestTemplateApi.scala | 45 +- .../client/JestClientTemplateApiSpec.scala | 2 + .../client/rest/RestHighLevelClientApi.scala | 77 +- .../client/rest/RestHighLevelClientApi.scala | 166 ++-- .../elastic/client/TemplateApiSpec.scala | 706 ++++++++++++++++-- 8 files changed, 1039 insertions(+), 197 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 64ef818d..2fabb9f4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -59,6 +59,27 @@ object ElasticsearchVersion { } } + /** Check if version is <= target version + */ + def isAtMost( + version: String, + targetMajor: Int, + targetMinor: Int = 0, + targetPatch: Int = 0 + ): Boolean = { + val (major, minor, patch) = parse(version) + + if (major < targetMajor) true + else if (major > targetMajor) false + else { // major == targetMajor + if (minor < targetMinor) true + else if (minor > targetMinor) false + else { // minor == targetMinor + patch <= targetPatch + } + } + } + /** Check if PIT is supported (ES >= 7.10) */ def supportsPit(version: String): Boolean = { @@ -102,4 +123,17 @@ object ElasticsearchVersion { isAtLeast(version, 7, 10) } + /** Check if Elasticsearch version requires _doc type wrapper in mappings + * + * ES 6.x requires mappings to be wrapped in a type name (e.g., "_doc") ES 7.x removed mapping + * types + * + * @param version + * the Elasticsearch version (e.g., "6.8.0") + * @return + * true if _doc wrapper is required + */ + def requiresDocTypeWrapper(version: String): Boolean = { + !isAtLeast(version, 6, 8) + } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala index 5ff51483..331c416b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala @@ -169,10 +169,20 @@ trait TemplateApi extends ElasticClientHelpers { _: VersionApi => } } - if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { - executeGetComposableTemplate(templateName) - } else { - executeGetLegacyTemplate(templateName) + (if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeGetComposableTemplate(templateName) + } else { + executeGetLegacyTemplate(templateName) + }) match { + case success @ ElasticSuccess(_) => success + case failure @ ElasticFailure(error) => + error.statusCode match { + case Some(404) => + logger.warn(s"⚠️ Template $templateName not found") + return ElasticSuccess(None) + case _ => + } + failure } } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala index 422837e7..4f6eaedd 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala @@ -1,21 +1,58 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.{Logger, LoggerFactory} import scala.util.{Success, Try} object TemplateConverter { - private lazy val mapper = JacksonConfig.objectMapper + private val logger: Logger = LoggerFactory.getLogger(getClass) - /** Detect if a template JSON is in legacy format + private lazy val mapper: ObjectMapper = JacksonConfig.objectMapper + + /** Normalize template format based on Elasticsearch version * - * Legacy format has settings/mappings/aliases at root level Composable format has them nested - * under 'template' key + * @param templateJson + * the template JSON (legacy or composable format) + * @param elasticVersion + * the Elasticsearch version (e.g., "7.10.0") + * @return + * normalized template JSON + */ + def normalizeTemplate(templateJson: String, elasticVersion: String): Try[String] = { + val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(elasticVersion) + val isLegacy = isLegacyFormat(templateJson) + + (supportsComposable, isLegacy) match { + case (true, true) => + // ES 7.8+ with legacy format → convert to composable + logger.debug("Converting legacy template to composable format") + convertLegacyToComposable(templateJson) + + case (false, _) => + // ES < 7.8 with composable format → convert to legacy + // ES < 6.8 with legacy format → ensure _doc wrapper in mappings + logger.debug( + "Converting composable template to legacy format for ES < 7.8 / Ensuring _doc wrapper in legacy template mappings for ES < 6.8" + ) + convertComposableToLegacy(templateJson, elasticVersion) + + case _ => + // Already in correct format + logger.debug("Template already in correct format") + Success(templateJson) + } + } + + /** Check if template is in legacy format + * + * Legacy format has 'order' field, composable has 'priority' * * @param templateJson - * the template JSON string + * the template JSON * @return * true if legacy format */ @@ -86,6 +123,8 @@ object TemplateConverter { // 2. Convert 'order' to 'priority' if (root.has("order")) { composable.put("priority", root.get("order").asInt()) + } else { + composable.put("priority", 1) // Default priority } // 3. Copy version if present @@ -93,12 +132,7 @@ object TemplateConverter { composable.set("version", root.get("version")) } - // 4. Copy _meta if present - if (root.has("_meta")) { - composable.set("_meta", root.get("_meta")) - } - - // 5. Create nested 'template' object with settings/mappings/aliases + // 4. Wrap settings, mappings, aliases in 'template' object val templateNode = mapper.createObjectNode() if (root.has("settings")) { @@ -106,16 +140,24 @@ object TemplateConverter { } if (root.has("mappings")) { - templateNode.set("mappings", root.get("mappings")) + // Remove _doc wrapper if present (legacy ES 6.x format) + val mappingsNode = root.get("mappings") + if (mappingsNode.isObject && mappingsNode.has("_doc")) { + templateNode.set("mappings", mappingsNode.get("_doc")) + } else { + templateNode.set("mappings", mappingsNode) + } } if (root.has("aliases")) { templateNode.set("aliases", root.get("aliases")) } - // Only add 'template' key if it has content - if (templateNode.size() > 0) { - composable.set("template", templateNode) + composable.set("template", templateNode) + + // 5. Copy _meta if present (valid in composable format) + if (root.has("_meta")) { + composable.set("_meta", root.get("_meta")) } mapper.writeValueAsString(composable) @@ -128,10 +170,15 @@ object TemplateConverter { * * @param composableJson * the composable template JSON + * @param elasticVersion + * the Elasticsearch version (e.g., "6.8.0") * @return * legacy template JSON */ - def convertComposableToLegacy(composableJson: String): Try[String] = { + def convertComposableToLegacy( + composableJson: String, + elasticVersion: String + ): Try[String] = { Try { val root = mapper.readTree(composableJson).asInstanceOf[ObjectNode] val legacy = mapper.createObjectNode() @@ -144,6 +191,11 @@ object TemplateConverter { // 2. Convert 'priority' to 'order' if (root.has("priority")) { legacy.put("order", root.get("priority").asInt()) + } else if (root.has("order")) { + // In case 'order' is already present + legacy.put("order", root.get("order").asInt()) + } else { + legacy.put("order", 1) // Default order } // 3. Copy version if present @@ -151,10 +203,8 @@ object TemplateConverter { legacy.set("version", root.get("version")) } - // 4. Copy _meta if present - if (root.has("_meta")) { - legacy.set("_meta", root.get("_meta")) - } + // 4. ❌ DO NOT copy _meta (not supported in legacy format) + // Legacy templates don't support _meta field // 5. Flatten 'template' object to root level if (root.has("template")) { @@ -165,7 +215,23 @@ object TemplateConverter { } if (templateNode.has("mappings")) { - legacy.set("mappings", templateNode.get("mappings")) + val mappingsNode = templateNode.get("mappings") + + // Check if we need to wrap in _doc for ES 6.x + if ( + ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mappingsNode.has("_doc") + ) { + val wrappedMappings = mapper.createObjectNode() + wrappedMappings.set("_doc", mappingsNode) + legacy.set("mappings", wrappedMappings) + logger.debug(s"Wrapped mappings in '_doc' for ES $elasticVersion") + } else if ( + !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mappingsNode.has("_doc") + ) { + legacy.set("mappings", mappingsNode.get("_doc")) + } else { + legacy.set("mappings", mappingsNode) + } } if (templateNode.has("aliases")) { @@ -173,43 +239,79 @@ object TemplateConverter { } } + if (root.has("settings")) { + legacy.set("settings", root.get("settings")) + } + + if (root.has("mappings")) { + val mappingsNode = root.get("mappings") + + // Check if we need to wrap in _doc for ES 6.x + if ( + ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mappingsNode.has("_doc") + ) { + val wrappedMappings = mapper.createObjectNode() + wrappedMappings.set("_doc", mappingsNode) + legacy.set("mappings", wrappedMappings) + logger.debug(s"Wrapped mappings in '_doc' for ES $elasticVersion") + } else if ( + !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mappingsNode.has("_doc") + ) { + legacy.set("mappings", mappingsNode.get("_doc")) + } else { + legacy.set("mappings", mappingsNode) + } + } + + if (root.has("aliases")) { + legacy.set("aliases", root.get("aliases")) + } + // Note: 'composed_of' and 'data_stream' are not supported in legacy format // They will be silently ignored + if (root.has("composed_of")) { + logger.warn( + "Composable templates are not supported in legacy template format and will be ignored" + ) + } + if (root.has("data_stream")) { + logger.warn("Data streams are not supported in legacy template format and will be ignored") + } mapper.writeValueAsString(legacy) } } - /** Validate and normalize template JSON based on ES version + /** Validate that a template has required fields * * @param templateJson - * the template JSON (legacy or composable format) - * @param esVersion - * the Elasticsearch version + * the template JSON * @return - * normalized template JSON appropriate for the ES version + * None if valid, Some(error message) if invalid */ - def normalizeTemplate(templateJson: String, esVersion: String): Try[String] = { - val isLegacy = isLegacyFormat(templateJson) - val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(esVersion) + def validateTemplate(templateJson: String): Option[String] = { + Try { + val root = mapper.readTree(templateJson) - (isLegacy, supportsComposable) match { - case (true, true) => - // Legacy format + ES 7.8+ → Convert to composable - convertLegacyToComposable(templateJson) + // Check for index_patterns (required in both formats) + if (!root.has("index_patterns")) { + return Some("Missing required field: index_patterns") + } - case (false, false) => - // Composable format + ES < 7.8 → Convert to legacy - convertComposableToLegacy(templateJson) + val indexPatterns = root.get("index_patterns") + if (!indexPatterns.isArray || indexPatterns.size() == 0) { + return Some("index_patterns must be a non-empty array") + } - case (true, false) => - // Legacy format + ES < 7.8 → Keep as is - Success(templateJson) + // Check for either priority (composable) or order (legacy) + if (!root.has("priority") && !root.has("order")) { + logger.warn("Template missing both 'priority' and 'order', will use default") + } - case (false, true) => - // Composable format + ES 7.8+ → Keep as is - Success(templateJson) - } + None + }.recover { case e: Exception => + Some(s"Invalid JSON: ${e.getMessage}") + }.get } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala index 9855c941..e54abea6 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala @@ -23,23 +23,58 @@ trait JestTemplateApi extends TemplateApi with JestClientHelpers with JestVersio override private[client] def executeCreateComposableTemplate( templateName: String, templateDefinition: String - ): ElasticResult[Boolean] = ElasticSuccess(false) + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("createTemplate") + ) + ) override private[client] def executeDeleteComposableTemplate( templateName: String, ifExists: Boolean - ): ElasticResult[Boolean] = ElasticSuccess(false) + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("deleteTemplate") + ) + ) override private[client] def executeGetComposableTemplate( templateName: String - ): ElasticResult[Option[String]] = ElasticSuccess(None) + ): ElasticResult[Option[String]] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("getTemplate") + ) + ) override private[client] def executeListComposableTemplates() - : ElasticResult[Map[String, String]] = ElasticSuccess(Map.empty[String, String]) + : ElasticResult[Map[String, String]] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("listTemplates") + ) + ) override private[client] def executeComposableTemplateExists( templateName: String - ): ElasticResult[Boolean] = ElasticSuccess(false) + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("templateExists") + ) + ) // ==================== LEGACY TEMPLATES ==================== diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala index c8e34ff2..17681307 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala @@ -5,4 +5,6 @@ import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit class JestClientTemplateApiSpec extends TemplateApiSpec with EmbeddedElasticTestKit { override def client: TemplateApi with VersionApi = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" } diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 2fa1741f..a88c4196 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, Ela import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.{Alias, IndicesAliasesRequest} @@ -73,6 +74,7 @@ import org.elasticsearch.client.indices.{ PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.cluster.metadata.AliasMetaData import org.elasticsearch.common.Strings import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.common.unit.TimeValue @@ -1644,21 +1646,76 @@ trait RestHighLevelClientTemplateApi // aliases if (template.aliases() != null && !template.aliases().isEmpty) { val aliasesNode = mapper.createObjectNode() - template.aliases().asScala.foreach { alias => - val aliasName = alias.key - try { - val aliasNode = mapper.readTree(alias.value.toString) - aliasesNode.set(alias.key, aliasNode) - } catch { - case e: Exception => - logger.warn(s"Failed to parse alias '$aliasName': ${e.getMessage}") - aliasesNode.set(aliasName, mapper.createObjectNode()) - } + + template.aliases().keysIt().asScala.foreach { aliasName => + // Type explicite pour éviter l'inférence vers Nothing$ + val aliasObjectNode: ObjectNode = + try { + val aliasMetadata = template.aliases().get(aliasName) + + // Convert AliasMetadata to JSON + val aliasJson = convertAliasMetadataToJson(aliasMetadata) + + // Parse and validate + val parsedAlias = mapper.readTree(aliasJson) + + if (parsedAlias.isInstanceOf[ObjectNode]) { + parsedAlias.asInstanceOf[ObjectNode].get(aliasName) match { + case objNode: ObjectNode => objNode + case _ => + logger.debug( + s"Alias '$aliasName' does not contain an object node, creating empty object" + ) + mapper.createObjectNode() + } + } else { + logger.debug( + s"Alias '$aliasName' is not an ObjectNode (type: ${parsedAlias.getClass.getName}), creating empty object" + ) + mapper.createObjectNode() + } + + } catch { + case e: Exception => + logger.warn(s"Failed to process alias '$aliasName': ${e.getMessage}", e) + mapper.createObjectNode() + } + + // Set with explicit type + aliasesNode.set[ObjectNode](aliasName, aliasObjectNode) } + root.set("aliases", aliasesNode) } mapper.writeValueAsString(root) } + /** Convert AliasMetadata to JSON string + * + * @param aliasMetadata + * the alias metadata + * @return + * JSON string representation + */ + private def convertAliasMetadataToJson( + aliasMetadata: AliasMetaData + ): String = { + try { + import org.elasticsearch.common.xcontent.{XContentFactory, ToXContent} + + val builder = XContentFactory.jsonBuilder() + builder.startObject() + aliasMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.endObject() + builder.close() + + org.elasticsearch.common.Strings.toString(builder) + + } catch { + case e: Exception => + logger.warn(s"Failed to convert AliasMetadata to JSON: ${e.getMessage}", e) + "{}" + } + } } diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 1a57564c..82a2b900 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,16 +21,12 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ -import app.softnetwork.elastic.client.result.{ - ElasticError, - ElasticFailure, - ElasticResult, - ElasticSuccess -} +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils import org.elasticsearch.action.admin.indices.alias.Alias @@ -86,7 +82,7 @@ import org.elasticsearch.client.indices.{ PutIndexTemplateRequest, PutMappingRequest } -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate +import org.elasticsearch.cluster.metadata.{AliasMetadata, ComposableIndexTemplate} import org.elasticsearch.common.Strings import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue @@ -1722,61 +1718,51 @@ trait RestHighLevelClientTemplateApi override private[client] def executeGetComposableTemplate( templateName: String ): ElasticResult[Option[String]] = { - try { - val request = new GetComposableIndexTemplateRequest(templateName) - val response = apply().indices().getIndexTemplate(request, RequestOptions.DEFAULT) - - val templates = response.getIndexTemplates - if (templates != null && !templates.isEmpty) { - val template = templates.get(templateName) - if (template != null) { - ElasticSuccess(Some(composableTemplateToJson(template))) + executeRestAction( + operation = "getTemplate", + retryable = true + )( + request = new GetComposableIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates + if (templates != null && !templates.isEmpty) { + val template = templates.get(templateName) + if (template != null) { + Some(composableTemplateToJson(template)) + } else { + None + } } else { - ElasticSuccess(None) + None } - } else { - ElasticSuccess(None) } - } catch { - case _: org.elasticsearch.index.IndexNotFoundException => - ElasticSuccess(None) - case e: Exception => - ElasticFailure( - ElasticError( - message = s"Failed to get composable template: ${e.getMessage}", - operation = Some("getTemplate"), - cause = Some(e) - ) - ) - } + ) } override private[client] def executeListComposableTemplates() : ElasticResult[Map[String, String]] = { - try { - val request = new GetComposableIndexTemplateRequest("*") - val response = apply().indices().getIndexTemplate(request, RequestOptions.DEFAULT) - - val templates = response.getIndexTemplates - if (templates != null) { - ElasticSuccess( + executeRestAction( + operation = "listTemplates", + retryable = true + )( + request = new GetComposableIndexTemplateRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates + if (templates != null) { templates.asScala.map { case (name, template) => name -> composableTemplateToJson(template) }.toMap - ) - } else { - ElasticSuccess(Map.empty) + } else { + Map.empty + } } - } catch { - case e: Exception => - ElasticFailure( - ElasticError( - message = s"Failed to list composable templates: ${e.getMessage}", - operation = Some("listTemplates"), - cause = Some(e) - ) - ) - } + ) } override private[client] def executeComposableTemplateExists( @@ -1907,9 +1893,7 @@ trait RestHighLevelClientTemplateApi } private def legacyTemplateToJson(template: IndexTemplateMetadata): String = { - import com.fasterxml.jackson.databind.ObjectMapper - val mapper = new ObjectMapper() val root = mapper.createObjectNode() // index_patterns @@ -1951,20 +1935,76 @@ trait RestHighLevelClientTemplateApi // aliases if (template.aliases() != null && !template.aliases().isEmpty) { val aliasesNode = mapper.createObjectNode() - template.aliases().asScala.foreach { alias => - val aliasName = alias.key - try { - val aliasNode = mapper.readTree(alias.value.toString) - aliasesNode.set(alias.key, aliasNode) - } catch { - case e: Exception => - logger.warn(s"Failed to parse alias '$aliasName': ${e.getMessage}") - aliasesNode.set(aliasName, mapper.createObjectNode()) - } + + template.aliases().keysIt().asScala.foreach { aliasName => + // Type explicite pour éviter l'inférence vers Nothing$ + val aliasObjectNode: ObjectNode = + try { + val aliasMetadata = template.aliases().get(aliasName) + + // Convert AliasMetadata to JSON + val aliasJson = convertAliasMetadataToJson(aliasMetadata) + + // Parse and validate + val parsedAlias = mapper.readTree(aliasJson) + + if (parsedAlias.isInstanceOf[ObjectNode]) { + parsedAlias.asInstanceOf[ObjectNode].get(aliasName) match { + case objNode: ObjectNode => objNode + case _ => + logger.debug( + s"Alias '$aliasName' does not contain an object node, creating empty object" + ) + mapper.createObjectNode() + } + } else { + logger.debug( + s"Alias '$aliasName' is not an ObjectNode (type: ${parsedAlias.getClass.getName}), creating empty object" + ) + mapper.createObjectNode() + } + + } catch { + case e: Exception => + logger.warn(s"Failed to process alias '$aliasName': ${e.getMessage}", e) + mapper.createObjectNode() + } + + // Set with explicit type + aliasesNode.set[ObjectNode](aliasName, aliasObjectNode) } + root.set("aliases", aliasesNode) } mapper.writeValueAsString(root) } + + /** Convert AliasMetadata to JSON string + * + * @param aliasMetadata + * the alias metadata + * @return + * JSON string representation + */ + private def convertAliasMetadataToJson( + aliasMetadata: AliasMetadata + ): String = { + try { + import org.elasticsearch.xcontent.{ToXContent, XContentFactory} + + val builder = XContentFactory.jsonBuilder() + builder.startObject() + aliasMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.endObject() + builder.close() + + org.elasticsearch.common.Strings.toString(builder) + + } catch { + case e: Exception => + logger.warn(s"Failed to convert AliasMetadata to JSON: ${e.getMessage}", e) + "{}" + } + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala index a756996b..b16ed8d3 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala @@ -3,6 +3,7 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.serialization.JacksonConfig import app.softnetwork.persistence.generateUUID import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.concurrent.ScalaFutures @@ -31,104 +32,665 @@ trait TemplateApiSpec def client: TemplateApi with VersionApi + // ==================== HELPER METHODS ==================== + def supportComposableTemplates: Boolean = { - // Get Elasticsearch version - val elasticVersion = { - client.version match { - case ElasticSuccess(v) => v - case ElasticFailure(error) => - log.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") - return false - } + client.version match { + case ElasticSuccess(v) => ElasticsearchVersion.supportsComposableTemplates(v) + case ElasticFailure(error) => + log.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + false } - ElasticsearchVersion.supportsComposableTemplates(elasticVersion) - } - - "createTemplate" should "create and retrieve template" in { - val template = - """ - |{ - | "index_patterns": ["test-*"], - | "priority": 100, - | "template": { - | "settings": { - | "number_of_shards": 1 - | } - | } - |} - |""".stripMargin - - // Create - val createResult = client.createTemplate("test-template", template) - createResult shouldBe a[ElasticSuccess[_]] + } + + def esVersion: String = { + client.version match { + case ElasticSuccess(v) => v + case ElasticFailure(_) => "unknown" + } + } - // Verify existence - val existsResult = client.templateExists("test-template") - existsResult shouldBe ElasticSuccess(true) + /** Sanitize pattern to create valid alias name + * + * Removes invalid characters: [' ','"','*',',','/','<','>','?','\','|'] + * + * @param pattern + * the index pattern (e.g., "test-*") + * @return + * valid alias name (e.g., "test-alias") + */ + private def sanitizeAliasName(pattern: String): String = { + pattern + .replaceAll("[\\s\"*,/<>?\\\\|]", "") // Remove invalid characters + .replaceAll("-+", "-") // Replace multiple dashes with single dash + .replaceAll("^-|-$", "") // Remove leading/trailing dashes + .toLowerCase + } - // Get - val getResult = client.getTemplate("test-template") - getResult match { - case ElasticSuccess(Some(json)) => - json should include("test-*") - if (supportComposableTemplates) - json should include("priority") - case _ => fail("Failed to get template") + /** Generate composable template JSON + * + * @param pattern + * the index pattern (e.g., "test-*") + * @param priority + * the template priority (default: 100) + * @param withAliases + * whether to include aliases (default: true) + * @return + * JSON string for composable template + */ + def composableTemplate( + pattern: String, + priority: Int = 100, + withAliases: Boolean = true + ): String = { + val aliasName = sanitizeAliasName(pattern) + val aliasesJson = if (withAliases) { + s""" + | "aliases": { + | "${aliasName}-alias": {} + | } + |""".stripMargin + } else { + "" } - // Cleanup - client.deleteTemplate("test-template", ifExists = true) - } - - "createTemplate" should "convert legacy format automatically" in { - val legacyTemplate = - """ - |{ - | "index_patterns": ["legacy-*"], - | "order": 1, - | "settings": { - | "number_of_shards": 1 - | } - |} - |""".stripMargin - - val result = client.createTemplate("legacy-test", legacyTemplate) + s""" + |{ + | "index_patterns": ["$pattern"], + | "priority": $priority, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 0 + | }, + | "mappings": { + | "properties": { + | "timestamp": { "type": "date" }, + | "message": { "type": "text" } + | } + | }${if (withAliases) s",\n\t$aliasesJson" else ""} + | }, + | "version": 1, + | "_meta": { + | "description": "Test template" + | } + |} + |""".stripMargin + } + + /** Generate legacy template JSON + * + * @param pattern + * the index pattern (e.g., "test-*") + * @param order + * the template order (default: 1) + * @param withAliases + * whether to include aliases (default: true) + * @return + * JSON string for legacy template + */ + def legacyTemplate( + pattern: String, + order: Int = 1, + withAliases: Boolean = true, + version: Int = 1 + ): String = { + val aliasName = sanitizeAliasName(pattern) + val aliasesJson = if (withAliases) { + s""" + | "aliases": { + | "${aliasName}-alias": {} + | }, + |""".stripMargin + } else { + "" + } + + s""" + |{ + | "index_patterns": ["$pattern"], + | "order": $order, + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 0 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": { "type": "date" }, + | "message": { "type": "text" } + | } + | } + | }${if (withAliases) s",\n\t$aliasesJson" else ""} + | "version": $version + |} + |""".stripMargin + } + + def cleanupTemplate(name: String): Unit = { + client.deleteTemplate(name, ifExists = true) + } + + override def afterEach(): Unit = { + super.afterEach() + // Cleanup all test templates + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.filter(_.startsWith("test-")).foreach(cleanupTemplate) + case _ => // Ignore + } + } + + override def beforeAll(): Unit = { + self.beforeAll() + info(s"Running tests against Elasticsearch $esVersion") + info(s"Composable templates supported: $supportComposableTemplates") + } + + // ==================== TEST SUITE ==================== + + behavior of "TemplateApi" + + // ========== CREATE TEMPLATE ========== + + it should "create a composable template (ES 7.8+)" in { + assume(supportComposableTemplates, "Composable templates not supported") + + val template = composableTemplate("test-composable-*") + val result = client.createTemplate("test-composable", template) + + result shouldBe a[ElasticSuccess[_]] + result.asInstanceOf[ElasticSuccess[Boolean]].value shouldBe true + } + + it should "create a legacy template (ES < 7.8)" in { + assume(!supportComposableTemplates, "Legacy templates only for ES < 7.8") + + val template = legacyTemplate("test-legacy-*") + val result = client.createTemplate("test-legacy", template) + + result shouldBe a[ElasticSuccess[_]] + result.asInstanceOf[ElasticSuccess[Boolean]].value shouldBe true + } + + it should "auto-convert legacy format to composable (ES 7.8+)" in { + assume(supportComposableTemplates, "Composable templates not supported") + + val legacyFormat = legacyTemplate("test-auto-convert-*") + val result = client.createTemplate("test-auto-convert", legacyFormat) + result shouldBe a[ElasticSuccess[_]] - // Verify conversion - client.getTemplate("legacy-test") match { - case ElasticSuccess(Some(json)) if supportComposableTemplates => + // Verify it was stored as composable + client.getTemplate("test-auto-convert") match { + case ElasticSuccess(Some(json)) => json should include("priority") json should include("template") + json should not include "order" + case _ => fail("Failed to get template") + } + } + + it should "auto-convert composable format to legacy (ES < 7.8)" in { + assume(!supportComposableTemplates, "Legacy templates only for ES < 7.8") + + val composableFormat = composableTemplate("test-auto-convert-legacy-*") + val result = client.createTemplate("test-auto-convert-legacy", composableFormat) + + result shouldBe a[ElasticSuccess[_]] + + // Verify it was stored as legacy + client.getTemplate("test-auto-convert-legacy") match { case ElasticSuccess(Some(json)) => - json should include("legacy-*") + json should include("order") + json should not include "priority" + json should not include "\"template\":" case _ => fail("Failed to get template") } + } + + it should "preserve order field in legacy templates" in { + assume(!supportComposableTemplates, "Test for legacy templates only") + + // Test with different orders + val orders = List(0, 1, 5, 10, 100) + + orders.foreach { order => + val templateName = s"test-order-$order" + val template = legacyTemplate(s"test-order-$order-*", order = order) + + // Check the JSON before creation + template should include(s""""order": $order""") + + // Create the template + val createResult = client.createTemplate(templateName, template) + createResult shouldBe a[ElasticSuccess[_]] + + // Retrieve and verify + client.getTemplate(templateName) match { + case ElasticSuccess(Some(json)) => + log.info(s"Template JSON for order=$order: $json") + + // Parse the JSON to check the order + val mapper = JacksonConfig.objectMapper + val root = mapper.readTree(json) + + if (root.has("order")) { + val actualOrder = root.get("order").asInt() + actualOrder shouldBe order + } else { + fail(s"Template does not contain 'order' field. JSON: $json") + } + + case other => fail(s"Failed to get template: $other") + } + + // Cleanup + client.deleteTemplate(templateName, ifExists = true) + } + } + + it should "update an existing template" in { + val template1 = if (supportComposableTemplates) { + composableTemplate("test-update-*", priority = 100) + } else { + legacyTemplate("test-update-*", order = 1) + } + + val template2 = if (supportComposableTemplates) { + composableTemplate("test-update-*", priority = 200) + } else { + legacyTemplate("test-update-*", order = 2, version = 2) + } + + // Create initial + val createResult = client.createTemplate("test-update", template1) + createResult shouldBe a[ElasticSuccess[_]] + + // Verify initial state + client.getTemplate("test-update") match { + case ElasticSuccess(Some(json)) => + info(s"Initial template: $json") + if (supportComposableTemplates) { + json should include("\"priority\":100") + } else { + json should include("\"order\":1") + json should include("\"version\":1") + } + case other => fail(s"Failed to get initial template: $other") + } + + // Update + val updateResult = client.createTemplate("test-update", template2) + updateResult shouldBe a[ElasticSuccess[_]] + + // Verify update + client.getTemplate("test-update") match { + case ElasticSuccess(Some(json)) => + info(s"Updated template: $json") + + if (supportComposableTemplates) { + json should include("\"priority\":200") + } else { + // Parse JSON to verify exact values + val mapper = JacksonConfig.objectMapper + val root = mapper.readTree(json) + + root.get("order").asInt() shouldBe 2 + root.get("version").asInt() shouldBe 2 + } + + case other => fail(s"Failed to get updated template: $other") + } // Cleanup - client.deleteTemplate("legacy-test", ifExists = true) + client.deleteTemplate("test-update", ifExists = true) } - "deleteTemplate" should "handle non-existing template with ifExists=true" in { - val result = client.deleteTemplate("non-existing", ifExists = true) - result shouldBe ElasticSuccess(false) + it should "reject invalid template name" in { + val invalidNames = Seq("", " ", "UPPERCASE", "with space", "_underscore", "-dash", "with/slash") + + invalidNames.foreach { name => + val result = client.createTemplate(name, composableTemplate("test-*")) + result shouldBe a[ElasticFailure] + } + } + + it should "reject invalid JSON definition" in { + val invalidJson = "{ invalid json }" + val result = client.createTemplate("test-invalid-json", invalidJson) + result shouldBe a[ElasticFailure] + } + + it should "reject template without index_patterns" in { + val noPatterns = """{"priority": 1, "template": {}}""" + val result = client.createTemplate("test-no-patterns", noPatterns) + result shouldBe a[ElasticFailure] + } + + // ========== GET TEMPLATE ========== + + it should "get an existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-get-*") + } else { + legacyTemplate("test-get-*") + } + + client.createTemplate("test-get", template) + + val result = client.getTemplate("test-get") + result match { + case ElasticSuccess(Some(json)) => + json should include("test-get-*") + json should include("number_of_shards") + case _ => fail("Failed to get template") + } } - "listTemplates" should "return all templates" in { - // Create test templates - client.createTemplate("list-1", """{"index_patterns":["test1-*"],"priority":1,"template":{}}""") - client.createTemplate("list-2", """{"index_patterns":["test2-*"],"priority":2,"template":{}}""") + it should "return None for non-existing template" in { + val result = client.getTemplate("non-existing-template") + result shouldBe ElasticSuccess(None) + } + + it should "reject invalid template name in get" in { + val result = client.getTemplate("") + result shouldBe a[ElasticFailure] + } + + // ========== LIST TEMPLATES ========== + + it should "list all templates" in { + val template1 = if (supportComposableTemplates) { + composableTemplate("test-list-1-*") + } else { + legacyTemplate("test-list-1-*") + } + + val template2 = if (supportComposableTemplates) { + composableTemplate("test-list-2-*") + } else { + legacyTemplate("test-list-2-*") + } + + client.createTemplate("test-list-1", template1) + client.createTemplate("test-list-2", template2) + + val result = client.listTemplates() + result match { + case ElasticSuccess(templates) => + templates.keys should contain allOf ("test-list-1", "test-list-2") + templates.values.foreach { json => + json should not be empty + } + case _ => fail("Failed to list templates") + } + } + + it should "return empty map when no templates exist" in { + // Cleanup all test templates + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.filter(_.startsWith("test-")).foreach(cleanupTemplate) + case _ => // Ignore + } - // List val result = client.listTemplates() result match { case ElasticSuccess(templates) => - templates.keys should contain allOf ("list-1", "list-2") + templates.keys.filter(_.startsWith("test-")) shouldBe empty + case _ => fail("Failed to list templates") + } + } + + // ========== TEMPLATE EXISTS ========== + + it should "return true for existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-exists-*") + } else { + legacyTemplate("test-exists-*") + } + + client.createTemplate("test-exists", template) + + val result = client.templateExists("test-exists") + result shouldBe ElasticSuccess(true) + } + + it should "return false for non-existing template" in { + val result = client.templateExists("non-existing-template") + result shouldBe ElasticSuccess(false) + } + + it should "reject invalid template name in exists" in { + val result = client.templateExists("") + result shouldBe a[ElasticFailure] + } + + // ========== DELETE TEMPLATE ========== + + it should "delete an existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-delete-*") + } else { + legacyTemplate("test-delete-*") + } + + client.createTemplate("test-delete", template) + client.templateExists("test-delete") shouldBe ElasticSuccess(true) + + val deleteResult = client.deleteTemplate("test-delete") + deleteResult shouldBe ElasticSuccess(true) + + client.templateExists("test-delete") shouldBe ElasticSuccess(false) + } + + it should "succeed when deleting non-existing template with ifExists=true" in { + val result = client.deleteTemplate("non-existing-template", ifExists = true) + result shouldBe ElasticSuccess(false) + } + + it should "fail when deleting non-existing template with ifExists=false" in { + val result = client.deleteTemplate("non-existing-template", ifExists = false) + // Comportement dépend de l'implémentation, mais devrait échouer ou retourner false + result match { + case ElasticSuccess(false) => succeed + case ElasticFailure(_) => succeed + case _ => fail("Should fail or return false") + } + } + + it should "reject invalid template name in delete" in { + val result = client.deleteTemplate("") + result shouldBe a[ElasticFailure] + } + + // ========== EDGE CASES ========== + + it should "handle template with complex mappings" in { + val complexTemplate = + s""" + |{ + | "index_patterns": ["test-complex-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 2, + | "number_of_replicas": 1, + | "refresh_interval": "30s" + | }, + | "mappings": { + | "properties": { + | "timestamp": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, + | "message": { "type": "text", "analyzer": "standard" }, + | "level": { "type": "keyword" }, + | "tags": { "type": "keyword" }, + | "metadata": { + | "type": "object", + | "properties": { + | "host": { "type": "keyword" }, + | "service": { "type": "keyword" } + | } + | } + | } + | }, + | "aliases": { + | "logs-all": {}, + | "logs-recent": { + | "filter": { + | "range": { + | "timestamp": { + | "gte": "now-7d" + | } + | } + | } + | } + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-complex", complexTemplate) + result shouldBe a[ElasticSuccess[_]] + + // Verify + client.getTemplate("test-complex") match { + case ElasticSuccess(Some(json)) => + json should include("test-complex-*") + json should include("metadata") + json should include("logs-all") + case _ => fail("Failed to get complex template") + } + } + + it should "handle template with multiple index patterns" in { + val multiPatternTemplate = + s""" + |{ + | "index_patterns": ["test-multi-1-*", "test-multi-2-*", "test-multi-3-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 1 + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-multi-pattern", multiPatternTemplate) + result shouldBe a[ElasticSuccess[_]] + + client.getTemplate("test-multi-pattern") match { + case ElasticSuccess(Some(json)) => + json should include("test-multi-1-*") + json should include("test-multi-2-*") + json should include("test-multi-3-*") + case _ => fail("Failed to get multi-pattern template") + } + } + + it should "handle template with version and metadata" in { + val versionedTemplate = + s""" + |{ + | "index_patterns": ["test-versioned-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | "version": 42, + | ${if (supportComposableTemplates) "\"_meta\": {" else ""} + | ${if (supportComposableTemplates) "\"description\": \"Test versioned template\"," + else ""} + | ${if (supportComposableTemplates) "\"author\": \"test-suite\"" else ""} + | ${if (supportComposableTemplates) "}," else ""} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 1 + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-versioned", versionedTemplate) + result shouldBe a[ElasticSuccess[_]] + + client.getTemplate("test-versioned") match { + case ElasticSuccess(Some(json)) => + json should include("\"version\":42") + if (supportComposableTemplates) { + json should include("_meta") + } + case _ => fail("Failed to get versioned template") + } + } + + // ========== CONCURRENT OPERATIONS ========== + + it should "handle concurrent template operations" in { + import scala.concurrent.Future + + val futures = (1 to 5).map { i => + Future { + val template = if (supportComposableTemplates) { + composableTemplate(s"test-concurrent-$i-*") + } else { + legacyTemplate(s"test-concurrent-$i-*") + } + client.createTemplate(s"test-concurrent-$i", template) + } + } + + val results = Future.sequence(futures).futureValue + results.foreach { result => + result shouldBe a[ElasticSuccess[_]] + } + + // Verify all were created + val listResult = client.listTemplates() + listResult match { + case ElasticSuccess(templates) => + (1 to 5).foreach { i => + templates.keys should contain(s"test-concurrent-$i") + } case _ => fail("Failed to list templates") } // Cleanup - client.deleteTemplate("list-1", ifExists = true) - client.deleteTemplate("list-2", ifExists = true) + (1 to 5).foreach { i => + client.deleteTemplate(s"test-concurrent-$i", ifExists = true) + } + } + + // ========== PERFORMANCE ========== + + it should "handle bulk template operations efficiently" in { + val startTime = System.currentTimeMillis() + + // Create 20 templates + (1 to 20).foreach { i => + val template = if (supportComposableTemplates) { + composableTemplate(s"test-bulk-$i-*") + } else { + legacyTemplate(s"test-bulk-$i-*") + } + client.createTemplate(s"test-bulk-$i", template) + } + + // List all + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.count(_.startsWith("test-bulk-")) should be >= 20 + case _ => fail("Failed to list templates") + } + + // Delete all + (1 to 20).foreach { i => + client.deleteTemplate(s"test-bulk-$i", ifExists = true) + } + + val duration = System.currentTimeMillis() - startTime + log.info(s"Bulk operations completed in ${duration}ms") + + duration should be < 30000L // Should complete in less than 30 seconds } + } From bee01d170dff4743f4bbd36d402c19e66d504fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 22 Dec 2025 14:28:13 +0100 Subject: [PATCH 28/95] add template api documentation --- README.md | 10 +- documentation/client/README.md | 1 + documentation/client/templates.md | 1601 +++++++++++++++++ .../elastic/client/java/JavaClientApi.scala | 6 +- .../elastic/client/java/JavaClientApi.scala | 6 +- 5 files changed, 1615 insertions(+), 9 deletions(-) create mode 100644 documentation/client/templates.md diff --git a/README.md b/README.md index abef34b9..a2b953b5 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ SoftClient4ES provides a trait-based interface (`ElasticClientApi`) that aggrega | **[SearchApi](documentation/client/search.md)** | Advanced search with SQL and aggregations support | [📖 Docs](documentation/client/search.md) | | **[ScrollApi](documentation/client/scroll.md)** | Stream large datasets with automatic strategy detection (PIT, search_after, scroll) | [📖 Docs](documentation/client/scroll.md) | | **[AggregationApi](documentation/client/aggregations.md)** | Type-safe way to execute aggregations using SQL queries | [📖 Docs](documentation/client/aggregations.md) | +| **[TemplateApi](documentation/client/templates.md)** | Templates management | [📖 Docs](documentation/client/templates.md) | #### **Client Implementations** @@ -212,7 +213,7 @@ SoftClient4ES includes a powerful SQL parser that translates standard SQL `SELEC - ✅ Aggregate functions (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `DISTINCT`, `FIRST_VALUE`, `LAST_VALUE`, `ARRAY_AGG`) - ✅ [Window functions](#32-window-functions-support) with `OVER` clause - ✅ [DML Support](#34-dml-support) (`INSERT`, `UPDATE`, `DELETE`) -- ✅ [DDL Support](#35-ddl-support) (`CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`) +- ✅ [DDL Support](#35-ddl-support) (`CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `CREATE PIPELINE`, `ALTER PIPELINE`, `DROP PIPELINE`) **Example:** @@ -1236,8 +1237,8 @@ The execution path depends on the **number of impacted rows**: SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to manage table schemas mapped to Elasticsearch indices. #### **Supported DDL Statements** -- ✅ `CREATE TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, `FIELDS` (multi‑fields or STRUCT) and `PARTITION BY …` -- ✅ `CREATE OR REPLACE TABLE … AS SELECT …` +- ✅ `CREATE [OR REPLACE] TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, `FIELDS` (multi‑fields or STRUCT) and `PARTITION BY …` +- ✅ `CREATE [OR REPLACE] TABLE … AS SELECT …` - ✅ `ALTER TABLE …` with multiple sub‑statements: - `ADD COLUMN [IF NOT EXISTS] …` - `DROP COLUMN [IF EXISTS] …` @@ -1249,6 +1250,9 @@ SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to - `ALTER COLUMN [IF EXISTS] … SET FIELDS (…)` (define nested STRUCT or multi‑fields) - ✅ `DROP TABLE [IF EXISTS] … [CASCADE]` - ✅ `TRUNCATE TABLE …` +- ✅ `CREATE [OR REPLACE] PIPELINE [IF NOT EXISTS] … WITH PROCESSORS (…)` +- ✅ `ALTER PIPELINE … [(]ADD|DROP PROCESSOR …[)]` +- ✅ `DROP PIPELINE [IF EXISTS] …` **Examples:** ```sql diff --git a/documentation/client/README.md b/documentation/client/README.md index fc37f028..5d4893c0 100644 --- a/documentation/client/README.md +++ b/documentation/client/README.md @@ -18,3 +18,4 @@ Welcome to the Client Engine Documentation. Navigate through the sections below: - [Search Documents](search.md) - [Scroll Search](scroll.md) - [Aggregations](aggregations.md) +- [Template Management](templates.md) diff --git a/documentation/client/templates.md b/documentation/client/templates.md new file mode 100644 index 00000000..55b97726 --- /dev/null +++ b/documentation/client/templates.md @@ -0,0 +1,1601 @@ +# TEMPLATE API + +## Overview + +The **TemplateApi** trait provides comprehensive functionality to manage Elasticsearch index templates. It supports both **legacy templates** (ES 6.x-7.x) and **composable (index) templates** (ES 7.8+), with automatic version detection and conversion capabilities. + +Index templates define settings, mappings, and aliases that are automatically applied to new indices matching specified patterns. + +**Dependencies:** Extends `ElasticClientHelpers` for validation and logging utilities. + +--- + +## Public Methods + +### createTemplate + +Creates or updates an index template with the specified configuration. + +**Signature:** + +```scala +def createTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to create or update +- `templateDefinition` - JSON string containing the template configuration + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template was created/updated successfully +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution (returns 400 error if invalid) +- Automatically detects template type (composable vs legacy) based on JSON structure +- Converts between formats if necessary based on Elasticsearch version +- For ES < 6.8: Uses legacy template format with `_doc` wrapper in mappings +- For ES 7.8+: Supports both legacy and composable templates +- Logs success with ✅ or failure with ❌ + +**Examples:** + +```scala +// Legacy template (ES 6.x-7.x) +val legacyTemplate = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "version": 1, + | "settings": { + | "number_of_shards": 2, + | "number_of_replicas": 1 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": {"type": "date"}, + | "message": {"type": "text"} + | } + | } + | }, + | "aliases": { + | "logs-current": {} + | } + |} + |""".stripMargin + +client.createTemplate("logs-template", legacyTemplate) match { + case ElasticSuccess(true) => println("Template created") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Composable template (ES 7.8+) +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 200, + | "version": 2, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "refresh_interval": "30s" + | }, + | "mappings": { + | "properties": { + | "cpu_usage": {"type": "float"}, + | "timestamp": {"type": "date"} + | } + | }, + | "aliases": { + | "metrics-latest": { + | "is_write_index": true + | } + | } + | }, + | "composed_of": ["component-template-1"], + | "_meta": { + | "description": "Metrics template" + | } + |} + |""".stripMargin + +client.createTemplate("metrics-template", composableTemplate) + +// Update existing template +val updatedTemplate = legacyTemplate.replace("\"order\": 1", "\"order\": 10") +client.createTemplate("logs-template", updatedTemplate) // Overwrites existing + +// Monadic chaining +for { + _ <- client.createTemplate("users-template", userTemplate) + _ <- client.createIndex("users-2024") + result <- client.search("users-2024", query) +} yield result +``` + +**Template Type Detection:** + +The API automatically detects template type based on JSON structure: + +| Field Present | Template Type | Supported Versions | +|-------------------------------|----------------|---------------------| +| `template` (object) | Composable | ES 7.8+ | +| `index_patterns` (root level) | Legacy | ES 6.x-7.x | +| `mappings` (root level) | Legacy | ES 6.x-7.x | + +--- + +### getTemplate + +Retrieves the configuration of one or more index templates. + +**Signature:** + +```scala +def getTemplate(templateName: String): ElasticResult[Option[String]] +``` + +**Parameters:** +- `templateName` - The name of the template to retrieve (supports wildcards) + +**Returns:** +- `ElasticSuccess[Some(String)]` with JSON template configuration if found +- `ElasticSuccess[None]` if template does not exist +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- Returns normalized JSON format regardless of template type +- For composable templates: Returns full template structure +- For legacy templates: Returns converted format with all fields +- Supports wildcard patterns (e.g., `logs-*`) +- Logs success with ✅ or failure with ❌ + +**Examples:** + +```scala +// Retrieve single template +client.getTemplate("logs-template") match { + case ElasticSuccess(Some(json)) => + println(s"Template found: $json") + case ElasticSuccess(None) => + println("Template not found") + case ElasticFailure(e) => + println(s"Error: ${e.message}") +} + +// Check if template exists +def templateExists(name: String): Boolean = { + client.getTemplate(name) match { + case ElasticSuccess(Some(_)) => true + case _ => false + } +} + +// Retrieve with wildcard +client.getTemplate("logs-*") match { + case ElasticSuccess(Some(json)) => + // Returns all matching templates + println(s"Found templates: $json") + case _ => println("No matching templates") +} + +// Parse template configuration +import com.fasterxml.jackson.databind.ObjectMapper + +client.getTemplate("users-template") match { + case ElasticSuccess(Some(json)) => + val mapper = new ObjectMapper() + val root = mapper.readTree(json) + + val patterns = root.get("index_patterns") + val order = root.get("order").asInt() + + println(s"Patterns: $patterns, Order: $order") + + case _ => println("Template not found") +} + +// Conditional operations +for { + existing <- client.getTemplate("my-template") + result <- existing match { + case Some(_) => + client.deleteTemplate("my-template", ifExists = true) + case None => + ElasticResult.success(true) + } + _ <- client.createTemplate("my-template", newTemplate) +} yield result +``` + +--- + +### deleteTemplate + +Deletes one or more index templates. + +**Signature:** + +```scala +def deleteTemplate( + templateName: String, + ifExists: Boolean = false +): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to delete (supports wildcards) +- `ifExists` - If `true`, returns success even if template doesn't exist (default: `false`) + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template was deleted, `false` if it didn't exist (when `ifExists = true`) +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- When `ifExists = true`: Checks existence before deletion, returns success if not found +- When `ifExists = false`: Returns failure if template doesn't exist +- Supports wildcard patterns for bulk deletion +- Logs success with ✅ or failure with ❌ + +**Examples:** + +```scala +// Delete single template +client.deleteTemplate("logs-template") match { + case ElasticSuccess(true) => println("Template deleted") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Safe deletion (no error if missing) +client.deleteTemplate("old-template", ifExists = true) + +// Delete multiple templates with wildcard +client.deleteTemplate("test-*", ifExists = true) + +// Cleanup after tests +def cleanupTemplates(names: List[String]): Unit = { + names.foreach { name => + client.deleteTemplate(name, ifExists = true) + } +} + +// Conditional deletion +client.getTemplate("temporary-template") match { + case ElasticSuccess(Some(_)) => + client.deleteTemplate("temporary-template") + case _ => + ElasticResult.success(true) +} + +// Monadic cleanup +for { + _ <- client.deleteTemplate("old-users-template", ifExists = true) + _ <- client.createTemplate("users-template", newTemplate) + _ <- client.createIndex("users-2024") +} yield () +``` + +--- + +### templateExists + +Checks if an index template exists. + +**Signature:** + +```scala +def templateExists(templateName: String): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to check + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template exists, `false` otherwise +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- Efficiently checks existence without retrieving full template +- Works with both legacy and composable templates +- Logs debug information + +**Examples:** + +```scala +// Simple existence check +client.templateExists("logs-template") match { + case ElasticSuccess(true) => println("Template exists") + case ElasticSuccess(false) => println("Template not found") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Conditional creation +def createTemplateIfMissing(name: String, definition: String): ElasticResult[Boolean] = { + client.templateExists(name).flatMap { + case true => + println(s"Template '$name' already exists") + ElasticResult.success(false) + case false => + client.createTemplate(name, definition) + } +} + +// Guard clause pattern +for { + exists <- client.templateExists("users-template") + _ <- if (!exists) { + client.createTemplate("users-template", userTemplate) + } else { + ElasticResult.success(true) + } + result <- client.createIndex("users-2024") +} yield result + +// Bulk existence check +val templates = List("logs", "metrics", "traces") +val existenceMap = templates.map { name => + name -> client.templateExists(name) +}.toMap +``` + +--- + +## Template Types + +### Legacy Templates (ES 6.x - 7.x) + +Legacy templates apply to indices matching specified patterns. + +**Structure:** + +``` +{ + "index_patterns": ["pattern-*"], + "order": 0, + "version": 1, + "settings": { ... }, + "mappings": { ... }, + "aliases": { ... } +} +``` + +**Key Fields:** + +| Field | Type | Description | +|------------------|---------|------------------------------------------------------| +| `index_patterns` | Array | Index patterns to match (e.g., `["logs-*"]`) | +| `order` | Integer | Priority when multiple templates match (higher wins) | +| `version` | Integer | Optional version number for tracking | +| `settings` | Object | Index settings (shards, replicas, etc.) | +| `mappings` | Object | Field mappings and types | +| `aliases` | Object | Index aliases to create | + +**Example:** + +```scala +val legacyTemplate = + """ + |{ + | "index_patterns": ["app-logs-*"], + | "order": 10, + | "version": 3, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": 2, + | "refresh_interval": "5s" + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": { + | "type": "date", + | "format": "strict_date_optional_time||epoch_millis" + | }, + | "level": { + | "type": "keyword" + | }, + | "message": { + | "type": "text", + | "fields": { + | "keyword": { + | "type": "keyword", + | "ignore_above": 256 + | } + | } + | }, + | "user_id": { + | "type": "keyword" + | } + | } + | } + | }, + | "aliases": { + | "app-logs-current": {}, + | "app-logs-errors": { + | "filter": { + | "term": { + | "level": "ERROR" + | } + | } + | } + | } + |} + |""".stripMargin + +client.createTemplate("app-logs-template", legacyTemplate) +``` + +--- + +### Composable Templates (ES 7.8+) + +Composable templates provide more flexibility with component templates and better composition. + +**Structure:** + +``` +{ + "index_patterns": ["pattern-*"], + "priority": 100, + "version": 1, + "template": { + "settings": { ... }, + "mappings": { ... }, + "aliases": { ... } + }, + "composed_of": ["component1", "component2"], + "_meta": { ... } +} +``` + +**Key Fields:** + +| Field | Type | Description | +|------------------|---------|--------------------------------------| +| `index_patterns` | Array | Index patterns to match | +| `priority` | Integer | Template priority (higher wins) | +| `version` | Integer | Optional version number | +| `template` | Object | Contains settings, mappings, aliases | +| `composed_of` | Array | Component templates to include | +| `_meta` | Object | Optional metadata | + +**Example:** + +```scala +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 500, + | "version": 2, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 1, + | "refresh_interval": "30s", + | "codec": "best_compression" + | }, + | "mappings": { + | "properties": { + | "timestamp": { + | "type": "date" + | }, + | "cpu_percent": { + | "type": "float" + | }, + | "memory_used": { + | "type": "long" + | }, + | "host": { + | "type": "keyword" + | } + | } + | }, + | "aliases": { + | "metrics-latest": { + | "is_write_index": true + | }, + | "metrics-high-cpu": { + | "filter": { + | "range": { + | "cpu_percent": { + | "gte": 80 + | } + | } + | } + | } + | } + | }, + | "composed_of": [ + | "metrics-component-settings", + | "metrics-component-mappings" + | ], + | "_meta": { + | "description": "Template for system metrics", + | "managed_by": "monitoring-system", + | "version": "2.0" + | } + |} + |""".stripMargin + +client.createTemplate("metrics-template", composableTemplate) +``` + +--- + +## Template Priority and Matching + +When multiple templates match an index pattern, priority determines which template applies: + +### Legacy Templates (`order`) + +```scala +// Lower order +val template1 = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + +// Higher order wins +val template2 = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 10, + | "settings": { + | "number_of_shards": 3 + | } + |} + |""".stripMargin + +client.createTemplate("logs-default", template1) +client.createTemplate("logs-production", template2) + +// New index "logs-2024" will have 3 shards (template2 wins) +``` + +### Composable Templates (`priority`) + +```scala +// Lower priority +val template1 = + """ + |{ + | "index_patterns": ["app-*"], + | "priority": 100, + | "template": { + | "settings": { + | "number_of_replicas": 1 + | } + | } + |} + |""".stripMargin + +// Higher priority wins +val template2 = + """ + |{ + | "index_patterns": ["app-*"], + | "priority": 200, + | "template": { + | "settings": { + | "number_of_replicas": 2 + | } + | } + |} + |""".stripMargin + +client.createTemplate("app-default", template1) +client.createTemplate("app-production", template2) + +// New index "app-users" will have 2 replicas (template2 wins) +``` + +--- + +## Common Use Cases + +### 1. Time-Series Data + +```scala +val timeSeriesTemplate = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": 1, + | "refresh_interval": "5s", + | "index.lifecycle.name": "logs-policy", + | "index.lifecycle.rollover_alias": "logs-current" + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "@timestamp": { + | "type": "date" + | }, + | "message": { + | "type": "text" + | }, + | "level": { + | "type": "keyword" + | } + | } + | } + | }, + | "aliases": { + | "logs-all": {} + | } + |} + |""".stripMargin + +client.createTemplate("logs-template", timeSeriesTemplate) + +// Indices created with date pattern automatically use template +client.createIndex("logs-2024-01-01") +client.createIndex("logs-2024-01-02") +``` + +### 2. Multi-Environment Templates + +```scala +def createEnvironmentTemplate(env: String, order: Int): String = { + val replicas = env match { + case "production" => 2 + case "staging" => 1 + case _ => 0 + } + + s""" + |{ + | "index_patterns": ["${env}-*"], + | "order": $order, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": $replicas + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "environment": { + | "type": "keyword" + | } + | } + | } + | } + |} + |""".stripMargin +} + +client.createTemplate("production-template", createEnvironmentTemplate("production", 100)) +client.createTemplate("staging-template", createEnvironmentTemplate("staging", 50)) +client.createTemplate("development-template", createEnvironmentTemplate("development", 10)) +``` + +### 3. Dynamic Mapping with Templates + +```scala +val dynamicTemplate = + """ + |{ + | "index_patterns": ["dynamic-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | }, + | "mappings": { + | "_doc": { + | "dynamic_templates": [ + | { + | "strings_as_keywords": { + | "match_mapping_type": "string", + | "mapping": { + | "type": "keyword" + | } + | } + | }, + | { + | "longs_as_integers": { + | "match_mapping_type": "long", + | "mapping": { + | "type": "integer" + | } + | } + | } + | ], + | "properties": { + | "timestamp": { + | "type": "date" + | } + | } + | } + | } + |} + |""".stripMargin + +client.createTemplate("dynamic-template", dynamicTemplate) +``` + +### 4. Filtered Aliases + +```scala +val templateWithFilteredAliases = + """ + |{ + | "index_patterns": ["events-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 2 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "event_type": {"type": "keyword"}, + | "severity": {"type": "keyword"}, + | "timestamp": {"type": "date"} + | } + | } + | }, + | "aliases": { + | "events-all": {}, + | "events-errors": { + | "filter": { + | "term": { + | "severity": "error" + | } + | } + | }, + | "events-warnings": { + | "filter": { + | "term": { + | "severity": "warning" + | } + | } + | }, + | "events-critical": { + | "filter": { + | "terms": { + | "severity": ["critical", "fatal"] + | } + | }, + | "routing": "1" + | } + | } + |} + |""".stripMargin + +client.createTemplate("events-template", templateWithFilteredAliases) + +// Search specific severity levels through aliases +client.search("events-errors", query) +client.search("events-critical", query) +``` + +--- + +## Version Compatibility + +### Automatic Format Normalization + +The API automatically normalizes template format based on Elasticsearch version: + +```scala +def normalizeTemplate(templateJson: String, elasticVersion: String): Try[String] +``` + +**Normalization Logic:** + +| ES Version | Input Format | Action Taken | +|------------|-------------|--------------| +| **ES 7.8+** | Legacy | Convert to Composable | +| **ES 7.8+** | Composable | Keep as-is | +| **ES 6.8 - 7.7** | Legacy | Keep as-is | +| **ES 6.8 - 7.7** | Composable | Convert to Legacy | +| **ES < 6.8** | Legacy | Ensure `_doc` wrapper in mappings | +| **ES < 6.8** | Composable | Convert to Legacy + ensure `_doc` wrapper | + +--- + +### Type Name Handling + +#### ES < 6.8: Requires `_doc` Type Wrapper + +For Elasticsearch versions **before 6.8**, mappings **must** be wrapped in a type name (typically `_doc`): + +**Input (Typeless Mappings):** + +```json +{ + "index_patterns": ["logs-*"], + "mappings": { + "properties": { + "message": {"type": "text"} + } + } +} +``` + +**Normalized Output (ES < 6.8):** + +```json +{ + "index_patterns": ["logs-*"], + "mappings": { + "_doc": { + "properties": { + "message": {"type": "text"} + } + } + } +} +``` + +**Code Example:** + +```scala +// ES < 6.8 +val template = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "mappings": { + | "properties": { + | "timestamp": {"type": "date"}, + | "message": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// API automatically wraps mappings in _doc for ES < 6.8 +client.createTemplate("logs-template", template) + +// Resulting template in ES < 6.8: +// { +// "index_patterns": ["logs-*"], +// "order": 1, +// "mappings": { +// "_doc": { +// "properties": { +// "timestamp": {"type": "date"}, +// "message": {"type": "text"} +// } +// } +// } +// } +``` + +--- + +#### ES 6.8 - 7.x: Optional Type Names + +Elasticsearch 6.8+ supports typeless mappings but still accepts `_doc` wrapper for backward compatibility: + +**Both formats work:** + +```scala +// Format 1: With _doc wrapper (backward compatible) +val templateWithType = + """ + |{ + | "index_patterns": ["logs-*"], + | "mappings": { + | "_doc": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + |} + |""".stripMargin + +// Format 2: Typeless (modern) +val templateTypeless = + """ + |{ + | "index_patterns": ["logs-*"], + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// Both work in ES 6.8-7.x +client.createTemplate("template1", templateWithType) +client.createTemplate("template2", templateTypeless) +``` + +--- + +#### ES 7.8+: Typeless Only (Composable Templates) + +Composable templates in ES 7.8+ **do not support type names**: + +```scala +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "properties": { + | "cpu": {"type": "float"} + | } + | } + | } + |} + |""".stripMargin + +// No _doc wrapper needed +client.createTemplate("metrics-template", composableTemplate) +``` + +--- + +### Composable → Legacy Conversion + +When using composable templates on **ES < 7.8**, the API automatically converts to legacy format: + +**Input (Composable Format):** + +```json +{ + "index_patterns": ["app-*"], + "priority": 200, + "version": 2, + "template": { + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "properties": { + "user_id": {"type": "keyword"} + } + }, + "aliases": { + "app-current": {} + } + }, + "composed_of": ["component1"], + "_meta": { + "description": "App template" + } +} +``` + +**Normalized Output (ES 6.8):** + +```json +{ + "index_patterns": ["app-*"], + "order": 200, + "version": 2, + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "_doc": { + "properties": { + "user_id": {"type": "keyword"} + } + } + }, + "aliases": { + "app-current": {} + } +} +``` + +**Conversion Rules:** + +| Composable Field | Legacy Field | Notes | +|-----------------|--------------|-------| +| `priority` | `order` | Direct value mapping | +| `template.settings` | `settings` | Flattened to root level | +| `template.mappings` | `mappings` | Flattened + wrapped in `_doc` for ES < 6.8 | +| `template.aliases` | `aliases` | Flattened to root level | +| `composed_of` | ❌ Ignored | Not supported in legacy format (warning logged) | +| `_meta` | ❌ Ignored | Not supported in legacy format | +| `data_stream` | ❌ Ignored | Not supported in legacy format (warning logged) | + +**Code Example:** + +```scala +// Using composable format on ES 6.8 +val composableTemplate = + """ + |{ + | "index_patterns": ["events-*"], + | "priority": 150, + | "template": { + | "settings": { + | "number_of_shards": 2 + | }, + | "mappings": { + | "properties": { + | "event_type": {"type": "keyword"} + | } + | } + | }, + | "composed_of": ["base-settings"], + | "_meta": { + | "owner": "analytics-team" + | } + |} + |""".stripMargin + +// API automatically converts for ES 6.8: +// - priority → order +// - Flattens template object +// - Wraps mappings in _doc +// - Logs warnings for composed_of and _meta +client.createTemplate("events-template", composableTemplate) + +// Logs: +// [WARN] Composable templates are not supported in legacy template format and will be ignored +// [DEBUG] Wrapped mappings in '_doc' for ES 6.8.0 +``` + +--- + +### Legacy → Composable Conversion + +When using legacy templates on **ES 7.8+**, the API can optionally convert to composable format: + +**Input (Legacy Format):** + +```json +{ + "index_patterns": ["logs-*"], + "order": 10, + "version": 1, + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "_doc": { + "properties": { + "message": {"type": "text"} + } + } + }, + "aliases": { + "logs-all": {} + } +} +``` + +**Normalized Output (ES 7.8+):** + +```json +{ + "index_patterns": ["logs-*"], + "priority": 10, + "version": 1, + "template": { + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "properties": { + "message": {"type": "text"} + } + }, + "aliases": { + "logs-all": {} + } + } +} +``` + +**Conversion Rules:** + +| Legacy Field | Composable Field | Notes | +|-------------|------------------|-------| +| `order` | `priority` | Direct value mapping | +| `settings` | `template.settings` | Nested under `template` | +| `mappings` | `template.mappings` | Nested + `_doc` wrapper removed | +| `aliases` | `template.aliases` | Nested under `template` | + +--- + +### Version Detection Helpers + +The API uses internal helpers to determine version-specific behavior: + +```scala +object ElasticsearchVersion { + + /** Check if ES version supports composable templates (7.8+) + */ + def supportsComposableTemplates(version: String): Boolean = { + // Returns true for ES >= 7.8 + } + + /** Check if ES version requires _doc type wrapper (< 6.8) + */ + def requiresDocTypeWrapper(version: String): Boolean = { + // Returns true for ES < 6.8 + } + + /** Check if ES version requires include_type_name parameter (6.8 - 7.x) + */ + def requiresIncludeTypeName(version: String): Boolean = { + // Returns true for ES 6.8 - 7.x + } +} +``` + +**Usage Example:** + +```scala +def createTemplateForVersion( + name: String, + template: String, + esVersion: String +): ElasticResult[Boolean] = { + + // Normalize based on version + val normalized = TemplateConverter.normalizeTemplate(template, esVersion) match { + case Success(json) => json + case Failure(e) => return ElasticResult.failure(e.getMessage) + } + + // Version-specific handling + if (ElasticsearchVersion.supportsComposableTemplates(esVersion)) { + println("Using composable template API") + executeCreateComposableTemplate(name, normalized) + } else if (ElasticsearchVersion.requiresIncludeTypeName(esVersion)) { + println("Using legacy template API with include_type_name") + executeCreateLegacyTemplateWithTypeName(name, normalized) + } else { + println("Using legacy template API") + executeCreateLegacyTemplate(name, normalized) + } +} +``` + +--- + +### Complete Version Matrix + +| ES Version | Template Type | Mappings Format | `include_type_name` | API Endpoint | +|---------------|------------------------|-------------------------|----------------------|----------------------------------------------------------| +| **6.0 - 6.7** | Legacy | `_doc` wrapper required | ❌ Not supported | `PUT /_template/{name}` | +| **6.8 - 7.7** | Legacy | Typeless or `_doc` | ✅ Optional | `PUT /_template/{name}` | +| **7.8+** | Legacy or Composable | Typeless only | ❌ Deprecated | `PUT /_template/{name}` or `PUT /_index_template/{name}` | +| **8.0+** | Composable (preferred) | Typeless only | ❌ Removed | `PUT /_index_template/{name}` | + +--- + +### Troubleshooting Version Issues + +#### Issue: Template creation fails with "unknown field [_doc]" + +**Cause:** Using `_doc` wrapper on ES 7.8+ composable templates + +**Solution:** + +```scala +// The API automatically unwrap _doc wrapper for ES >= 6.8 +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "_doc": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + | } + |} + |""".stripMargin + +// ✅ Correct for ES 6.8+ +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + |} + |""".stripMargin +``` + +#### Issue: Template creation fails with "missing type name" + +**Cause:** Missing `_doc` wrapper on ES < 6.8 + +**Solution :** + +```scala +// The API automatically adds _doc wrapper for ES < 6.8 +// No manual action needed - just use typeless format +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// API automatically wraps for ES < 6.8 +client.createTemplate("test-template", template) +``` + +#### Issue: Warning "Composable templates are not supported" + +**Cause:** Using `composed_of` or `data_stream` on ES < 7.8 + +**Solution :** + +```scala +// These fields are automatically ignored with warning +// Remove them for ES < 7.8 to avoid confusion +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + +// No composed_of or _meta fields +client.createTemplate("test-template", template) +``` + +--- + +## Error Handling + +### Invalid Template Name + +```scala +client.createTemplate("", templateDefinition) match { + case ElasticFailure(error) => + assert(error.statusCode.contains(400)) + assert(error.operation.contains("createTemplate")) + assert(error.message.contains("Invalid")) +} +``` + +### Invalid JSON + +```scala +val invalidJson = """{"index_patterns": ["test-*"}""" // Missing closing brace + +client.createTemplate("test", invalidJson) match { + case ElasticFailure(error) => + println(s"JSON parsing error: ${error.message}") +} +``` + +### Template Not Found + +```scala +client.getTemplate("non-existent") match { + case ElasticSuccess(None) => + println("Template not found") + case ElasticSuccess(Some(json)) => + println(s"Template: $json") + case ElasticFailure(e) => + println(s"Error: ${e.message}") +} +``` + +### Conflicting Templates + +```scala +// Template with same pattern but different order +val template1 = """{"index_patterns": ["app-*"], "order": 1}""" +val template2 = """{"index_patterns": ["app-*"], "order": 10}""" + +client.createTemplate("template1", template1) +client.createTemplate("template2", template2) + +// template2 will take precedence due to higher order +``` + +--- + +## Performance Considerations + +### Template Application + +⚠️ Templates are applied **only when indices are created**, not to existing indices. + +```scala +// ❌ Bad - template won't affect existing index +client.createIndex("logs-2024") +client.createTemplate("logs-template", template) // Too late! + +// ✅ Good - create template first +client.createTemplate("logs-template", template) +client.createIndex("logs-2024") // Template applied +``` + +### Bulk Template Creation + +```scala +// ✅ Create templates before bulk indexing +val templates = Map( + "logs-template" -> logsTemplate, + "metrics-template" -> metricsTemplate, + "events-template" -> eventsTemplate +) + +templates.foreach { case (name, definition) => + client.createTemplate(name, definition, ifExists = true) +} + +// Now bulk create indices +val indices = List("logs-2024", "metrics-2024", "events-2024") +indices.foreach(client.createIndex) +``` + +### Template Caching + +```scala +// Templates are cached by Elasticsearch +// Frequent updates can impact performance + +// ❌ Bad - updating template repeatedly +(1 to 100).foreach { i => + val updated = template.replace("\"version\": 1", s""""version": $i""") + client.createTemplate("my-template", updated) +} + +// ✅ Good - update once with final configuration +client.createTemplate("my-template", finalTemplate) +``` + +--- + +## Implementation Requirements + +### executeCreateLegacyTemplate + +Must be implemented by each client-specific trait for legacy templates. + +**Signature:** + +```scala +private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**REST High Level Client (ES 6-7):** + +```scala +private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) +} +``` + +### executeCreateComposableTemplate + +Must be implemented for composable templates (ES 7.8+). + +**Signature:** + +```scala +private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**Java Client (ES 8+):** + +```scala +private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "createTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) +} +``` + +### executeGetLegacyTemplate / executeGetComposableTemplate + +Retrieves legacy / composable template configuration. + +**Signature:** + +```scala +private[client] def executeGetLegacyTemplate( + templateName: String +): ElasticResult[Option[String]] +``` + +### executeDeleteLegacyTemplate / executeDeleteComposableTemplate + +Deletes legacy / composable template. + +**Signature:** + +```scala +private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean +): ElasticResult[Boolean] +``` + +### executeListLegacyTemplates / executeListComposableTemplates + +Retrieves legacy / composable template configurations. + +**Signature:** + +```scala +private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] +``` + +--- + +## Testing Examples + +### Complete Test Suite + +```scala +class TemplateApiSpec extends AnyFlatSpec with Matchers { + + val client: ElasticClient = // Initialize client + + "createTemplate" should "create a legacy template" in { + val template = + """ + |{ + | "index_patterns": ["test-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + + client.createTemplate("test-template", template) shouldBe a[ElasticSuccess[_]] + + client.templateExists("test-template") match { + case ElasticSuccess(true) => succeed + case _ => fail("Template not created") + } + + client.deleteTemplate("test-template", ifExists = true) + } + + it should "update an existing template" in { + val template1 = """{"index_patterns": ["update-*"], "order": 1}""" + val template2 = """{"index_patterns": ["update-*"], "order": 2, "version": 2}""" + + client.createTemplate("update-test", template1) + client.createTemplate("update-test", template2) + + client.getTemplate("update-test") match { + case ElasticSuccess(Some(json)) => + json should include("\"order\":2") + json should include("\"version\":2") + case _ => fail("Failed to get updated template") + } + + client.deleteTemplate("update-test", ifExists = true) + } + + "getTemplate" should "return None for non-existent template" in { + client.getTemplate("non-existent-template") match { + case ElasticSuccess(None) => succeed + case _ => fail("Should return None") + } + } + + "deleteTemplate" should "delete existing template" in { + val template = """{"index_patterns": ["delete-*"], "order": 1}""" + + client.createTemplate("delete-test", template) + client.deleteTemplate("delete-test") shouldBe a[ElasticSuccess[_]] + + client.templateExists("delete-test") match { + case ElasticSuccess(false) => succeed + case _ => fail("Template not deleted") + } + } + + it should "succeed with ifExists=true for non-existent template" in { + client.deleteTemplate("non-existent", ifExists = true) match { + case ElasticSuccess(false) => succeed + case _ => fail("Should succeed with ifExists=true") + } + } + + "templateExists" should "return true for existing template" in { + val template = """{"index_patterns": ["exists-*"], "order": 1}""" + + client.createTemplate("exists-test", template) + + client.templateExists("exists-test") match { + case ElasticSuccess(true) => succeed + case _ => fail("Template should exist") + } + + client.deleteTemplate("exists-test", ifExists = true) + } + + it should "return false for non-existent template" in { + client.templateExists("non-existent") match { + case ElasticSuccess(false) => succeed + case _ => fail("Template should not exist") + } + } +} +``` + +--- + +[Back to index](README.md) | [Next: Component Templates](component-templates.md) \ No newline at end of file diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index da65820a..f81045bd 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1599,7 +1599,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java templateDefinition: String ): ElasticResult[Boolean] = executeJavaBooleanAction( - operation = "createComposableTemplate", + operation = "createTemplate", retryable = false )( apply() @@ -1630,7 +1630,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java } } executeJavaBooleanAction( - operation = "deleteComposableTemplate", + operation = "deleteTemplate", index = None, retryable = false )( @@ -1668,7 +1668,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java override private[client] def executeListComposableTemplates() : ElasticResult[Map[String, String]] = executeJavaAction( - operation = "getTemplate", + operation = "listTemplates", index = None, retryable = true )( diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 8b16650f..c8ad1059 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1593,7 +1593,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java templateDefinition: String ): ElasticResult[Boolean] = executeJavaBooleanAction( - operation = "createComposableTemplate", + operation = "createTemplate", retryable = false )( apply() @@ -1624,7 +1624,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java } } executeJavaBooleanAction( - operation = "deleteComposableTemplate", + operation = "deleteTemplate", index = None, retryable = false )( @@ -1662,7 +1662,7 @@ trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with Java override private[client] def executeListComposableTemplates() : ElasticResult[Map[String, String]] = executeJavaAction( - operation = "getTemplate", + operation = "listTemplates", index = None, retryable = true )( From 5e99ae1c3f0cb9785e3be153b0348fb24805bc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 24 Dec 2025 08:44:19 +0100 Subject: [PATCH 29/95] refactoring traits and classes inside schema packages, add get index api --- .../softnetwork/elastic/client/AliasApi.scala | 2 +- .../client/ElasticClientDelegator.scala | 23 +- .../elastic/client/ElasticClientHelpers.scala | 7 +- .../elastic/client/IndicesApi.scala | 123 ++++++- .../elastic/client/MappingApi.scala | 6 +- .../elastic/client/NopeClientApi.scala | 74 +++- .../elastic/client/PipelineApi.scala | 6 +- .../client/metrics/MetricsElasticClient.scala | 22 +- .../elastic/client/AliasApiSpec.scala | 78 +++- .../elastic/client/IndicesApiSpec.scala | 215 ++++++++++- .../elastic/client/MappingApiSpec.scala | 32 +- .../elastic/client/SettingsApiSpec.scala | 210 ++++++++++- documentation/client/indices.md | 21 +- .../elastic/client/jest/JestAliasApi.scala | 4 +- .../elastic/client/jest/JestBulkApi.scala | 4 +- .../elastic/client/jest/JestDeleteApi.scala | 4 +- .../elastic/client/jest/JestIndexApi.scala | 4 +- .../elastic/client/jest/JestIndicesApi.scala | 33 +- .../elastic/client/jest/JestMappingApi.scala | 4 +- .../elastic/client/jest/JestPipelineApi.scala | 4 +- .../elastic/client/jest/JestScrollApi.scala | 12 +- .../elastic/client/jest/JestSettingsApi.scala | 4 +- .../elastic/client/jest/JestTemplateApi.scala | 4 +- .../elastic/client/jest/JestUpdateApi.scala | 4 +- .../client/jest/actions/GetIndex.scala | 18 + .../client/rest/RestHighLevelClientApi.scala | 96 +++-- .../client/rest/RestHighLevelClientApi.scala | 87 +++-- .../elastic/client/java/JavaClientApi.scala | 71 +++- .../elastic/client/java/JavaClientApi.scala | 64 +++- .../persistence/query/ElasticProvider.scala | 6 +- .../softnetwork/elastic/schema/package.scala | 275 ++++++++++---- .../elastic/sql/parser/Parser.scala | 76 ++-- .../elastic/sql/query/package.scala | 109 +++--- .../{DdlTableDiff.scala => TableDiff.scala} | 37 +- .../elastic/sql/schema/package.scala | 348 ++++++++++++------ .../elastic/sql/serialization/package.scala | 5 + .../elastic/sql/type/SQLTypes.scala | 4 +- .../elastic/sql/parser/ParserSpec.scala | 36 +- .../elastic/client/MockElasticClientApi.scala | 3 +- 39 files changed, 1649 insertions(+), 486 deletions(-) create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala rename sql/src/main/scala/app/softnetwork/elastic/sql/schema/{DdlTableDiff.scala => TableDiff.scala} (84%) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala index 18e76d7c..7c596bb5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala @@ -115,7 +115,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => ) } - indexExists(index) match { + indexExists(index, false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 668bce63..9642d312 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -22,7 +22,9 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} +import app.softnetwork.elastic.sql.schema.TableAlias import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -84,10 +86,20 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = delegate.createIndex(index, settings, mappings, aliases) + /** Get an index with the provided name. + * + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + override def getIndex(index: String): ElasticResult[Option[Index]] = + delegate.getIndex(index) + /** Delete an index with the provided name. * * @param index @@ -145,20 +157,23 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * true if the index exists, false otherwise */ - override def indexExists(index: String): ElasticResult[Boolean] = - delegate.indexExists(index) + override def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = + delegate.indexExists(index, pattern) override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = delegate.executeCreateIndex(index, settings, None, Nil) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = delegate.executeDeleteIndex(index) + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + delegate.executeGetIndex(index) + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = delegate.executeCloseIndex(index) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 4d28b8f6..23f82066 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -39,7 +39,7 @@ trait ElasticClientHelpers { * @return * Some(ElasticError) if invalid, None if valid */ - protected def validateIndexName(index: String): Option[ElasticError] = { + protected def validateIndexName(index: String, pattern: Boolean = false): Option[ElasticError] = { if (index == null || index.trim.isEmpty) { return Some( ElasticError( @@ -87,12 +87,11 @@ trait ElasticClientHelpers { ) } - val invalidChars = """[\\/*?"<>| ,#]""".r + val invalidChars = if (pattern) """[\\/?"<>| ,#]""".r else """[\\/*?"<>| ,#]""".r if (invalidChars.findFirstIn(trimmed).isDefined) { return Some( ElasticError( - message = - "Index name contains invalid characters: \\, /, *, ?, \", <, >, |, space, comma, #", + message = "Index name contains invalid characters: /, *, ?, \", <, >, |, space, comma, #", cause = None, statusCode = Some(400), operation = Some("validateIndexName") diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 9b31fc3c..9d1e39e5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -17,6 +17,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ /** Index management API. * @@ -26,7 +29,7 @@ import app.softnetwork.elastic.client.result._ * - Parameter validation * - Automatic retry for transient errors */ -trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => +trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi => // ======================================================================== // PUBLIC METHODS @@ -95,7 +98,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => index: String, settings: String = defaultSettings, mappings: Option[String] = None, - aliases: Seq[String] = Nil + aliases: Seq[TableAlias] = Nil ): ElasticResult[Boolean] = { validateIndexName(index) match { case Some(error) => @@ -134,7 +137,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => case _ => // OK } - aliases.flatMap(alias => validateAliasName(alias)) match { + aliases.flatMap(alias => validateAliasName(alias.alias)) match { case error :: _ => return ElasticFailure( error.copy( @@ -161,6 +164,108 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => } } + /** Get an index with the provided name. + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + def getIndex(index: String): ElasticResult[Option[Index]] = { + validateIndexName(index) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("getIndex"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${error.message}" + ) + ) + case None => // OK + } + + logger.info(s"Getting index '$index'") + + executeGetIndex(index) match { + case ElasticSuccess(Some(json)) => + logger.info(s"✅ Index '$index' retrieved successfully") + var tempIndex = Index(index, json) + tempIndex.defaultIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has default ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"✅ Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(defaultPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = + s"Default ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + tempIndex.finalIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has final ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"✅ Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(finalPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = + s"Final ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + ElasticSuccess(Some(tempIndex)) + case ElasticSuccess(None) => + logger.info(s"✅ Index '$index' not found") + ElasticSuccess(None) + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to get index '$index': ${error.message}") + failure + } + } + /** Delete an index with the provided name. * @param index * - the name of the index to delete @@ -325,7 +430,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => logger.info(s"Reindexing from '$sourceIndex' to '$targetIndex' (refresh=$refresh)") // Existence checks... - indexExists(sourceIndex) match { + indexExists(sourceIndex, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -340,7 +445,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => case _ => // OK } - indexExists(targetIndex) match { + indexExists(targetIndex, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -399,8 +504,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => * @return * true if the index exists, false otherwise */ - def indexExists(index: String): ElasticResult[Boolean] = { - validateIndexName(index) match { + def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = { + validateIndexName(index, pattern) match { case Some(error) => return ElasticFailure( error.copy( @@ -434,9 +539,11 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] + private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] + private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index 0c2b4114..689aac60 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -155,7 +155,7 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w mapping: String, settings: String = defaultSettings ): ElasticResult[Boolean] = { - indexExists(index).flatMap { + indexExists(index, false).flatMap { case false => // Scenario 1: Index doesn't exist createIndex(index, settings, Some(mapping), Nil).flatMap { @@ -305,10 +305,10 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w for { // Check if temp index exists and has data - tempExists <- indexExists(tempIndex) + tempExists <- indexExists(tempIndex, false) // Delete current (potentially corrupted) index if it exists - _ <- indexExists(index).flatMap { + _ <- indexExists(index, false).flatMap { case true => deleteIndex(index) case false => ElasticResult.success(true) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index ffafb00d..b68bf812 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -23,6 +23,7 @@ import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.query.SQLAggregation import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.schema.TableAlias import scala.concurrent.{ExecutionContext, Future} import scala.language.implicitConversions @@ -122,7 +123,7 @@ trait NopeClientApi extends ElasticClientApi { index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ElasticResult.success(false) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = @@ -239,4 +240,75 @@ trait NopeClientApi extends ElasticClientApi { override private[client] def executeVersion(): ElasticResult[String] = ElasticResult.success("0.0.0") + + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index 65f0aa37..eff1bc1e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -24,7 +24,7 @@ import app.softnetwork.elastic.client.result.{ } import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{AlterPipeline, CreatePipeline, DropPipeline} -import app.softnetwork.elastic.sql.schema.{DdlPipeline, GenericProcessor} +import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline} trait PipelineApi extends ElasticClientHelpers { _: VersionApi => @@ -72,7 +72,7 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => case ddl: AlterPipeline => getPipeline(ddl.name) match { case ElasticSuccess(Some(existing)) => - val existingPipeline = DdlPipeline(name = ddl.name, json = existing) + val existingPipeline = IngestPipeline(name = ddl.name, json = existing) val updatingPipeline = existingPipeline.merge(ddl.statements) val elasticVersion = { this.version match { @@ -137,7 +137,7 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => ElasticResult.failure(error) } case ElasticFailure(elasticError) => - ElasticResult.failure(elasticError) + ElasticResult.failure(elasticError.copy(operation = Some("pipeline"))) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 4d7eeacb..dc660cbb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -25,12 +25,16 @@ import app.softnetwork.elastic.client.{ ElasticQueries, ElasticQuery, ElasticResponse, + JSONQuery, SingleValueAggregateResult } import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.schema +import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement} +import app.softnetwork.elastic.sql.schema.TableAlias import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -91,13 +95,25 @@ class MetricsElasticClient( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { measureResult("createIndex", Some(index)) { delegate.createIndex(index, settings, mappings, aliases) } } + /** Get an index with the provided name. + * + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + override def getIndex(index: String): ElasticResult[Option[Index]] = + measureResult("getIndex", Some(index)) { + delegate.getIndex(index) + } + override def deleteIndex(index: String): ElasticResult[Boolean] = { measureResult("deleteIndex", Some(index)) { delegate.deleteIndex(index) @@ -127,9 +143,9 @@ class MetricsElasticClient( } } - override def indexExists(index: String): ElasticResult[Boolean] = { + override def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = { measureResult("indexExists", Some(index)) { - delegate.indexExists(index) + delegate.indexExists(index, pattern) } } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index 3d38d302..6cbf6c19 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for AliasApi */ @@ -21,7 +22,13 @@ class AliasApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestAliasApi extends AliasApi with IndicesApi with RefreshApi { + class TestAliasApi + extends AliasApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger // Control variables @@ -33,6 +40,7 @@ class AliasApiSpec ) var executeSwapAliasResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeIndexExistsResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) override private[client] def executeAddAlias( index: String, @@ -68,23 +76,43 @@ class AliasApiSpec executeIndexExistsResult } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeGetIndexResult + } + // Other required methods override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( - sourceIndex: JSONQuery, - targetIndex: JSONQuery, + sourceIndex: String, + targetIndex: String, refresh: Boolean, - pipeline: Option[JSONQuery] + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } var aliasApi: TestAliasApi = _ @@ -1629,7 +1657,12 @@ class AliasApiSpec "not call execute methods when validation fails" in { // Given var executeCalled = false - val validatingApi = new AliasApi with IndicesApi with RefreshApi { + val validatingApi = new AliasApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeAddAlias( @@ -1658,22 +1691,43 @@ class AliasApiSpec ): ElasticResult[Boolean] = ??? override private[client] def executeCreateIndex( index: String, - settings: JSONQuery, - mappings: Option[JSONQuery], - aliases: Seq[JSONQuery] + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( - sourceIndex: JSONQuery, - targetIndex: JSONQuery, + sourceIndex: String, + targetIndex: String, refresh: Boolean, - pipeline: Option[JSONQuery] + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index bd42e076..7c3907b2 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for IndicesApi */ @@ -21,11 +22,17 @@ class IndicesApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestIndicesApi extends IndicesApi with RefreshApi { + class TestIndicesApi + extends IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger // Control variables for each operation var executeCreateIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) var executeDeleteIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeCloseIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeOpenIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) @@ -39,11 +46,15 @@ class IndicesApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCreateIndexResult } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeGetIndexResult + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = { executeDeleteIndexResult } @@ -72,6 +83,22 @@ class IndicesApiSpec override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = { executeRefreshResult } + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } var indicesApi: TestIndicesApi = _ @@ -472,7 +499,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result shouldBe ElasticSuccess(true) @@ -486,7 +513,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(false) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result shouldBe ElasticSuccess(false) @@ -500,7 +527,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticFailure(error) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result.isFailure shouldBe true @@ -512,7 +539,7 @@ class IndicesApiSpec "reject invalid index name" in { // When - val result = indicesApi.indexExists("INVALID") + val result = indicesApi.indexExists("INVALID", false) // Then result.isFailure shouldBe true @@ -670,7 +697,11 @@ class IndicesApiSpec "fail when target index does not exist" in { // Given var callCount = 0 - val checkingApi = new IndicesApi with RefreshApi { + val checkingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -683,8 +714,13 @@ class IndicesApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -697,6 +733,22 @@ class IndicesApiSpec pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -724,7 +776,11 @@ class IndicesApiSpec "fail when target existence check fails" in { // Given var callCount = 0 - val checkingApi = new IndicesApi with RefreshApi { + val checkingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -737,8 +793,13 @@ class IndicesApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -751,6 +812,22 @@ class IndicesApiSpec pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -786,7 +863,7 @@ class IndicesApiSpec // When val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil).flatMap { _ => - indicesApi.indexExists("my-index") + indicesApi.indexExists("my-index", false) } // Then @@ -934,19 +1011,27 @@ class IndicesApiSpec "validate index name before calling execute methods" in { // Given var executeCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) } + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -961,6 +1046,22 @@ class IndicesApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -973,19 +1074,27 @@ class IndicesApiSpec "validate settings after index name validation" in { // Given var executeCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) } + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -1000,6 +1109,22 @@ class IndicesApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1012,7 +1137,11 @@ class IndicesApiSpec "validate both indices in reindex before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1024,8 +1153,13 @@ class IndicesApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -1038,6 +1172,22 @@ class IndicesApiSpec pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1050,7 +1200,11 @@ class IndicesApiSpec "check source and target are different before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1062,8 +1216,13 @@ class IndicesApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = @@ -1076,6 +1235,22 @@ class IndicesApiSpec pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1116,7 +1291,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - indicesApi.indexExists("my-index") + indicesApi.indexExists("my-index", false) // Then verify(mockLogger, atLeastOnce).debug(any[String]) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index 26a125f2..dd4fc402 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -6,6 +6,7 @@ import org.scalatest.BeforeAndAfterEach import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for MappingApi */ @@ -27,7 +28,14 @@ class MappingApiSpec """{"properties":{"name":{"type":"text"},"age":{"type":"integer"}}}""" // Concrete implementation for testing - class TestMappingApi extends MappingApi with SettingsApi with IndicesApi with RefreshApi { + class TestMappingApi + extends MappingApi + with SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger // Control variables @@ -35,6 +43,7 @@ class MappingApiSpec var executeGetMappingResult: ElasticResult[String] = ElasticSuccess(validMapping) var executeIndexExistsResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeCreateIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) var executeDeleteIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeReindexFunction : (String, String, Boolean) => ElasticResult[(Boolean, Option[Long])] = @@ -62,11 +71,14 @@ class MappingApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCreateIndexResult } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + executeGetIndexResult + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = { executeDeleteIndexResult } @@ -98,6 +110,22 @@ class MappingApiSpec index: String, settings: String ): ElasticResult[Boolean] = ElasticSuccess(true) + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } var mappingApi: TestMappingApi = _ diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index bbf5b3ad..378dc045 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias import com.google.gson.JsonParser /** Unit tests for SettingsApi Coverage target: 80%+ Using mockito-scala 1.17.12 @@ -22,7 +23,13 @@ class SettingsApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestSettingsApi extends SettingsApi with IndicesApi with RefreshApi { + class TestSettingsApi + extends SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger // Control variables @@ -57,8 +64,11 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( sourceIndex: String, @@ -68,6 +78,22 @@ class SettingsApiSpec ): ElasticResult[(Boolean, Option[Long])] = ??? override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } var settingsApi: TestSettingsApi = _ @@ -712,7 +738,12 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -741,8 +772,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -754,6 +790,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -770,7 +822,12 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -796,8 +853,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -809,6 +871,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -823,7 +901,12 @@ class SettingsApiSpec // Given var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -848,8 +931,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -861,6 +949,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1397,7 +1501,12 @@ class SettingsApiSpec "validate index name before calling executeCloseIndex" in { // Given var closeCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1416,8 +1525,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -1429,6 +1543,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1441,7 +1571,12 @@ class SettingsApiSpec "validate settings after index name" in { // Given var closeCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1460,8 +1595,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -1473,6 +1613,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When @@ -1485,7 +1641,12 @@ class SettingsApiSpec "validate index name before calling executeLoadSettings" in { // Given var loadCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new SettingsApi + with IndicesApi + with RefreshApi + with PipelineApi + with VersionApi + with SerializationApi { override protected def logger: Logger = mockLogger override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { @@ -1504,8 +1665,13 @@ class SettingsApiSpec index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetIndex( + index: String + ): ElasticResult[Option[String]] = ??? + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? override private[client] def executeReindex( @@ -1517,6 +1683,22 @@ class SettingsApiSpec override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = ??? + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ??? + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = ??? + + override private[client] def executeVersion(): ElasticResult[String] = ??? } // When diff --git a/documentation/client/indices.md b/documentation/client/indices.md index c8de7f50..df278727 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -84,7 +84,7 @@ def createIndex( index: String, settings: String = defaultSettings, mappings: Option[String] = None, - aliases: Seq[String] = Seq.empty + aliases: Seq[TableAlias] = Seq.empty ): ElasticResult[Boolean] ``` @@ -154,6 +154,25 @@ for { --- +### getIndex + +Gets an existing index. + +**Signature:** + +```scala +def getIndex(index: String): ElasticResult[Option[Index]] +``` + +**Parameters:** +- `index` - Name of the index to get + +**Returns:** +- `ElasticSuccess[Option[Index]]` with index configuration if index found, None otherwise +- `ElasticFailure` with error details + +--- + ### deleteIndex Deletes an existing index. diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala index 3e773733..997a7869 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{AliasApi, IndicesApi} +import app.softnetwork.elastic.client.AliasApi import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.client.JestResult import io.searchbox.indices.aliases.{AddAliasMapping, GetAliases, ModifyAliases, RemoveAliasMapping} @@ -28,7 +28,7 @@ import scala.jdk.CollectionConverters._ * [[AliasApi]] for generic API documentation */ trait JestAliasApi extends AliasApi with JestClientHelpers { - _: IndicesApi with JestClientCompanion => + _: JestIndicesApi with JestClientCompanion => /** Add an alias to an index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala index 9399ecec..9a9c4fec 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.client.jest import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Flow -import app.softnetwork.elastic.client.{BulkApi, IndexApi, RefreshApi, SettingsApi} +import app.softnetwork.elastic.client.BulkApi import app.softnetwork.elastic.client.bulk.{ BulkAction, BulkElasticAction, @@ -40,7 +40,7 @@ import scala.language.implicitConversions import scala.util.{Failure, Success, Try} trait JestBulkApi extends BulkApi { - _: RefreshApi with SettingsApi with IndexApi with JestClientCompanion => + _: JestRefreshApi with JestSettingsApi with JestIndexApi with JestClientCompanion => // ======================================================================== // TYPE ALIASES FOR JEST diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala index 7dd90f95..c5bdcc99 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{DeleteApi, SettingsApi} +import app.softnetwork.elastic.client.DeleteApi import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Delete @@ -27,7 +27,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[DeleteApi]] for generic API documentation */ trait JestDeleteApi extends DeleteApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion => + _: JestSettingsApi with JestClientCompanion => /** Delete an entity from the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala index 60404a0c..f914c461 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndexApi, SerializationApi, SettingsApi} +import app.softnetwork.elastic.client.{IndexApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Index @@ -27,7 +27,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[IndexApi]] for generic API documentation */ trait JestIndexApi extends IndexApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion with SerializationApi => + _: JestSettingsApi with JestClientCompanion with SerializationApi => /** Index a document in the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index cd0c17e4..b168d758 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -17,7 +17,9 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.IndicesApi +import app.softnetwork.elastic.client.jest.actions.GetIndex import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} import io.searchbox.client.JestResult import io.searchbox.indices.{CloseIndex, CreateIndex, DeleteIndex, IndicesExists, OpenIndex} import io.searchbox.indices.reindex.Reindex @@ -28,8 +30,8 @@ import scala.util.Try * @see * [[IndicesApi]] for generic API documentation */ -trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpers { - _: JestClientCompanion => +trait JestIndicesApi extends IndicesApi with JestClientHelpers { + _: JestRefreshApi with JestPipelineApi with JestClientCompanion => /** Create an index with the given settings. * @see @@ -39,7 +41,7 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe index: String, settings: String = defaultSettings, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeJestBooleanAction( operation = "createIndex", @@ -49,7 +51,13 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe val builder = new CreateIndex.Builder(index) .settings(settings) if (aliases.nonEmpty) { - builder.aliases(aliases.map(alias => s"""{"$alias":{}}""").mkString(",")) + val root = mapper.createObjectNode() + val as = mapper.createObjectNode() + aliases.foreach { alias => + as.set(alias.alias, alias.node) + } + root.set("aliases", as) + builder.aliases(root.toString) } mappings.foreach { mapping => builder.mappings(mapping) @@ -58,7 +66,24 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe } } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJestAction[JestResult, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + ) { + new GetIndex.Builder(index).build() + } { result => + if (result.isSucceeded) { + Some(result.getJsonString) + } else { + None + } + } + } + /** Delete an index. + * * @see * [[IndicesApi.deleteIndex]] */ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index c85ebd11..d18f8897 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndicesApi, MappingApi, RefreshApi, SettingsApi} +import app.softnetwork.elastic.client.MappingApi import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, @@ -33,7 +33,7 @@ import scala.util.Try * [[MappingApi]] for generic API documentation */ trait JestMappingApi extends MappingApi with JestClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JestClientCompanion => + _: JestSettingsApi with JestIndicesApi with JestRefreshApi with JestClientCompanion => /** Set the mapping for an index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala index c684110f..4bd621d1 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala @@ -7,8 +7,8 @@ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult -trait JestPipelineApi extends PipelineApi with JestClientHelpers with JestVersionApi { - _: SerializationApi with JestClientCompanion => +trait JestPipelineApi extends PipelineApi with JestClientHelpers { + _: JestVersionApi with SerializationApi with JestClientCompanion => override private[client] def executeCreatePipeline( pipelineName: String, diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala index 59096d33..8fefa648 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala @@ -19,15 +19,7 @@ package app.softnetwork.elastic.client.jest import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Source -import app.softnetwork.elastic.client.{ - retryWithBackoff, - ClientAggregation, - ElasticQuery, - ElasticResponse, - ScrollApi, - SearchApi, - VersionApi -} +import app.softnetwork.elastic.client.{retryWithBackoff, ClientAggregation, ElasticQuery, ScrollApi} import app.softnetwork.elastic.client.scroll.ScrollConfig import app.softnetwork.elastic.sql.query.SQLAggregation import com.google.gson.{JsonNull, JsonObject, JsonParser} @@ -40,7 +32,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} trait JestScrollApi extends ScrollApi with JestClientHelpers { - _: VersionApi with SearchApi with JestClientCompanion => + _: JestVersionApi with JestSearchApi with JestClientCompanion => /** Classic scroll (works for both hits and aggregations) */ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala index dd29e89e..cb022a5a 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndicesApi, SettingsApi} +import app.softnetwork.elastic.client.SettingsApi import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.indices.settings.{GetSettings, UpdateSettings} @@ -25,7 +25,7 @@ import io.searchbox.indices.settings.{GetSettings, UpdateSettings} * [[SettingsApi]] for generic API documentation */ trait JestSettingsApi extends SettingsApi with JestClientHelpers { - _: IndicesApi with JestClientCompanion => + _: JestIndicesApi with JestClientCompanion => /** Update index settings. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala index e54abea6..aec2ceb6 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala @@ -15,8 +15,8 @@ import io.searchbox.client.JestResult import scala.jdk.CollectionConverters._ -trait JestTemplateApi extends TemplateApi with JestClientHelpers with JestVersionApi { - _: SerializationApi with JestClientCompanion => +trait JestTemplateApi extends TemplateApi with JestClientHelpers { + _: JestVersionApi with SerializationApi with JestClientCompanion => // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala index 6a6a924c..9ef0648f 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{SerializationApi, SettingsApi, UpdateApi} +import app.softnetwork.elastic.client.{SerializationApi, UpdateApi} import app.softnetwork.elastic.client.bulk.docAsUpsert import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Update @@ -28,7 +28,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[UpdateApi]] for generic API documentation */ trait JestUpdateApi extends UpdateApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion with SerializationApi => + _: JestSettingsApi with JestClientCompanion with SerializationApi => /** Update an entity in the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala new file mode 100644 index 00000000..6e413508 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala @@ -0,0 +1,18 @@ +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.action.AbstractAction +import io.searchbox.action.GenericResultAbstractAction +import io.searchbox.client.config.ElasticsearchVersion + +object GetIndex { + class Builder(var index: String) extends AbstractAction.Builder[GetIndex, GetIndex.Builder] { + override def build = new GetIndex(this) + } +} + +class GetIndex protected (builder: GetIndex.Builder) extends GenericResultAbstractAction(builder) { + override def getRestMethodName = "GET" + + override def buildURI(elasticsearchVersion: ElasticsearchVersion): String = s"/${builder.index}" + +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index a88c4196..53c7faee 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,8 +23,10 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.{ObjectValue, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.schema.TableAlias import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils @@ -61,7 +63,7 @@ import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} -import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions} +import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CreateIndexRequest, @@ -69,6 +71,7 @@ import org.elasticsearch.client.indices.{ GetIndexTemplatesRequest, GetIndexTemplatesResponse, GetMappingsRequest, + GetMappingsResponse, IndexTemplateMetaData, IndexTemplatesExistRequest, PutIndexTemplateRequest, @@ -108,7 +111,7 @@ trait RestHighLevelClientApi with RestHighLevelClientBulkApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion - with RestHighLevelClientVersion + with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi @@ -116,7 +119,7 @@ trait RestHighLevelClientApi * @see * [[VersionApi]] for generic API documentation */ -trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { +trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => override private[client] def executeVersion(): ElasticResult[String] = executeRestLowLevelAction[String]( @@ -140,12 +143,14 @@ trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelp * [[IndicesApi]] for generic API documentation */ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { - _: RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", @@ -154,13 +159,46 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH )( request = new CreateIndexRequest(index) .settings(settings, XContentType.JSON) - .aliases(aliases.map(alias => new Alias(alias)).asJava) + .aliases( + aliases + .map(alias => { + var a = new Alias(alias.alias).writeIndex(alias.isWriteIndex) + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + a = a.filter(filterNode.toString) + } + alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) + alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) + a + }) + .asJava + ) .mapping(mappings.getOrElse("{}"), XContentType.JSON) )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeRestAction[Request, Response, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + )( + request = new Request("GET", s"/$index") + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )(resp => { + resp.getStatusLine match { + case statusLine if statusLine.getStatusCode >= 400 => + None + case _ => + val json = scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + Some(json) + } + }) + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", @@ -261,7 +299,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH * [[AliasApi]] for generic API documentation */ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( index: String, @@ -355,7 +393,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe * [[SettingsApi]] for generic API documentation */ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeUpdateSettings( index: String, @@ -390,7 +428,10 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien * [[MappingApi]] for generic API documentation */ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientCompanion => override private[client] def executeSetMapping( index: String, mapping: String @@ -409,7 +450,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeRestAction[ GetMappingsRequest, - org.elasticsearch.client.indices.GetMappingsResponse, + GetMappingsResponse, String ]( operation = "getMapping", @@ -511,7 +552,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe * [[IndexApi]] for generic API documentation */ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, @@ -581,7 +622,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe * [[UpdateApi]] for generic API documentation */ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, id: String, @@ -657,7 +698,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel * [[DeleteApi]] for generic API documentation */ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => override private[client] def executeDelete( index: String, @@ -905,7 +946,10 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel * [[BulkApi]] for generic API documentation */ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndexApi + with RestHighLevelClientCompanion => override type BulkActionType = DocWriteRequest[_] override type BulkResultType = BulkResponse @@ -1138,7 +1182,9 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers * [[ScrollApi]] for generic API documentation */ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { - _: VersionApi with SearchApi with RestHighLevelClientCompanion => + _: RestHighLevelClientVersionApi + with RestHighLevelClientSearchApi + with RestHighLevelClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1412,11 +1458,8 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } -trait RestHighLevelClientPipelineApi - extends PipelineApi - with RestHighLevelClientHelpers - with RestHighLevelClientVersion { - _: RestHighLevelClientCompanion with SerializationApi => +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( pipelineName: String, @@ -1450,9 +1493,9 @@ trait RestHighLevelClientPipelineApi } override private[client] def executeGetPipeline( - pipelineName: JSONQuery - ): ElasticResult[Option[JSONQuery]] = { - executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[String]]( operation = "getPipeline", retryable = true )( @@ -1474,11 +1517,8 @@ trait RestHighLevelClientPipelineApi } } -trait RestHighLevelClientTemplateApi - extends TemplateApi - with RestHighLevelClientHelpers - with RestHighLevelClientVersion { - _: RestHighLevelClientCompanion with SerializationApi => +trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 82a2b900..4caaad31 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,8 +23,10 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.{ObjectValue, Value} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser @@ -64,7 +66,7 @@ import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} -import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions} +import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CloseIndexRequest, @@ -116,7 +118,7 @@ trait RestHighLevelClientApi with RestHighLevelClientBulkApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion - with RestHighLevelClientVersion + with RestHighLevelClientVersionApi with RestHighLevelClientPipelineApi with RestHighLevelClientTemplateApi @@ -124,7 +126,7 @@ trait RestHighLevelClientApi * @see * [[VersionApi]] for generic API documentation */ -trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { +trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => override private[client] def executeVersion(): ElasticResult[String] = @@ -150,13 +152,15 @@ trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelp * [[IndicesApi]] for generic API documentation */ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { - _: RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", @@ -165,13 +169,46 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH )( request = new CreateIndexRequest(index) .settings(settings, XContentType.JSON) - .aliases(aliases.map(alias => new Alias(alias)).asJava) + .aliases( + aliases + .map(alias => { + var a = new Alias(alias.alias).writeIndex(alias.isWriteIndex).isHidden(alias.isHidden) + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + a = a.filter(filterNode.toString) + } + alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) + alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) + a + }) + .asJava + ) .mapping(mappings.getOrElse("{}"), XContentType.JSON) )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeRestAction[Request, Response, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + )( + request = new Request("GET", s"/$index") + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )(resp => { + resp.getStatusLine match { + case statusLine if statusLine.getStatusCode >= 400 => + None + case _ => + val json = scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + Some(json) + } + }) + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", @@ -272,7 +309,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH * [[AliasApi]] for generic API documentation */ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( index: String, @@ -365,7 +402,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe * [[SettingsApi]] for generic API documentation */ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeUpdateSettings( index: String, @@ -400,7 +437,10 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien * [[MappingApi]] for generic API documentation */ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientCompanion => override private[client] def executeSetMapping( index: String, @@ -523,7 +563,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe * [[IndexApi]] for generic API documentation */ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, @@ -591,7 +631,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe * [[UpdateApi]] for generic API documentation */ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, id: String, @@ -667,7 +707,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel * [[DeleteApi]] for generic API documentation */ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => override private[client] def executeDelete( index: String, @@ -915,7 +955,10 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel * [[BulkApi]] for generic API documentation */ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndexApi + with RestHighLevelClientCompanion => override type BulkActionType = DocWriteRequest[_] override type BulkResultType = BulkResponse @@ -1141,7 +1184,9 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers * [[ScrollApi]] for generic API documentation */ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { - _: SearchApi with VersionApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSearchApi + with RestHighLevelClientVersionApi + with RestHighLevelClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1591,11 +1636,8 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } -trait RestHighLevelClientPipelineApi - extends PipelineApi - with RestHighLevelClientHelpers - with RestHighLevelClientVersion { - _: RestHighLevelClientCompanion with SerializationApi => +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( pipelineName: String, @@ -1654,11 +1696,8 @@ trait RestHighLevelClientPipelineApi } } -trait RestHighLevelClientTemplateApi - extends TemplateApi - with RestHighLevelClientHelpers - with RestHighLevelClientVersion { - _: RestHighLevelClientCompanion with SerializationApi => +trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index f81045bd..2a20c9e9 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -31,6 +31,7 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.schema.TableAlias import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -115,13 +116,13 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * @see * [[IndicesApi]] for index management operations */ -trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHelpers { - _: JavaClientCompanion => +trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { + _: JavaClientRefreshApi with JavaClientPipelineApi with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", @@ -134,7 +135,19 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) - .aliases(aliases.map(key => (key, new Alias.Builder().build())).toMap.asJava) + .aliases( + aliases + .map(alias => { + var builder = + new Alias.Builder().isWriteIndex(alias.isWriteIndex).isHidden(alias.isHidden) + alias.routing.foreach(r => builder = builder.routing(r)) + alias.indexRouting.foreach(ir => builder = builder.indexRouting(ir)) + alias.searchRouting.foreach(sr => builder = builder.searchRouting(sr)) + (alias.alias, builder.build()) + }) + .toMap + .asJava + ) .mappings( new TypeMapping.Builder() .withJson( @@ -146,6 +159,26 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.acknowledged()) + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getIndex", + index = Some(index), + retryable = true + )( + apply() + .indices() + .get( + new GetIndexRequest.Builder().index(index).build() + ) + )(response => { + val valueOpt = response.result.asScala.get(index) + valueOpt match { + case Some(value) => Some(convertToJson(value)) + case None => None + } + }) + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", @@ -228,7 +261,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel * [[AliasApi]] for alias management operations */ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( index: String, @@ -335,7 +368,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { * [[SettingsApi]] for settings management operations */ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeUpdateSettings( index: String, @@ -376,7 +409,10 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { * [[MappingApi]] for mapping management operations */ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JavaClientCompanion => + _: JavaClientSettingsApi + with JavaClientIndicesApi + with JavaClientRefreshApi + with JavaClientCompanion => override private[client] def executeSetMapping( index: String, @@ -536,7 +572,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { * [[IndexApi]] for index operations */ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, @@ -596,7 +632,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { * [[UpdateApi]] for update operations */ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, @@ -671,7 +707,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { * [[DeleteApi]] for delete operations */ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion => + _: JavaClientSettingsApi with JavaClientCompanion => override private[client] def executeDelete( index: String, @@ -865,7 +901,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * [[BulkApi]] for bulk operations */ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientSettingsApi + with JavaClientIndexApi + with JavaClientCompanion => override type BulkActionType = BulkOperation override type BulkResultType = BulkResponse @@ -1114,7 +1153,7 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: VersionApi with SearchApi with JavaClientCompanion => + _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1526,8 +1565,8 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } -trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with JavaClientVersionApi { - _: JavaClientCompanion with SerializationApi => +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion with SerializationApi => override private[client] def executeCreatePipeline( pipelineName: String, @@ -1589,8 +1628,8 @@ trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with Java } } -trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with JavaClientVersionApi { - _: JavaClientCompanion with SerializationApi => +trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion with SerializationApi => // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index c8ad1059..d2393b9c 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -25,13 +25,13 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} -import app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.schema.TableAlias import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -111,13 +111,13 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * @see * [[IndicesApi]] for index management operations */ -trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHelpers { - _: JavaClientCompanion => +trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { + _: JavaClientRefreshApi with JavaClientPipelineApi with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", @@ -130,7 +130,19 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) - .aliases(aliases.map(key => (key, new Alias.Builder().build())).toMap.asJava) + .aliases( + aliases + .map(alias => { + var builder = + new Alias.Builder().isWriteIndex(alias.isWriteIndex).isHidden(alias.isHidden) + alias.routing.foreach(r => builder = builder.routing(r)) + alias.indexRouting.foreach(ir => builder = builder.indexRouting(ir)) + alias.searchRouting.foreach(sr => builder = builder.searchRouting(sr)) + (alias.alias, builder.build()) + }) + .toMap + .asJava + ) .mappings( new TypeMapping.Builder() .withJson( @@ -142,6 +154,26 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.acknowledged()) + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getIndex", + index = Some(index), + retryable = true + )( + apply() + .indices() + .get( + new GetIndexRequest.Builder().index(index).build() + ) + )(response => { + val valueOpt = response.indices().asScala.get(index) + valueOpt match { + case Some(value) => Some(convertToJson(value)) + case None => None + } + }) + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", @@ -224,7 +256,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel * [[AliasApi]] for alias management operations */ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( index: String, @@ -331,7 +363,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { * [[SettingsApi]] for settings management operations */ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeUpdateSettings( index: String, @@ -372,7 +404,10 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { * [[MappingApi]] for mapping management operations */ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JavaClientCompanion => + _: JavaClientSettingsApi + with JavaClientIndicesApi + with JavaClientRefreshApi + with JavaClientCompanion => override private[client] def executeSetMapping( index: String, @@ -533,7 +568,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { * [[IndexApi]] for index operations */ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, @@ -596,7 +631,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { * [[UpdateApi]] for update operations */ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, @@ -663,7 +698,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { * [[DeleteApi]] for delete operations */ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion => + _: JavaClientSettingsApi with JavaClientCompanion => override private[client] def executeDelete( index: String, @@ -859,7 +894,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * [[BulkApi]] for bulk operations */ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientSettingsApi + with JavaClientIndexApi + with JavaClientCompanion => override type BulkActionType = BulkOperation override type BulkResultType = BulkResponse @@ -1108,7 +1146,7 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: VersionApi with SearchApi with JavaClientCompanion => + _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ diff --git a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala index a86a611e..eef61be8 100644 --- a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala +++ b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala @@ -26,7 +26,7 @@ import app.softnetwork.persistence._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.persistence.query.ExternalPersistenceProvider import app.softnetwork.serialization.commonFormats -import app.softnetwork.elastic.persistence.typed.Elastic._ +import app.softnetwork.elastic.persistence.typed.Elastic import org.slf4j.Logger import scala.reflect.ClassTag @@ -45,11 +45,11 @@ trait ElasticProvider[T <: Timestamped] implicit def formats: Formats = commonFormats - protected lazy val index: String = getIndex[T](manifestWrapper.wrapped) + protected lazy val index: String = Elastic.getIndex[T](manifestWrapper.wrapped) protected lazy val _type: String = getType[T](manifestWrapper.wrapped) - protected lazy val alias: String = getAlias[T](manifestWrapper.wrapped) + protected lazy val alias: String = Elastic.getAlias[T](manifestWrapper.wrapped) protected def mappingPath: Option[String] = None diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 6bc7a493..6d5bcf56 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -19,14 +19,15 @@ package app.softnetwork.elastic import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ - DdlColumn, - DdlDateIndexNameProcessor, - DdlDefaultValueProcessor, - DdlPartition, - DdlPrimaryKeyProcessor, - DdlProcessor, - DdlScriptProcessor, - DdlTable + Column, + DateIndexNameProcessor, + DefaultValueProcessor, + IngestPipelineType, + IngestProcessor, + PartitionDate, + PrimaryKeyProcessor, + ScriptProcessor, + Table } import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.time.TimeUnit @@ -36,18 +37,18 @@ import scala.jdk.CollectionConverters._ package object schema { - final case class EsField( + final case class IndexField( name: String, `type`: String, - script: Option[DdlScriptProcessor] = None, + script: Option[ScriptProcessor] = None, null_value: Option[Value[_]] = None, not_null: Option[Boolean] = None, comment: Option[String] = None, - fields: List[EsField] = Nil, + fields: List[IndexField] = Nil, options: Map[String, Value[_]] = Map.empty ) { - lazy val ddlColumn: DdlColumn = { - DdlColumn( + lazy val ddlColumn: Column = { + Column( name = name, dataType = SQLTypes(this), script = script, @@ -60,8 +61,8 @@ package object schema { } } - object EsField { - def apply(name: String, node: JsonNode): EsField = { + object IndexField { + def apply(name: String, node: JsonNode): IndexField = { val tpe = Option(node.get("type")).map(_.asText()).getOrElse("object") val nullValue = @@ -107,7 +108,7 @@ package object schema { map.get("painless") match { case Some(source: StringValue) => Some( - DdlScriptProcessor( + ScriptProcessor( script = script.value, column = name, dataType = SQLTypes(tpe), @@ -121,7 +122,7 @@ package object schema { } case _ => None } - EsField( + IndexField( name = name, `type` = tpe, script = script, @@ -135,22 +136,22 @@ package object schema { } } - final case class EsMappings( - fields: List[EsField] = Nil, + final case class IndexMappings( + fields: List[IndexField] = Nil, primaryKey: List[String] = Nil, - partitionBy: Option[EsPartitionBy] = None, + partitionBy: Option[IndexDatePartition] = None, options: Map[String, Value[_]] = Map.empty ) - object EsMappings { - def apply(root: JsonNode): EsMappings = { + object IndexMappings { + def apply(root: JsonNode): IndexMappings = { val mappings = root.path("mappings") val fields = Option(mappings.get("properties")) .orElse(Option(mappings.path("_doc").get("properties"))) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue - EsField(name, value) + IndexField(name, value) }.toList) .getOrElse(Nil) @@ -168,7 +169,7 @@ package object schema { } .getOrElse(List.empty) - val partitionBy: Option[EsPartitionBy] = meta.flatMap { + val partitionBy: Option[IndexDatePartition] = meta.flatMap { case m: ObjectValue => m.value.get("partition_by") match { case Some(pb: ObjectValue) => @@ -176,8 +177,8 @@ package object schema { case Some(column: StringValue) => // valid pb.value.get("granularity") match { case Some(granularity: StringValue) => - Some(EsPartitionBy(column.value, granularity.value)) - case _ => Some(EsPartitionBy(column.value, "d")) + Some(IndexDatePartition(column.value, granularity.value)) + case _ => Some(IndexDatePartition(column.value, "d")) } case _ => None } @@ -186,7 +187,7 @@ package object schema { case _ => None } - EsMappings( + IndexMappings( fields = fields, primaryKey = primaryKey, partitionBy = partitionBy, @@ -196,62 +197,153 @@ package object schema { } - final case class EsPartitionBy( + final case class IndexDatePartition( column: String, granularity: String // "d", "M", "y", etc. ) - final case class EsSettings( + final case class IndexSettings( options: Map[String, Value[_]] = Map.empty ) - object EsSettings { - def apply(settings: JsonNode): EsSettings = { + object IndexSettings { + def apply(settings: JsonNode): IndexSettings = { val index = settings.path("settings").path("index") val options = extractObject(index) - EsSettings( + IndexSettings( options = options ) } } - final case class EsProcessor( + final case class IndexIngestProcessor( + pipelineType: IngestPipelineType, processor: JsonNode ) { - lazy val ddlProcesor: DdlProcessor = DdlProcessor(processor) + lazy val ddlProcesor: IngestProcessor = IngestProcessor(pipelineType, processor) } - final case class EsPipeline( + final case class IndexIngestPipeline( + pipelineType: IngestPipelineType, pipeline: JsonNode ) { - lazy val processors: Seq[DdlProcessor] = { + lazy val processors: Seq[IngestProcessor] = { val processorsNode = pipeline.get("processors") if (processorsNode != null && processorsNode.isArray) { - processorsNode.elements().asScala.toSeq.map(EsProcessor(_).ddlProcesor) + processorsNode + .elements() + .asScala + .toSeq + .map(IndexIngestProcessor(pipelineType, _).ddlProcesor) } else { Seq.empty } } } - final case class EsIndex( + final case class IndexAlias( + name: String, + filter: Map[String, Any] = Map.empty, + routing: Option[String] = None, + indexRouting: Option[String] = None, + searchRouting: Option[String] = None, + isWriteIndex: Boolean = false, + isHidden: Boolean = false, + node: JsonNode + ) + + object IndexAlias { + def apply(name: String, node: JsonNode): IndexAlias = { + if (node.has(name)) { + val aliasNode = node.path(name) + return apply(name, aliasNode) + } + val filter: Map[String, Any] = + if (node.has("filter")) node.path("filter") + else Map.empty + val routing = + Option(node.get("routing")).map(_.asText()) + val indexRouting = + Option(node.get("index_routing")).map(_.asText()) + val searchRouting = + Option(node.get("search_routing")).map(_.asText()) + val isWriteIndex = + Option(node.get("is_write_index")).exists(_.asBoolean()) + val isHidden = + Option(node.get("is_hidden")).exists(_.asBoolean()) + IndexAlias( + name = name, + filter = filter, + routing = routing, + indexRouting = indexRouting, + searchRouting = searchRouting, + isWriteIndex = isWriteIndex, + isHidden = isHidden, + node = node + ) + } + } + + final case class IndexAliases( + aliases: Map[String, IndexAlias] = Map.empty + ) + + object IndexAliases { + def apply(nodes: Seq[(String, JsonNode)]): IndexAliases = { + IndexAliases(nodes.map(entry => entry._1 -> IndexAlias(entry._1, entry._2)).toMap) + } + } + + final case class Index( name: String, mappings: JsonNode, settings: JsonNode, - pipeline: Option[JsonNode] = None + aliases: Map[String, JsonNode] = Map.empty, + defaultPipeline: Option[JsonNode] = None, + finalPipeline: Option[JsonNode] = None ) { - lazy val esMappings: EsMappings = EsMappings(mappings) + lazy val defaultIngestPipelineName: Option[String] = esSettings.options.get("index") match { + case Some(obj: ObjectValue) => + obj.value.get("default_pipeline") match { + case Some(s: StringValue) => Some(s.value) + case _ => None + } + case _ => None + } - lazy val esSettings: EsSettings = EsSettings(settings) + lazy val finalIngestPipelineName: Option[String] = esSettings.options.get("index") match { + case Some(obj: ObjectValue) => + obj.value.get("final_pipeline") match { + case Some(s: StringValue) => Some(s.value) + case _ => None + } + case _ => None + } - lazy val esPipeline: Option[EsPipeline] = pipeline.map(EsPipeline(_)) + lazy val esMappings: IndexMappings = IndexMappings(mappings) - lazy val ddlTable: DdlTable = { + lazy val esSettings: IndexSettings = IndexSettings(settings) + + lazy val esAliases: IndexAliases = IndexAliases(aliases.toSeq) + + lazy val esDefaultPipeline: Option[IndexIngestPipeline] = + defaultPipeline.map(IndexIngestPipeline(IngestPipelineType.Default, _)) + + lazy val esFinalPipeline: Option[IndexIngestPipeline] = + finalPipeline.map(IndexIngestPipeline(IngestPipelineType.Final, _)) + + lazy val esProcessors: Seq[IngestProcessor] = { + val defaultProcessors = esDefaultPipeline.map(_.processors).getOrElse(Seq.empty) + val finalProcessors = esFinalPipeline.map(_.processors).getOrElse(Seq.empty) + defaultProcessors ++ finalProcessors + } + + lazy val ddlTable: Table = { // 1. Columns from the mapping - val initialCols: Map[String, DdlColumn] = + val initialCols: Map[String, Column] = esMappings.fields.map { field => val name = field.name name -> field.ddlColumn @@ -259,53 +351,92 @@ package object schema { // 2. PK + partition + pipelines from index mappings and settings var primaryKey: List[String] = esMappings.primaryKey - var partitionBy: Option[DdlPartition] = esMappings.partitionBy.map { p => + var partitionBy: Option[PartitionDate] = esMappings.partitionBy.map { p => val granularity = TimeUnit(p.granularity) - DdlPartition(p.column, granularity) + PartitionDate(p.column, granularity) } // 3. Enrichment from the pipeline (if provided) val enrichedCols = scala.collection.mutable.Map(initialCols.toSeq: _*) - esPipeline.foreach { pipeline => - pipeline.processors.foreach { - case p: DdlScriptProcessor => - val col = p.column - enrichedCols.get(col).foreach { c => - enrichedCols.update(col, c.copy(script = Some(p))) - } + var processors: collection.mutable.Seq[IngestProcessor] = collection.mutable.Seq.empty - case p: DdlDefaultValueProcessor => - val col = p.column - enrichedCols.get(col).foreach { c => - enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) - } + esProcessors.foreach { + case p: ScriptProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(script = Some(p))) + } - case p: DdlDateIndexNameProcessor => - if (partitionBy.isEmpty) { - val granularity = TimeUnit(p.dateRounding) - partitionBy = Some(DdlPartition(p.column, granularity)) - } + case p: DefaultValueProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) + } - case p: DdlPrimaryKeyProcessor => - if (primaryKey.isEmpty) { - primaryKey = p.value.toList - } + case p: DateIndexNameProcessor => + if (partitionBy.isEmpty) { + val granularity = TimeUnit(p.dateRounding) + partitionBy = Some(PartitionDate(p.column, granularity)) + } + + case p: PrimaryKeyProcessor => + if (primaryKey.isEmpty) { + primaryKey = p.value.toList + } + + case p: IngestProcessor => processors = processors :+ p - case _ => // ignore others (rename/remove...) ou gère-les si tu veux les remonter en DDL - } } // 4. Final construction of the DdlTable - DdlTable( + Table( name = name, columns = enrichedCols.values.toList.sortBy(_.name), primaryKey = primaryKey, partitionBy = partitionBy, - esMappings.options, - esSettings.options + mappings = esMappings.options, + settings = esSettings.options, + processors = processors.toSeq, + aliases = esAliases.aliases.map(entry => entry._1 -> entry._2.node) ).update() } } + object Index { + def apply(name: String, json: String): Index = { + val root: JsonNode = json + apply(name, root) + } + + def apply(name: String, root: JsonNode): Index = { + if (root.has(name)) { + val indexNode = root.path(name) + return apply(name, indexNode) + } + val mappings = root.path("mappings") + val settings = root.path("settings") + val aliasesNode = root.path("aliases") + val aliases: Map[String, JsonNode] = + if (aliasesNode != null && aliasesNode.isObject) { + aliasesNode + .properties() + .asScala + .map { entry => + val aliasName = entry.getKey + val aliasValue = entry.getValue + aliasName -> aliasValue + } + .toMap + } else { + Map.empty + } + Index( + name = name, + mappings = mappings, + settings = settings, + aliases = aliases + ) + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 5daacfba..d57c9bfa 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -31,12 +31,12 @@ import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ - DdlColumn, - DdlPartition, - DdlPipelineType, - DdlProcessor, - DdlProcessorType, - DdlScriptProcessor + Column, + IngestPipelineType, + IngestProcessor, + IngestProcessorType, + PartitionDate, + ScriptProcessor } import app.softnetwork.elastic.sql.time.TimeUnit @@ -91,21 +91,21 @@ object Parser opts.toMap } - def processorType: PackratParser[DdlProcessorType] = + def processorType: PackratParser[IngestProcessorType] = ident ^^ { name => name.toLowerCase match { - case "set" => DdlProcessorType.Set - case "script" => DdlProcessorType.Script - case "rename" => DdlProcessorType.Rename - case "remove" => DdlProcessorType.Remove - case "date_index_name" => DdlProcessorType.DateIndexName - case other => DdlProcessorType(other) + case "set" => IngestProcessorType.Set + case "script" => IngestProcessorType.Script + case "rename" => IngestProcessorType.Rename + case "remove" => IngestProcessorType.Remove + case "date_index_name" => IngestProcessorType.DateIndexName + case other => IngestProcessorType(other) } } - def processor: PackratParser[DdlProcessor] = + def processor: PackratParser[IngestProcessor] = processorType ~ objectValue ^^ { case pt ~ opts => - DdlProcessor(pt, opts) + IngestProcessor(pt, opts) } def createOrReplacePipeline: PackratParser[CreatePipeline] = @@ -113,7 +113,7 @@ object Parser processor, separator ) ~ end ^^ { case _ ~ name ~ _ ~ _ ~ proc ~ _ => - CreatePipeline(name, DdlPipelineType.Custom, orReplace = true, processors = proc) + CreatePipeline(name, IngestPipelineType.Custom, orReplace = true, processors = proc) } def createPipeline: PackratParser[CreatePipeline] = @@ -121,7 +121,7 @@ object Parser processor, separator ) <~ end ^^ { case _ ~ ine ~ name ~ _ ~ proc => - CreatePipeline(name, DdlPipelineType.Custom, ifNotExists = ine, processors = proc) + CreatePipeline(name, IngestPipelineType.Custom, ifNotExists = ine, processors = proc) } def dropPipeline: PackratParser[DropPipeline] = @@ -157,7 +157,7 @@ object Parser AlterPipeline(pipeline, ie, stmts) } - def multiFields: PackratParser[List[DdlColumn]] = + def multiFields: PackratParser[List[Column]] = "FIELDS" ~ start ~> repsep(column, separator) <~ end ^^ (cols => cols) | success(Nil) def ifExists: PackratParser[Boolean] = @@ -200,7 +200,7 @@ object Parser identifierWithIntervalFunction | identifierWithFunction) ~ end ^^ { case _ ~ _ ~ s ~ _ => s } - def column: PackratParser[DdlColumn] = + def column: PackratParser[Column] = ident ~ extension_type ~ (script | multiFields) ~ defaultVal ~ notNull ~ comment ~ (options | success( Map.empty[String, Value[_]] )) ^^ { case name ~ dt ~ mfs ~ dv ~ nn ~ ct ~ opts => @@ -219,11 +219,11 @@ object Parser val temp = multiple.dropRight(1) :+ s" ctx.$name = $last" temp.mkString(";") } - DdlColumn( + Column( name, dt, Some( - DdlScriptProcessor( + ScriptProcessor( script = script.sql, column = name, dataType = dt, @@ -236,12 +236,12 @@ object Parser ct, opts ) - case cols: List[DdlColumn] => - DdlColumn(name, dt, None, cols, dv, nn, ct, opts) + case cols: List[Column] => + Column(name, dt, None, cols, dv, nn, ct, opts) } } - def columns: PackratParser[List[DdlColumn]] = + def columns: PackratParser[List[Column]] = start ~ repsep(column, separator) ~ end ^^ { case _ ~ cols ~ _ => cols } def primaryKey: PackratParser[List[String]] = @@ -258,14 +258,14 @@ object Parser ("MINUTE" ^^^ TimeUnit.MINUTES) | ("SECOND" ^^^ TimeUnit.SECONDS)) ~ end ^^ { case _ ~ gf ~ _ => gf } - def partitionBy: PackratParser[Option[DdlPartition]] = + def partitionBy: PackratParser[Option[PartitionDate]] = opt("PARTITION" ~ "BY" ~ ident ~ opt(granularity)) ^^ { - case Some(_ ~ _ ~ pb ~ gf) => Some(DdlPartition(pb, gf.getOrElse(TimeUnit.DAYS))) + case Some(_ ~ _ ~ pb ~ gf) => Some(PartitionDate(pb, gf.getOrElse(TimeUnit.DAYS))) case None => None } def columnsWithPartitionBy - : PackratParser[(List[DdlColumn], List[String], Option[DdlPartition], Map[String, Any])] = + : PackratParser[(List[Column], List[String], Option[PartitionDate], Map[String, Any])] = start ~ repsep( column, separator @@ -280,9 +280,9 @@ object Parser case _ ~ name ~ lr => lr match { case ( - cols: List[DdlColumn], + cols: List[Column], pk: List[String], - p: Option[DdlPartition], + p: Option[PartitionDate], opts: Map[String, Value[_]] ) => CreateTable( @@ -304,9 +304,9 @@ object Parser case _ ~ ine ~ name ~ lr => lr match { case ( - cols: List[DdlColumn], + cols: List[Column], pk: List[String], - p: Option[DdlPartition], + p: Option[PartitionDate], opts: Map[String, Value[_]] ) => CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p, options = opts) @@ -398,7 +398,7 @@ object Parser } AlterColumnScript( name, - DdlScriptProcessor( + ScriptProcessor( script = ns.sql, column = name, dataType = ns.out, @@ -459,6 +459,14 @@ object Parser def dropTableSetting: PackratParser[DropTableSetting] = ("DROP" ~ "SETTING") ~> ident ^^ { m => DropTableSetting(m) } + def alterTableAlias: PackratParser[AlterTableAlias] = + (("SET" | "ADD") ~ "ALIAS") ~ start ~> option <~ end ^^ { opt => + AlterTableAlias(opt._1, opt._2) + } + + def dropTableAlias: PackratParser[DropTableAlias] = + ("DROP" ~ "ALIAS") ~> ident ^^ { m => DropTableAlias(m) } + def alterTableStatement: PackratParser[AlterTableStatement] = addColumn | dropColumn | @@ -481,7 +489,9 @@ object Parser alterTableMapping | dropTableMapping | alterTableSetting | - dropTableSetting + dropTableSetting | + alterTableAlias | + dropTableAlias def alterTable: PackratParser[AlterTable] = ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ start.? ~ repsep( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index e9be244f..2b115c06 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -18,17 +18,17 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.schema.{ - DdlColumn, - DdlDefaultValueProcessor, - DdlPartition, - DdlPipeline, - DdlPipelineType, - DdlProcessor, - DdlProcessorType, - DdlRemoveProcessor, - DdlRenameProcessor, - DdlScriptProcessor, - DdlTable + Column, + DefaultValueProcessor, + IngestPipeline, + IngestPipelineType, + IngestProcessor, + IngestProcessorType, + PartitionDate, + RemoveProcessor, + RenameProcessor, + ScriptProcessor, + Table => DdlTable } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction @@ -317,10 +317,10 @@ package object query { case class CreatePipeline( name: String, - pipelineType: DdlPipelineType, + pipelineType: IngestPipelineType, ifNotExists: Boolean = false, orReplace: Boolean = false, - processors: Seq[DdlProcessor] + processors: Seq[IngestProcessor] ) extends PipelineStatement { override def sql: String = { val processorsDdl = processors.map(_.ddl).mkString(", ") @@ -329,23 +329,23 @@ package object query { s"CREATE$replaceClause PIPELINE$ineClause $name WITH PROCESSORS ($processorsDdl)" } - lazy val ddlPipeline: DdlPipeline = - DdlPipeline(name, pipelineType, processors) + lazy val ddlPipeline: IngestPipeline = + IngestPipeline(name, pipelineType, processors) } sealed trait AlterPipelineStatement extends AlterTableStatement - case class AddPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + case class AddPipelineProcessor(processor: IngestProcessor) extends AlterPipelineStatement { override def sql: String = s"ADD PROCESSOR ${processor.ddl}" - override def ddlProcessor: Option[DdlProcessor] = Some(processor) + override def ddlProcessor: Option[IngestProcessor] = Some(processor) } - case class DropPipelineProcessor(processorType: DdlProcessorType, column: String) + case class DropPipelineProcessor(processorType: IngestProcessorType, column: String) extends AlterPipelineStatement { override def sql: String = s"DROP PROCESSOR ${processorType.name.toUpperCase}($column)" } - case class AlterPipelineProcessor(processor: DdlProcessor) extends AlterPipelineStatement { + case class AlterPipelineProcessor(processor: IngestProcessor) extends AlterPipelineStatement { override def sql: String = s"ALTER PROCESSOR ${processor.ddl}" - override def ddlProcessor: Option[DdlProcessor] = Some(processor) + override def ddlProcessor: Option[IngestProcessor] = Some(processor) } case class AlterPipeline( @@ -364,10 +364,14 @@ package object query { s"ALTER PIPELINE $name$ifExistsClause $statementsSql" } - lazy val ddlProcessors: Seq[DdlProcessor] = statements.flatMap(_.ddlProcessor) + lazy val ddlProcessors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) - lazy val pipeline: DdlPipeline = - DdlPipeline(s"alter-pipeline-$name-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) + lazy val pipeline: IngestPipeline = + IngestPipeline( + s"alter-pipeline-$name-${Instant.now}", + IngestPipelineType.Custom, + ddlProcessors + ) } case class DropPipeline(name: String, ifExists: Boolean = false) extends PipelineStatement { @@ -379,14 +383,16 @@ package object query { case class CreateTable( table: String, - ddl: Either[DqlStatement, List[DdlColumn]], + ddl: Either[DqlStatement, List[Column]], ifNotExists: Boolean = false, orReplace: Boolean = false, primaryKey: List[String] = Nil, - partitionBy: Option[DdlPartition] = None, + partitionBy: Option[PartitionDate] = None, options: Map[String, Value[_]] = Map.empty ) extends DdlStatement { + lazy val partitioned: Boolean = partitionBy.isDefined + override def sql: String = { val replaceClause = if (orReplace) " OR REPLACE" else "" val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" @@ -399,16 +405,16 @@ package object query { } } - lazy val columns: Seq[DdlColumn] = { + lazy val columns: Seq[Column] = { ddl match { case Left(select) => select match { case s: SingleSearch => - s.select.fields.map(f => DdlColumn(f.identifier.aliasOrName, f.out)) + s.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) case m: MultiSearch => m.requests.headOption .map { req => - req.select.fields.map(f => DdlColumn(f.identifier.aliasOrName, f.out)) + req.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) } .getOrElse(Nil) case _ => Nil @@ -435,16 +441,26 @@ package object query { case None => Map.empty } + lazy val aliases: Map[String, Value[_]] = options.get("aliases") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + lazy val ddlTable: DdlTable = DdlTable( name = table, columns = columns.toList, primaryKey = primaryKey, partitionBy = partitionBy, mappings = mappings, - settings = settings + settings = settings, + aliases = aliases ).update() - lazy val ddlPipeline: DdlPipeline = ddlTable.ddlPipeline + lazy val ddlPipeline: IngestPipeline = ddlTable.ddlPipeline } @@ -461,17 +477,16 @@ package object query { s"ALTER TABLE $table$ifExistsClause $statementsSql" } - lazy val ddlProcessors: Seq[DdlProcessor] = statements.flatMap(_.ddlProcessor) + lazy val ddlProcessors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) - lazy val pipeline: DdlPipeline = - DdlPipeline(s"alter-$table-${Instant.now}", DdlPipelineType.Custom, ddlProcessors) + lazy val pipeline: IngestPipeline = + IngestPipeline(s"alter-$table-${Instant.now}", IngestPipelineType.Custom, ddlProcessors) } sealed trait AlterTableStatement extends Token { - def ddlProcessor: Option[DdlProcessor] = None + def ddlProcessor: Option[IngestProcessor] = None } - case class AddColumn(column: DdlColumn, ifNotExists: Boolean = false) - extends AlterTableStatement { + case class AddColumn(column: Column, ifNotExists: Boolean = false) extends AlterTableStatement { override def sql: String = { val ifNotExistsClause = if (ifNotExists) " IF NOT EXISTS" else "" s"ADD COLUMN$ifNotExistsClause ${column.sql}" @@ -482,11 +497,11 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"DROP COLUMN$ifExistsClause $columnName" } - override def ddlProcessor: Option[DdlProcessor] = Some(DdlRemoveProcessor(sql, columnName)) + override def ddlProcessor: Option[IngestProcessor] = Some(RemoveProcessor(sql, columnName)) } case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { override def sql: String = s"RENAME COLUMN $oldName TO $newName" - override def ddlProcessor: Option[DdlProcessor] = Some(DdlRenameProcessor(oldName, newName)) + override def ddlProcessor: Option[IngestProcessor] = Some(RenameProcessor(oldName, newName)) } case class AlterColumnOptions( columnName: String, @@ -530,7 +545,7 @@ package object query { } case class AlterColumnScript( columnName: String, - newScript: DdlScriptProcessor, + newScript: ScriptProcessor, ifExists: Boolean = false ) extends AlterTableStatement { override def sql: String = { @@ -554,9 +569,9 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"ALTER COLUMN$ifExistsClause $columnName SET DEFAULT $defaultValue" } - override def ddlProcessor: Option[DdlProcessor] = + override def ddlProcessor: Option[IngestProcessor] = Some( - DdlDefaultValueProcessor( + DefaultValueProcessor( sql, columnName, defaultValue @@ -603,7 +618,7 @@ package object query { } case class AlterColumnFields( columnName: String, - fields: Seq[DdlColumn], + fields: Seq[Column], ifExists: Boolean = false ) extends AlterTableStatement { override def sql: String = { @@ -614,7 +629,7 @@ package object query { } case class AlterColumnField( columnName: String, - field: DdlColumn, + field: Column, ifExists: Boolean = false ) extends AlterTableStatement { override def sql: String = { @@ -650,6 +665,14 @@ package object query { override def sql: String = s"DROP SETTING $optionKey" } + case class AlterTableAlias(optionKey: String, optionValue: Value[_]) extends AlterTableStatement { + override def sql: String = + s"SET ALIAS ($optionKey = $optionValue)" + } + case class DropTableAlias(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP ALIAS $optionKey" + } case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) extends DdlStatement { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala similarity index 84% rename from sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala index 12d8b382..ce18d118 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/DdlTableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala @@ -26,7 +26,7 @@ sealed trait AlterTableStatementDiff { sealed trait ColumnDiff extends AlterTableStatementDiff -case class ColumnAdded(column: DdlColumn) extends ColumnDiff { +case class ColumnAdded(column: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AddColumn(column) } case class ColumnRemoved(name: String) extends ColumnDiff { @@ -45,7 +45,7 @@ case class ColumnDefaultRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnDefault(name) } -case class ColumnScriptSet(name: String, script: DdlScriptProcessor) extends ColumnDiff { +case class ColumnScriptSet(name: String, script: ScriptProcessor) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnScript(name, script) } case class ColumnScriptRemoved(name: String) extends ColumnDiff { @@ -73,13 +73,13 @@ case class ColumnOptionRemoved(name: String, key: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnOption(name, key) } -case class FieldAdded(column: String, field: DdlColumn) extends ColumnDiff { +case class FieldAdded(column: String, field: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnField(column, field) } case class FieldRemoved(column: String, fieldName: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnField(column, fieldName) } -case class FieldAltered(column: String, field: DdlColumn) extends ColumnDiff { +case class FieldAltered(column: String, field: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnField(column, field) } @@ -101,22 +101,31 @@ case class SettingRemoved(key: String) extends SettingDiff { override def stmt: AlterTableStatement = DropTableSetting(key) } +sealed trait AliasDiff extends AlterTableStatementDiff + +case class AliasSet(key: String, value: Value[_]) extends AliasDiff { + override def stmt: AlterTableStatement = AlterTableAlias(key, value) +} +case class AliasRemoved(key: String) extends AliasDiff { + override def stmt: AlterTableStatement = DropTableAlias(key) +} + sealed trait AlterPipelineStatementDiff { def stmt: AlterPipelineStatement } sealed trait PipelineDiff extends AlterPipelineStatementDiff -case class ProcessorAdded(processor: DdlProcessor) extends PipelineDiff { +case class ProcessorAdded(processor: IngestProcessor) extends PipelineDiff { override def stmt: AlterPipelineStatement = AddPipelineProcessor(processor) } -case class ProcessorRemoved(processor: DdlProcessor) extends PipelineDiff { +case class ProcessorRemoved(processor: IngestProcessor) extends PipelineDiff { override def stmt: AlterPipelineStatement = DropPipelineProcessor(processor.processorType, processor.column) } case class ProcessorTypeChanged( - actual: DdlProcessorType, - desired: DdlProcessorType + actual: IngestProcessorType, + desired: IngestProcessorType ) sealed trait ProcessorPropertyDiff case class ProcessorPropertyAdded(key: String, value: Any) extends ProcessorPropertyDiff @@ -127,18 +136,19 @@ case class ProcessorDiff( propertyDiffs: List[ProcessorPropertyDiff] ) case class ProcessorChanged( - from: DdlProcessor, - to: DdlProcessor, + from: IngestProcessor, + to: IngestProcessor, diff: ProcessorDiff ) extends PipelineDiff { override def stmt: AlterPipelineStatement = AlterPipelineProcessor(to) } -case class DdlTableDiff( +case class TableDiff( columns: List[ColumnDiff], mappings: List[MappingDiff], settings: List[SettingDiff], - pipeline: List[PipelineDiff] + pipeline: List[PipelineDiff], + aliases: List[AliasDiff] ) { def isEmpty: Boolean = columns.isEmpty && mappings.isEmpty && settings.isEmpty && pipeline.isEmpty @@ -149,7 +159,8 @@ case class DdlTableDiff( } else { val statements = columns.map(_.stmt) ++ mappings.map(_.stmt) ++ - settings.map(_.stmt) + settings.map(_.stmt) ++ + aliases.map(_.stmt) Some( AlterTable( tableName, diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 08213451..77cf4066 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -33,32 +33,32 @@ package object schema { lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() - sealed trait DdlProcessorType { + sealed trait IngestProcessorType { def name: String } - object DdlProcessorType { - case object Script extends DdlProcessorType { + object IngestProcessorType { + case object Script extends IngestProcessorType { def name: String = "script" } - case object Rename extends DdlProcessorType { + case object Rename extends IngestProcessorType { def name: String = "rename" } - case object Remove extends DdlProcessorType { + case object Remove extends IngestProcessorType { def name: String = "remove" } - case object Set extends DdlProcessorType { + case object Set extends IngestProcessorType { def name: String = "set" } - case object DateIndexName extends DdlProcessorType { + case object DateIndexName extends IngestProcessorType { def name: String = "date_index_name" } - def apply(n: String): DdlProcessorType = new DdlProcessorType { + def apply(n: String): IngestProcessorType = new IngestProcessorType { override def name: String = n } } - sealed trait DdlProcessor extends DdlToken { + sealed trait IngestProcessor extends DdlToken { def column: String def ignoreFailure: Boolean final def node: ObjectNode = { @@ -67,7 +67,7 @@ package object schema { node } def json: String = mapper.writeValueAsString(node) - def processorType: DdlProcessorType + def processorType: IngestProcessorType def description: String = sql.trim def name: String = processorType.name def properties: Map[String, Any] @@ -92,7 +92,7 @@ package object schema { .toMap } - def diff(to: DdlProcessor): Option[ProcessorDiff] = { + def diff(to: IngestProcessor): Option[ProcessorDiff] = { val from = this @@ -137,17 +137,17 @@ package object schema { override def ddl: String = s"${processorType.name.toUpperCase}${Value(properties).ddl}" } - object DdlProcessor { + object IngestProcessor { private val ScriptDescRegex = """^\s*([a-zA-Z0-9_\\.]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r - def apply(processorType: DdlProcessorType, properties: ObjectValue): DdlProcessor = { + def apply(processorType: IngestProcessorType, properties: ObjectValue): IngestProcessor = { val node = mapper.createObjectNode() node.set(processorType.name, properties.toJson) - apply(node) + apply(IngestPipelineType.Default, node) } - def apply(processor: JsonNode): DdlProcessor = { + def apply(pipelineType: IngestPipelineType, processor: JsonNode): IngestProcessor = { val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. val props = processor.get(processorType) @@ -168,14 +168,14 @@ package object schema { .trim // DdlPrimaryKeyProcessor val cols = value.split(sqlConfig.compositeKeySeparator).toSet - DdlPrimaryKeyProcessor( + PrimaryKeyProcessor( sql = desc, column = "_id", value = cols, ignoreFailure = ignoreFailure ) } else { - DdlDefaultValueProcessor( + DefaultValueProcessor( sql = desc, column = field, value = Value(valueNode.asText()), @@ -192,7 +192,7 @@ package object schema { desc match { case ScriptDescRegex(col, dataType, script) => - DdlScriptProcessor( + ScriptProcessor( script = script, column = col, dataType = SQLTypes(dataType), @@ -201,7 +201,7 @@ package object schema { ) case _ => GenericProcessor( - processorType = DdlProcessorType.Script, + processorType = IngestProcessorType.Script, properties = mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap ) @@ -216,7 +216,7 @@ package object schema { .getOrElse(Nil) val prefix = props.get("index_name_prefix").asText() - DdlDateIndexNameProcessor( + DateIndexNameProcessor( sql = desc, column = field, dateRounding = rounding, @@ -226,7 +226,7 @@ package object schema { case other => GenericProcessor( - processorType = DdlProcessorType(other), + processorType = IngestProcessorType(other), properties = mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap ) @@ -235,8 +235,8 @@ package object schema { } } - case class GenericProcessor(processorType: DdlProcessorType, properties: Map[String, Any]) - extends DdlProcessor { + case class GenericProcessor(processorType: IngestProcessorType, properties: Map[String, Any]) + extends IngestProcessor { override def sql: String = ddl override def column: String = properties.get("field") match { case Some(s: String) => s @@ -248,18 +248,18 @@ package object schema { } } - case class DdlScriptProcessor( + case class ScriptProcessor( script: String, column: String, dataType: SQLType, source: String, ignoreFailure: Boolean = true - ) extends DdlProcessor { + ) extends IngestProcessor { override def sql: String = s"$column $dataType SCRIPT AS ($script)" override def baseType: SQLType = dataType - def processorType: DdlProcessorType = DdlProcessorType.Script + def processorType: IngestProcessorType = IngestProcessorType.Script override def properties: Map[String, Any] = Map( "description" -> description, @@ -270,13 +270,13 @@ package object schema { } - case class DdlRenameProcessor( + case class RenameProcessor( column: String, newName: String, ignoreFailure: Boolean = true, ignoreMissing: Boolean = true - ) extends DdlProcessor { - def processorType: DdlProcessorType = DdlProcessorType.Rename + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Rename def sql: String = s"$column RENAME TO $newName" @@ -290,13 +290,13 @@ package object schema { } - case class DdlRemoveProcessor( + case class RemoveProcessor( sql: String, column: String, ignoreFailure: Boolean = true, ignoreMissing: Boolean = true - ) extends DdlProcessor { - def processorType: DdlProcessorType = DdlProcessorType.Remove + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Remove override def properties: Map[String, Any] = Map( "description" -> description, @@ -307,15 +307,15 @@ package object schema { } - case class DdlPrimaryKeyProcessor( + case class PrimaryKeyProcessor( sql: String, column: String, value: Set[String], ignoreFailure: Boolean = false, ignoreEmptyValue: Boolean = false, separator: String = "\\|\\|" - ) extends DdlProcessor { - def processorType: DdlProcessorType = DdlProcessorType.Set + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Set override def properties: Map[String, Any] = Map( "description" -> description, @@ -327,13 +327,13 @@ package object schema { } - case class DdlDefaultValueProcessor( + case class DefaultValueProcessor( sql: String, column: String, value: Value[_], ignoreFailure: Boolean = true - ) extends DdlProcessor { - def processorType: DdlProcessorType = DdlProcessorType.Set + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Set def _if: String = { if (column.contains(".")) @@ -356,7 +356,7 @@ package object schema { ) } - case class DdlDateIndexNameProcessor( + case class DateIndexNameProcessor( sql: String, column: String, dateRounding: String, @@ -364,8 +364,8 @@ package object schema { prefix: String, separator: String = "-", ignoreFailure: Boolean = true - ) extends DdlProcessor { - def processorType: DdlProcessorType = DdlProcessorType.DateIndexName + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.DateIndexName override def properties: Map[String, Any] = Map( "description" -> description, @@ -381,10 +381,10 @@ package object schema { implicit def primaryKeyToDdlProcessor( primaryKey: List[String] - ): Seq[DdlProcessor] = { + ): Seq[IngestProcessor] = { if (primaryKey.nonEmpty) { Seq( - DdlPrimaryKeyProcessor( + PrimaryKeyProcessor( sql = s"PRIMARY KEY (${primaryKey.mkString(", ")})", column = "_id", value = primaryKey.toSet @@ -395,26 +395,26 @@ package object schema { } } - sealed trait DdlPipelineType { + sealed trait IngestPipelineType { def name: String } - object DdlPipelineType { - case object Default extends DdlPipelineType { + object IngestPipelineType { + case object Default extends IngestPipelineType { def name: String = "DEFAULT" } - case object Final extends DdlPipelineType { + case object Final extends IngestPipelineType { def name: String = "FINAL" } - case object Custom extends DdlPipelineType { + case object Custom extends IngestPipelineType { def name: String = "CUSTOM" } } - case class DdlPipeline( + case class IngestPipeline( name: String, - ddlPipelineType: DdlPipelineType, - ddlProcessors: Seq[DdlProcessor] + ddlPipelineType: IngestPipelineType, + ddlProcessors: Seq[IngestProcessor] ) extends DdlToken { def sql: String = s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql.trim).mkString(", ")})" @@ -435,14 +435,14 @@ package object schema { def json: String = mapper.writeValueAsString(node) - def diff(pipeline: DdlPipeline): List[PipelineDiff] = { + def diff(pipeline: IngestPipeline): List[PipelineDiff] = { val actual = this.ddlProcessors val desired = pipeline.ddlProcessors // 1. Index processors by logical key - def key(p: DdlProcessor) = (p.processorType, p.column) + def key(p: IngestProcessor) = (p.processorType, p.column) val desiredMap = desired.map(p => key(p) -> p).toMap val actualMap = actual.map(p => key(p) -> p).toMap @@ -472,7 +472,7 @@ package object schema { diffs.toList } - def merge(statements: Seq[AlterPipelineStatement]): DdlPipeline = { + def merge(statements: Seq[AlterPipelineStatement]): IngestPipeline = { statements.foldLeft(this) { (current, alter) => alter match { case AddPipelineProcessor(processor) => @@ -500,41 +500,48 @@ package object schema { } } - object DdlPipeline { - def apply(name: String, json: String): DdlPipeline = { + object IngestPipeline { + def apply( + name: String, + json: String, + pipelineType: Option[IngestPipelineType] = None + ): IngestPipeline = { + val ddlPipelineType = pipelineType.getOrElse( + if (name.startsWith("default-")) IngestPipelineType.Default + else if (name.startsWith("final-")) IngestPipelineType.Final + else IngestPipelineType.Custom + ) val node = mapper.readTree(json) val processorsNode = node.get("processors") val processors = processorsNode.elements().asScala.toSeq.map { p => - DdlProcessor(p) + IngestProcessor(ddlPipelineType, p) } - DdlPipeline( + IngestPipeline( name = name, - ddlPipelineType = - if (name.startsWith("default-")) DdlPipelineType.Default - else if (name.startsWith("final-")) DdlPipelineType.Final - else DdlPipelineType.Custom, + ddlPipelineType = ddlPipelineType, ddlProcessors = processors ) } } - case class DdlColumn( + + case class Column( name: String, dataType: SQLType, - script: Option[DdlScriptProcessor] = None, - multiFields: List[DdlColumn] = Nil, + script: Option[ScriptProcessor] = None, + multiFields: List[Column] = Nil, defaultValue: Option[Value[_]] = None, notNull: Boolean = false, comment: Option[String] = None, options: Map[String, Value[_]] = Map.empty, - struct: Option[DdlColumn] = None + struct: Option[Column] = None ) extends DdlToken { def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) private def level: Int = struct.map(_.level + 1).getOrElse(0) - private def cols: Map[String, DdlColumn] = multiFields.map(field => field.name -> field).toMap + private def cols: Map[String, Column] = multiFields.map(field => field.name -> field).toMap /* Recursive find */ - def find(path: String): Option[DdlColumn] = { + def find(path: String): Option[Column] = { if (path.contains(".")) { val parts = path.split("\\.") cols.get(parts.head).flatMap { col => @@ -547,7 +554,7 @@ package object schema { } } - def update(struct: Option[DdlColumn] = None): DdlColumn = { + def update(struct: Option[Column] = None): Column = { val updated = this.copy(struct = struct) val updated_script = script.map { sc => @@ -610,9 +617,9 @@ package object schema { s"$tabs$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" } - def ddlProcessors: Seq[DdlProcessor] = script.map(st => st.copy(column = path)).toSeq ++ + def ddlProcessors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => - DdlDefaultValueProcessor( + DefaultValueProcessor( sql = s"$path DEFAULT $dv", column = path, value = dv @@ -642,7 +649,7 @@ package object schema { root } - def diff(desired: DdlColumn, parent: Option[DdlColumn] = None): List[ColumnDiff] = { + def diff(desired: Column, parent: Option[Column] = None): List[ColumnDiff] = { val actual = this val diffs = scala.collection.mutable.ListBuffer[ColumnDiff]() @@ -811,7 +818,7 @@ package object schema { } } - case class DdlPartition(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends DdlToken { + case class PartitionDate(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends DdlToken { def sql: String = s" PARTITION BY $column ($granularity)" val dateRounding: String = granularity.script.get @@ -828,8 +835,8 @@ package object schema { case _ => List.empty } - def ddlProcessor(table: DdlTable): DdlDateIndexNameProcessor = - DdlDateIndexNameProcessor( + def ddlProcessor(table: Table): DateIndexNameProcessor = + DateIndexNameProcessor( sql, column, dateRounding, @@ -838,21 +845,112 @@ package object schema { ) } - case class DdlColumnNotFound(column: String, table: String) + case class ColumnNotFound(column: String, table: String) extends Exception(s"Column $column does not exist in table $table") - case class DdlTable( + case class TableAlias( + table: String, + alias: String, + filter: Map[String, Any] = Map.empty, + routing: Option[String] = None, + indexRouting: Option[String] = None, + searchRouting: Option[String] = None, + isWriteIndex: Boolean = false, + isHidden: Boolean = false + ) { + + def node: ObjectNode = { + val node = mapper.createObjectNode() + if (filter.nonEmpty) { + node.set("filter", Value(filter).asInstanceOf[ObjectValue].toJson) + } + routing.foreach(r => node.put("routing", r)) + indexRouting.foreach(r => node.put("index_routing", r)) + searchRouting.foreach(r => node.put("search_routing", r)) + if (isWriteIndex) { + node.put("is_write_index", true) + } + if (isHidden) { + node.put("is_hidden", true) + } + node + } + + def json: String = node + + def ddl: String = { + val opts = scala.collection.mutable.ListBuffer[String]() + if (filter.nonEmpty) { + opts += s"filter = ${Value(filter).ddl}" + } + routing.foreach(r => opts += s"routing = '$r'") + indexRouting.foreach(r => opts += s"index_routing = '$r'") + searchRouting.foreach(r => opts += s"search_routing = '$r'") + if (isWriteIndex) { + opts += s"is_write_index = true" + } + if (isHidden) { + opts += s"is_hidden = true" + } + s"ALTER TABLE $table SET ALIAS ($alias = ${if (opts.nonEmpty) s"(${opts.mkString(", ")})" + else "()"})" + } + } + + object TableAlias { + def apply(table: String, name: String, value: Value[_]): TableAlias = { + val obj = value.asInstanceOf[ObjectValue] + val filter = obj.value.get("filter") match { + case Some(ObjectValue(f)) => + f.map { case (k, v) => k -> v.value } + case _ => Map.empty[String, Any] + } + val routing = obj.value.get("routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val indexRouting = obj.value.get("index_routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val searchRouting = obj.value.get("search_routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val isWriteIndex = obj.value.get("is_write_index") match { + case Some(BooleanValue(b)) => b + case _ => false + } + val isHidden = obj.value.get("is_hidden") match { + case Some(BooleanValue(b)) => b + case _ => false + } + TableAlias( + table = table, + alias = name, + filter = filter, + routing = routing, + indexRouting = indexRouting, + searchRouting = searchRouting, + isWriteIndex = isWriteIndex, + isHidden = isHidden + ) + } + } + + case class Table( name: String, - columns: List[DdlColumn], + columns: List[Column], primaryKey: List[String] = Nil, - partitionBy: Option[DdlPartition] = None, + partitionBy: Option[PartitionDate] = None, mappings: Map[String, Value[_]] = Map.empty, settings: Map[String, Value[_]] = Map.empty, - processors: Seq[DdlProcessor] = Seq.empty + processors: Seq[IngestProcessor] = Seq.empty, + aliases: Map[String, Value[_]] = Map.empty ) extends DdlToken { - private[schema] lazy val cols: Map[String, DdlColumn] = columns.map(c => c.name -> c).toMap + private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap - def find(path: String): Option[DdlColumn] = { + def find(path: String): Option[Column] = { if (path.contains(".")) { val parts = path.split("\\.") cols.get(parts.head).flatMap { col => @@ -883,7 +981,7 @@ package object schema { ) .getOrElse(Map.empty) - def update(): DdlTable = this.copy( + def update(): Table = this.copy( columns = columns.map(_.update()), mappings = mappings ++ Map( "_meta" -> @@ -910,8 +1008,14 @@ package object schema { } else { "" } + val aliasesOpts = + if (aliases.nonEmpty) { + s"aliases = ${ObjectValue(aliases).ddl}" + } else { + "" + } val separator = if (partitionBy.nonEmpty) "," else "" - s"$separator OPTIONS = (${Seq(mappingOpts, settingsOpts).filter(_.nonEmpty).mkString(", ")})" + s"$separator OPTIONS = (${Seq(mappingOpts, settingsOpts, aliasesOpts).filter(_.nonEmpty).mkString(", ")})" } else { "" } @@ -924,12 +1028,12 @@ package object schema { s"CREATE OR REPLACE TABLE $name (\n\t$cols$pkStr)${partitionBy.getOrElse("")}$opts" } - def ddlProcessors: Seq[DdlProcessor] = + def ddlProcessors: Seq[IngestProcessor] = columns.flatMap(_.ddlProcessors) ++ partitionBy .map(_.ddlProcessor(this)) - .toSeq ++ implicitly[Seq[DdlProcessor]](primaryKey) + .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) - def merge(statements: Seq[AlterTableStatement]): DdlTable = { + def merge(statements: Seq[AlterTableStatement]): Table = { statements.foldLeft(this) { (table, statement) => statement match { // table columns @@ -937,12 +1041,12 @@ package object schema { if (ifNotExists && table.cols.contains(column.name)) table else if (!table.cols.contains(column.name)) table.copy(columns = table.columns :+ column) - else throw DdlColumnNotFound(column.name, table.name) + else throw ColumnNotFound(column.name, table.name) case DropColumn(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) table.copy(columns = table.columns.filterNot(_.name == columnName)) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case RenameColumn(oldName, newName) => if (cols.contains(oldName)) table.copy( @@ -950,7 +1054,7 @@ package object schema { if (col.name == oldName) col.copy(name = newName) else col } ) - else throw DdlColumnNotFound(oldName, table.name) + else throw ColumnNotFound(oldName, table.name) // column type case AlterColumnType(columnName, newType, ifExists) => if (ifExists && !table.cols.contains(columnName)) table @@ -961,7 +1065,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // column script case AlterColumnScript(columnName, newScript, ifExists) => if (ifExists && !table.cols.contains(columnName)) table @@ -973,7 +1077,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case DropColumnScript(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -983,7 +1087,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // column default value case AlterColumnDefault(columnName, newDefault, ifExists) => if (ifExists && !table.cols.contains(columnName)) table @@ -994,7 +1098,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case DropColumnDefault(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1004,7 +1108,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // column not null case AlterColumnNotNull(columnName, ifExists) => if (!table.cols.contains(columnName) && ifExists) table @@ -1015,7 +1119,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case DropColumnNotNull(columnName, ifExists) => if (!table.cols.contains(columnName) && ifExists) table else if (table.cols.contains(columnName)) @@ -1025,7 +1129,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // column options case AlterColumnOptions(columnName, newOptions, ifExists) => if (ifExists && !table.cols.contains(columnName)) table @@ -1037,7 +1141,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case AlterColumnOption( columnName, optionKey, @@ -1055,7 +1159,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case DropColumnOption( columnName, optionKey, @@ -1072,7 +1176,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // column comments case AlterColumnComment(columnName, newComment, ifExists) => if (ifExists && !table.cols.contains(columnName)) table @@ -1084,7 +1188,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) case DropColumnComment(columnName, ifExists) => if (ifExists && !table.cols.contains(columnName)) table else if (table.cols.contains(columnName)) @@ -1095,7 +1199,7 @@ package object schema { else col } ) - else throw DdlColumnNotFound(columnName, table.name) + else throw ColumnNotFound(columnName, table.name) // multi-fields case AlterColumnFields(columnName, newFields, ifExists) => val col = find(columnName) @@ -1108,7 +1212,7 @@ package object schema { multiFields = newFields.toList ) table - case _ => throw DdlColumnNotFound(columnName, table.name) + case _ => throw ColumnNotFound(columnName, table.name) } } case AlterColumnField( @@ -1125,7 +1229,7 @@ package object schema { val updatedFields = c.multiFields.filterNot(_.name == field.name) :+ field c.copy(multiFields = updatedFields) table - case _ => throw DdlColumnNotFound(columnName, table.name) + case _ => throw ColumnNotFound(columnName, table.name) } } case DropColumnField( @@ -1143,7 +1247,7 @@ package object schema { multiFields = c.multiFields.filterNot(_.name == fieldName) ) table - case _ => throw DdlColumnNotFound(columnName, table.name) + case _ => throw ColumnNotFound(columnName, table.name) } } // mappings / settings @@ -1163,6 +1267,14 @@ package object schema { table.copy( settings = ObjectValue(table.settings).remove(optionKey).value ) + case AlterTableAlias(aliasName, aliasValue) => + table.copy( + aliases = table.aliases + (aliasName -> aliasValue) + ) + case DropTableAlias(aliasName) => + table.copy( + aliases = table.aliases - aliasName + ) case _ => table } } @@ -1186,11 +1298,11 @@ package object schema { if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) } - lazy val ddlPipeline: DdlPipeline = { + lazy val ddlPipeline: IngestPipeline = { val processorsFromColumns = ddlProcessors.map(p => p.column -> p).toMap - DdlPipeline( + IngestPipeline( name = s"${name}_ddl_default_pipeline", - ddlPipelineType = DdlPipelineType.Default, + ddlPipelineType = IngestPipelineType.Default, ddlProcessors = ddlProcessors ++ processors.filterNot(p => processorsFromColumns.contains(p.column)) ) @@ -1235,11 +1347,15 @@ package object schema { node } + lazy val indexAliases: Seq[TableAlias] = aliases.map { case (aliasName, value) => + TableAlias(name, aliasName, value) + }.toSeq + lazy val pipeline: ObjectNode = { ddlPipeline.node } - def diff(desired: DdlTable): DdlTableDiff = { + def diff(desired: Table): TableDiff = { val actual = this.update() val desiredUpdated = desired.update() @@ -1278,7 +1394,14 @@ package object schema { case Removed(name) => SettingRemoved(name) } - // 6. Pipeline + // 6. Aliases + val aliasDiffs = scala.collection.mutable.ListBuffer[AliasDiff]() + aliasDiffs ++= ObjectValue(actual.aliases).diff(ObjectValue(desired.aliases)).map { + case Altered(name, value) => AliasSet(name, value) + case Removed(name) => AliasRemoved(name) + } + + // 7. Pipeline val pipelineDiffs = scala.collection.mutable.ListBuffer[PipelineDiff]() val actualPipeline = actual.ddlPipeline val desiredPipeline = desiredUpdated.ddlPipeline @@ -1286,11 +1409,12 @@ package object schema { pipelineDiffs += d } - DdlTableDiff( + TableDiff( columns = columnDiffs.toList, mappings = mappingDiffs.toList, settings = settingDiffs.toList, - pipeline = pipelineDiffs.toList + pipeline = pipelineDiffs.toList, + aliases = aliasDiffs.toList ) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index e5b6fc34..f1873c63 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -69,6 +69,11 @@ package object serialization { mapper.writeValueAsString(node) } + implicit def jsonNodeToMap(value: JsonNode): Map[String, Any] = { + import JacksonConfig.{objectMapper => mapper} + mapper.convertValue(value, classOf[Map[String, Any]]) + } + implicit def objectValueToObjectNode(value: ObjectValue): ObjectNode = { import JacksonConfig.{objectMapper => mapper} val node = mapper.createObjectNode() diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 6a15c837..1c7c021e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.`type` -import app.softnetwork.elastic.schema.EsField +import app.softnetwork.elastic.schema.IndexField object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } @@ -74,7 +74,7 @@ object SQLTypes { case _ => Any } - def apply(field: EsField): SQLType = field.`type` match { + def apply(field: IndexField): SQLType = field.`type` match { case "null" => Null case "boolean" => Boolean case "integer" => Int diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index e850809e..7d0af37b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,17 +1,17 @@ package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.schema.EsIndex +import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.{IngestTimestampValue, StringValue} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ mapper, - DdlDateIndexNameProcessor, - DdlDefaultValueProcessor, - DdlPartition, - DdlPrimaryKeyProcessor, - DdlProcessorType, - DdlScriptProcessor + DateIndexNameProcessor, + DefaultValueProcessor, + IngestProcessorType, + PartitionDate, + PrimaryKeyProcessor, + ScriptProcessor } import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec @@ -964,7 +964,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { true, false, List("id"), - Some(DdlPartition("birthdate", TimeUnit.MONTHS)), + Some(PartitionDate("birthdate", TimeUnit.MONTHS)), _ ) => cols.map(_.name) should contain allOf ("id", "name") @@ -1007,11 +1007,11 @@ class ParserSpec extends AnyFlatSpec with Matchers { mappings.set("mappings", indexMappings) val settings = mapper.createObjectNode() settings.set("settings", indexSettings) - val esIndex = EsIndex( + val esIndex = Index( name = "users", mappings = mappings, settings = settings, - pipeline = Some(pipeline) + defaultPipeline = Some(pipeline) ) val ddlTable = esIndex.ddlTable println(s"""esIndex ddl -> ${ddlTable.sql}""") @@ -1340,7 +1340,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.size shouldBe 6 processors.find(_.column == "name") match { case Some( - DdlDefaultValueProcessor( + DefaultValueProcessor( "DEFAULT 'anonymous'", "name", StringValue("anonymous"), @@ -1351,7 +1351,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "age") match { case Some( - DdlScriptProcessor( + ScriptProcessor( "DATE_DIFF(birthdate, CURRENT_DATE, YEAR)", "age", SQLTypes.Int, @@ -1366,7 +1366,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "ingested_at") match { case Some( - DdlDefaultValueProcessor( + DefaultValueProcessor( "DEFAULT _ingest.timestamp", "ingested_at", IngestTimestampValue, @@ -1377,7 +1377,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "profile.seniority") match { case Some( - DdlScriptProcessor( + ScriptProcessor( "DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)", "profile.seniority", SQLTypes.Int, @@ -1392,7 +1392,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "birthdate") match { case Some( - DdlDateIndexNameProcessor( + DateIndexNameProcessor( "PARTITION BY birthdate (MONTH)", "birthdate", "M", @@ -1406,7 +1406,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "_id") match { case Some( - DdlPrimaryKeyProcessor( + PrimaryKeyProcessor( "PRIMARY KEY (id)", "_id", cols, @@ -1446,7 +1446,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) if ie => statements.size shouldBe 2 statements.collect { case AddPipelineProcessor(p) => p } match { - case DdlDefaultValueProcessor( + case DefaultValueProcessor( "status DEFAULT 'active'", "status", StringValue("active"), @@ -1454,7 +1454,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) :: Nil => case other => fail(s"Expected AddPipelineProcessor with DdlSetProcessor, got $other") } - statements.collect { case DropPipelineProcessor(DdlProcessorType.Set, f) => + statements.collect { case DropPipelineProcessor(IngestProcessorType.Set, f) => f } should contain("_id") case _ => fail("Expected AlterPipeline") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 829c2db3..96d07a6c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -23,6 +23,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.serialization._ import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -96,7 +97,7 @@ trait MockElasticClientApi extends ElasticClientApi { index: String, settings: String, mappings: Option[String], - aliases: Seq[String] + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ElasticResult.success(true) From 4ec5e96f63b3123f0195eacf52f0fc07a40c57b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 24 Dec 2025 13:29:44 +0100 Subject: [PATCH 30/95] finalize get index api, fix bulk api for java client --- .../elastic/client/IndicesApi.scala | 39 ++++++++++++- .../elastic/client/jest/JestIndicesApi.scala | 9 ++- .../elastic/client/JestClientSpec.scala | 4 +- .../client/rest/RestHighLevelClientApi.scala | 13 +++-- .../client/rest/RestHighLevelClientApi.scala | 42 ++++++++------ .../elastic/client/java/JavaClientApi.scala | 40 ++++++++----- .../elastic/client/java/JavaClientApi.scala | 35 +++++++----- .../softnetwork/elastic/schema/package.scala | 2 +- .../elastic/sql/schema/package.scala | 7 ++- .../elastic/sql/parser/ParserSpec.scala | 2 +- .../elastic/client/ElasticClientSpec.scala | 57 +++++++++++++++++++ 11 files changed, 189 insertions(+), 61 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 9d1e39e5..abb773f6 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -20,6 +20,8 @@ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode /** Index management API. * @@ -29,7 +31,7 @@ import app.softnetwork.elastic.sql.serialization._ * - Parameter validation * - Automatic retry for transient errors */ -trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi => +trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi with VersionApi => // ======================================================================== // PUBLIC METHODS @@ -151,7 +153,40 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi = logger.info(s"Creating index '$index' with settings: $settings") - executeCreateIndex(index, settings, mappings, aliases) match { + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + val updatedMappings = + if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion)) { + mappings match { + case Some(m) => + val node: JsonNode = m + if (node.has("properties")) { + logger.info(s"Wrapping mappings with '_doc' type for ES version $elasticVersion") + val doc = mapper.createObjectNode() + val properties = node.get("properties") + doc.set("properties", properties) + val root: ObjectNode = node.asInstanceOf[ObjectNode] + root.remove("properties") + root.set[ObjectNode]("_doc", doc) + Some(root.toString) + } else { + Some(m) + } + case None => None + } + } else { + mappings + } + + executeCreateIndex(index, settings, updatedMappings, aliases) match { case success @ ElasticSuccess(true) => logger.info(s"✅ Index '$index' created successfully") success diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index b168d758..2e8c3013 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -20,6 +20,7 @@ import app.softnetwork.elastic.client.IndicesApi import app.softnetwork.elastic.client.jest.actions.GetIndex import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} +import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult import io.searchbox.indices.{CloseIndex, CreateIndex, DeleteIndex, IndicesExists, OpenIndex} import io.searchbox.indices.reindex.Reindex @@ -31,7 +32,7 @@ import scala.util.Try * [[IndicesApi]] for generic API documentation */ trait JestIndicesApi extends IndicesApi with JestClientHelpers { - _: JestRefreshApi with JestPipelineApi with JestClientCompanion => + _: JestRefreshApi with JestPipelineApi with JestVersionApi with JestClientCompanion => /** Create an index with the given settings. * @see @@ -51,13 +52,11 @@ trait JestIndicesApi extends IndicesApi with JestClientHelpers { val builder = new CreateIndex.Builder(index) .settings(settings) if (aliases.nonEmpty) { - val root = mapper.createObjectNode() val as = mapper.createObjectNode() aliases.foreach { alias => - as.set(alias.alias, alias.node) + as.set[ObjectNode](alias.alias, alias.node) } - root.set("aliases", as) - builder.aliases(root.toString) + builder.aliases(as.toString) } mappings.foreach { mapping => builder.mappings(mapping) diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala index fbbf9b0e..8808ca47 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala @@ -1,3 +1,5 @@ package app.softnetwork.elastic.client -class JestClientSpec extends ElasticClientSpec +class JestClientSpec extends ElasticClientSpec { + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 53c7faee..03e8fc2e 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -145,6 +145,7 @@ trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientH trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { _: RestHighLevelClientRefreshApi with RestHighLevelClientPipelineApi + with RestHighLevelClientVersionApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, @@ -156,8 +157,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH operation = "createIndex", index = Some(index), retryable = false - )( - request = new CreateIndexRequest(index) + )(request = { + val req = new CreateIndexRequest(index) .settings(settings, XContentType.JSON) .aliases( aliases @@ -173,8 +174,12 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH }) .asJava ) - .mapping(mappings.getOrElse("{}"), XContentType.JSON) - )( + mappings match { + case Some(m) if m.trim.startsWith("{") && m.trim.endsWith("}") => + req.mapping(m, XContentType.JSON) + case _ => req + } + })( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) } diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 4caaad31..31354fd0 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -154,6 +154,7 @@ trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientH trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { _: RestHighLevelClientRefreshApi with RestHighLevelClientPipelineApi + with RestHighLevelClientVersionApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( @@ -167,23 +168,30 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH index = Some(index), retryable = false )( - request = new CreateIndexRequest(index) - .settings(settings, XContentType.JSON) - .aliases( - aliases - .map(alias => { - var a = new Alias(alias.alias).writeIndex(alias.isWriteIndex).isHidden(alias.isHidden) - if (alias.filter.nonEmpty) { - val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson - a = a.filter(filterNode.toString) - } - alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) - alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) - a - }) - .asJava - ) - .mapping(mappings.getOrElse("{}"), XContentType.JSON) + request = { + val req = new CreateIndexRequest(index) + .settings(settings, XContentType.JSON) + .aliases( + aliases + .map(alias => { + var a = + new Alias(alias.alias).writeIndex(alias.isWriteIndex).isHidden(alias.isHidden) + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + a = a.filter(filterNode.toString) + } + alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) + alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) + a + }) + .asJava + ) + mappings match { + case Some(m) if m.trim.startsWith("{") && m.trim.endsWith("}") => + req.mapping(m, XContentType.JSON) + case _ => req + } + } )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 2a20c9e9..c188a0cf 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -32,6 +32,7 @@ import app.softnetwork.elastic.client.result.{ ElasticSuccess } import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -64,6 +65,7 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -117,7 +119,10 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * [[IndicesApi]] for index management operations */ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { - _: JavaClientRefreshApi with JavaClientPipelineApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientPipelineApi + with JavaClientVersionApi + with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, @@ -131,8 +136,8 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { )( apply() .indices() - .create( - new CreateIndexRequest.Builder() + .create { + val req = new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) .aliases( @@ -148,15 +153,20 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { .toMap .asJava ) - .mappings( - new TypeMapping.Builder() - .withJson( - new StringReader(mappings.getOrElse("{}")) + mappings match { + case None => req.build() + case Some(m) => + req + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(m) + ) + .build() ) .build() - ) - .build() - ) + } + } )(_.acknowledged()) override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { @@ -1079,14 +1089,15 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { action match { case BulkAction.UPDATE => + val doc: JsonNode = document new BulkOperation.Builder() .update( new UpdateOperation.Builder() .index(bulkItem.index) .id(id.orNull) .action( - new UpdateAction.Builder[JMap[String, Object], JMap[String, Object]]() - .doc(mapper.readValue(document, classOf[JMap[String, Object]])) + new UpdateAction.Builder[JsonNode, JsonNode]() + .doc(doc) .docAsUpsert(true) .build() ) @@ -1103,12 +1114,13 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { .build() case _ => + val doc: JsonNode = document new BulkOperation.Builder() .index( - new IndexOperation.Builder[JMap[String, Object]]() + new IndexOperation.Builder[JsonNode]() .index(bulkItem.index) .id(id.orNull) - .document(mapper.readValue(document, classOf[JMap[String, Object]])) + .document(doc) .build() ) .build() diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index d2393b9c..ef0dc96c 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -32,6 +32,7 @@ import app.softnetwork.elastic.client.result.{ ElasticSuccess } import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ FieldSort, @@ -59,6 +60,7 @@ import co.elastic.clients.elasticsearch.ingest.{ GetPipelineRequest, PutPipelineRequest } +import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -126,8 +128,8 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { )( apply() .indices() - .create( - new CreateIndexRequest.Builder() + .create { + val req = new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) .aliases( @@ -143,15 +145,20 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { .toMap .asJava ) - .mappings( - new TypeMapping.Builder() - .withJson( - new StringReader(mappings.getOrElse("{}")) + mappings match { + case None => req.build() + case Some(m) => + req + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(m) + ) + .build() ) .build() - ) - .build() - ) + } + } )(_.acknowledged()) override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { @@ -1072,14 +1079,15 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { action match { case BulkAction.UPDATE => + val doc: JsonNode = document new BulkOperation.Builder() .update( new UpdateOperation.Builder() .index(bulkItem.index) .id(id.orNull) .action( - new UpdateAction.Builder[JMap[String, Object], JMap[String, Object]]() - .doc(mapper.readValue(document, classOf[JMap[String, Object]])) + new UpdateAction.Builder[JsonNode, JsonNode]() + .doc(doc) .docAsUpsert(true) .build() ) @@ -1096,12 +1104,13 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { .build() case _ => + val doc: JsonNode = document new BulkOperation.Builder() .index( - new IndexOperation.Builder[JMap[String, Object]]() + new IndexOperation.Builder[JsonNode]() .index(bulkItem.index) .id(id.orNull) - .document(mapper.readValue(document, classOf[JMap[String, Object]])) + .document(doc) .build() ) .build() diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 6d5bcf56..33eab9e4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -341,7 +341,7 @@ package object schema { defaultProcessors ++ finalProcessors } - lazy val ddlTable: Table = { + lazy val asTable: Table = { // 1. Columns from the mapping val initialCols: Map[String, Column] = esMappings.fields.map { field => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 77cf4066..50cd6310 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -862,7 +862,8 @@ package object schema { def node: ObjectNode = { val node = mapper.createObjectNode() if (filter.nonEmpty) { - node.set("filter", Value(filter).asInstanceOf[ObjectValue].toJson) + val filterNode = mapper.valueToTree[JsonNode](filter.asJava) + node.set("filter", filterNode) } routing.foreach(r => node.put("routing", r)) indexRouting.foreach(r => node.put("index_routing", r)) @@ -898,7 +899,7 @@ package object schema { } object TableAlias { - def apply(table: String, name: String, value: Value[_]): TableAlias = { + def apply(table: String, alias: String, value: Value[_]): TableAlias = { val obj = value.asInstanceOf[ObjectValue] val filter = obj.value.get("filter") match { case Some(ObjectValue(f)) => @@ -927,7 +928,7 @@ package object schema { } TableAlias( table = table, - alias = name, + alias = alias, filter = filter, routing = routing, indexRouting = indexRouting, diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 7d0af37b..bde0b55a 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1013,7 +1013,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { settings = settings, defaultPipeline = Some(pipeline) ) - val ddlTable = esIndex.ddlTable + val ddlTable = esIndex.asTable println(s"""esIndex ddl -> ${ddlTable.sql}""") println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 958219a2..6214a823 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -25,7 +25,9 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.model.{Binary, Child, Parent, Sample} import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.schema.{Index, IndexAlias} import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.persistence._ import app.softnetwork.persistence.person.model.Person import com.fasterxml.jackson.core.JsonParseException @@ -114,6 +116,61 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "create_delete" should not(beCreated()) } + "Creating an index with mappings and aliases" should "work fine" in { + val mappings = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + val aliases: Seq[TableAlias] = Seq( + TableAlias(table = "create_mappings_aliases", alias = "create_mappings_aliases_alias1"), + TableAlias( + table = "create_mappings_aliases", + alias = "create_mappings_aliases_alias2", + filter = Map("term" -> Map("name.raw" -> "Homer Simpson")) + ) + ) + + pClient + .createIndex("create_mappings_aliases", mappings = Some(mappings), aliases = aliases) + .get shouldBe true + "create_mappings_aliases" should beCreated() + + val index: Option[Index] = pClient.getIndex("create_mappings_aliases").get + + index match { + case Some(idx) => + val indexAliases = idx.aliases.keys + indexAliases should contain("create_mappings_aliases_alias1") + indexAliases should contain("create_mappings_aliases_alias2") + case None => fail("Index not found") + } + + pClient.deleteIndex("create_mappings_aliases").get shouldBe true + "create_mappings_aliases" should not(beCreated()) + } + "Adding an alias and then removing it" should "work" in { pClient.addAlias("person", "person_alias") From be72b722750d248b88d3ffbf57ffa144ee43e524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Dec 2025 09:27:47 +0100 Subject: [PATCH 31/95] retrieve table default and final pipelines --- .../elastic/client/PipelineApi.scala | 12 +- .../elastic/sql/query/package.scala | 16 ++- .../elastic/sql/schema/package.scala | 122 +++++++++++------- .../elastic/sql/parser/ParserSpec.scala | 18 ++- 4 files changed, 104 insertions(+), 64 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index eff1bc1e..a467fdda 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -57,10 +57,10 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => } if (ElasticsearchVersion.isEs6(elasticVersion)) { val pipeline = ddl.ddlPipeline.copy( - ddlProcessors = ddl.ddlPipeline.ddlProcessors.map { processor => + processors = ddl.ddlPipeline.processors.map { processor => GenericProcessor( - processor.processorType, - processor.properties.filterNot(_._1 == "description") + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "description") ) } ) @@ -86,10 +86,10 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => } if (ElasticsearchVersion.isEs6(elasticVersion)) { val pipeline = updatingPipeline.copy( - ddlProcessors = updatingPipeline.ddlProcessors.map { processor => + processors = updatingPipeline.processors.map { processor => GenericProcessor( - processor.processorType, - processor.properties.filterNot(_._1 == "description") + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "description") ) } ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 2b115c06..b23656fd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -460,7 +460,7 @@ package object query { aliases = aliases ).update() - lazy val ddlPipeline: IngestPipeline = ddlTable.ddlPipeline + lazy val defaultPipeline: IngestPipeline = ddlTable.defaultPipeline } @@ -497,11 +497,15 @@ package object query { val ifExistsClause = if (ifExists) " IF EXISTS" else "" s"DROP COLUMN$ifExistsClause $columnName" } - override def ddlProcessor: Option[IngestProcessor] = Some(RemoveProcessor(sql, columnName)) + override def ddlProcessor: Option[IngestProcessor] = Some( + RemoveProcessor(sql = sql, column = columnName) + ) } case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { override def sql: String = s"RENAME COLUMN $oldName TO $newName" - override def ddlProcessor: Option[IngestProcessor] = Some(RenameProcessor(oldName, newName)) + override def ddlProcessor: Option[IngestProcessor] = Some( + RenameProcessor(column = oldName, newName = newName) + ) } case class AlterColumnOptions( columnName: String, @@ -572,9 +576,9 @@ package object query { override def ddlProcessor: Option[IngestProcessor] = Some( DefaultValueProcessor( - sql, - columnName, - defaultValue + sql = sql, + column = columnName, + value = defaultValue ) ) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 50cd6310..bf091a57 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -67,6 +67,7 @@ package object schema { node } def json: String = mapper.writeValueAsString(node) + def pipelineType: IngestPipelineType def processorType: IngestProcessorType def description: String = sql.trim def name: String = processorType.name @@ -226,6 +227,7 @@ package object schema { case other => GenericProcessor( + pipelineType = pipelineType, processorType = IngestProcessorType(other), properties = mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap @@ -235,8 +237,11 @@ package object schema { } } - case class GenericProcessor(processorType: IngestProcessorType, properties: Map[String, Any]) - extends IngestProcessor { + case class GenericProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + processorType: IngestProcessorType, + properties: Map[String, Any] + ) extends IngestProcessor { override def sql: String = ddl override def column: String = properties.get("field") match { case Some(s: String) => s @@ -249,6 +254,7 @@ package object schema { } case class ScriptProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, script: String, column: String, dataType: SQLType, @@ -271,6 +277,7 @@ package object schema { } case class RenameProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, column: String, newName: String, ignoreFailure: Boolean = true, @@ -291,6 +298,7 @@ package object schema { } case class RemoveProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, sql: String, column: String, ignoreFailure: Boolean = true, @@ -308,6 +316,7 @@ package object schema { } case class PrimaryKeyProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, sql: String, column: String, value: Set[String], @@ -328,6 +337,7 @@ package object schema { } case class DefaultValueProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, sql: String, column: String, value: Value[_], @@ -357,6 +367,7 @@ package object schema { } case class DateIndexNameProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, sql: String, column: String, dateRounding: String, @@ -413,16 +424,16 @@ package object schema { case class IngestPipeline( name: String, - ddlPipelineType: IngestPipelineType, - ddlProcessors: Seq[IngestProcessor] + pipelineType: IngestPipelineType, + processors: Seq[IngestProcessor] ) extends DdlToken { def sql: String = - s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.sql.trim).mkString(", ")})" + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${processors.map(_.sql.trim).mkString(", ")})" def node: ObjectNode = { val node = mapper.createObjectNode() val processorsNode = mapper.createArrayNode() - ddlProcessors.foreach { processor => + processors.foreach { processor => processorsNode.add(processor.node) } node.put("description", sql) @@ -431,18 +442,18 @@ package object schema { } override def ddl: String = - s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${ddlProcessors.map(_.ddl.trim).mkString(", ")})" + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${processors.map(_.ddl.trim).mkString(", ")})" def json: String = mapper.writeValueAsString(node) def diff(pipeline: IngestPipeline): List[PipelineDiff] = { - val actual = this.ddlProcessors + val actual = this.processors - val desired = pipeline.ddlProcessors + val desired = pipeline.processors // 1. Index processors by logical key - def key(p: IngestProcessor) = (p.processorType, p.column) + def key(p: IngestProcessor) = s"${p.pipelineType.name}-${p.processorType.name}-${p.column}" val desiredMap = desired.map(p => key(p) -> p).toMap val actualMap = actual.map(p => key(p) -> p).toMap @@ -476,19 +487,19 @@ package object schema { statements.foldLeft(this) { (current, alter) => alter match { case AddPipelineProcessor(processor) => - current.copy(ddlProcessors = - current.ddlProcessors.filterNot(p => + current.copy(processors = + current.processors.filterNot(p => p.processorType == processor.processorType && p.column == processor.column ) :+ processor ) case DropPipelineProcessor(processorType, column) => - current.copy(ddlProcessors = - current.ddlProcessors.filterNot(p => + current.copy(processors = + current.processors.filterNot(p => p.processorType == processorType && p.column == column ) ) case AlterPipelineProcessor(processor) => - current.copy(ddlProcessors = current.ddlProcessors.map { p => + current.copy(processors = current.processors.map { p => if (p.processorType == processor.processorType && p.column == processor.column) { processor } else { @@ -518,8 +529,8 @@ package object schema { } IngestPipeline( name = name, - ddlPipelineType = ddlPipelineType, - ddlProcessors = processors + pipelineType = ddlPipelineType, + processors = processors ) } } @@ -617,14 +628,14 @@ package object schema { s"$tabs$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" } - def ddlProcessors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ + def processors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => DefaultValueProcessor( sql = s"$path DEFAULT $dv", column = path, value = dv ) - }.toSeq ++ multiFields.flatMap(_.ddlProcessors) + }.toSeq ++ multiFields.flatMap(_.processors) def node: ObjectNode = { val root = mapper.createObjectNode() @@ -835,12 +846,12 @@ package object schema { case _ => List.empty } - def ddlProcessor(table: Table): DateIndexNameProcessor = + def processor(table: Table): DateIndexNameProcessor = DateIndexNameProcessor( - sql, - column, - dateRounding, - dateFormats, + sql = sql, + column = column, + dateRounding = dateRounding, + dateFormats = dateFormats, prefix = s"${table.name}-" ) } @@ -1029,9 +1040,9 @@ package object schema { s"CREATE OR REPLACE TABLE $name (\n\t$cols$pkStr)${partitionBy.getOrElse("")}$opts" } - def ddlProcessors: Seq[IngestProcessor] = - columns.flatMap(_.ddlProcessors) ++ partitionBy - .map(_.ddlProcessor(this)) + def tableProcessors: Seq[IngestProcessor] = + columns.flatMap(_.processors) ++ partitionBy + .map(_.processor(this)) .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) def merge(statements: Seq[AlterTableStatement]): Table = { @@ -1299,35 +1310,52 @@ package object schema { if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) } - lazy val ddlPipeline: IngestPipeline = { - val processorsFromColumns = ddlProcessors.map(p => p.column -> p).toMap + private[schema] lazy val diffPipeline: IngestPipeline = { + val processorsFromColumns = tableProcessors.map(p => p.column -> p).toMap + IngestPipeline( + name = s"${name}_ddl_diff_pipeline", + pipelineType = IngestPipelineType.Custom, + processors = + tableProcessors ++ processors.filterNot(p => processorsFromColumns.contains(p.column)) + ) + } + + lazy val defaultPipeline: IngestPipeline = { + val processorsFromColumns = tableProcessors.map(p => p.column -> p).toMap IngestPipeline( - name = s"${name}_ddl_default_pipeline", - ddlPipelineType = IngestPipelineType.Default, - ddlProcessors = - ddlProcessors ++ processors.filterNot(p => processorsFromColumns.contains(p.column)) + name = defaultPipelineName.getOrElse(s"${name}_ddl_default_pipeline"), + pipelineType = IngestPipelineType.Default, + processors = tableProcessors ++ processors + .filter(p => p.pipelineType == IngestPipelineType.Default) + .filterNot(p => processorsFromColumns.contains(p.column)) ) } - lazy val defaultPipeline: String = + lazy val finalPipeline: IngestPipeline = { + IngestPipeline( + name = finalPipelineName.getOrElse(s"${name}_ddl_final_pipeline"), + pipelineType = IngestPipelineType.Final, + processors = processors.filter(p => p.pipelineType == IngestPipelineType.Final) + ) + } + + lazy val defaultPipelineName: Option[String] = settings .get("default_pipeline") .map(_.value) .flatMap { - case v: String => Some(v) - case _ => None + case v: String if v != "_none" => Some(v) + case _ => None } - .getOrElse("_none") - lazy val finalPipeline: String = + lazy val finalPipelineName: Option[String] = settings .get("final_pipeline") .map(_.value) .flatMap { - case v: String => Some(v) - case _ => None + case v: String if v != "_none" => Some(v) + case _ => None } - .getOrElse("_none") lazy val indexMappings: ObjectNode = { val node = mapper.createObjectNode() @@ -1352,9 +1380,9 @@ package object schema { TableAlias(name, aliasName, value) }.toSeq - lazy val pipeline: ObjectNode = { - ddlPipeline.node - } + lazy val defaultPipelineNode: ObjectNode = defaultPipeline.node + + lazy val finalPipelineNode: ObjectNode = finalPipeline.node def diff(desired: Table): TableDiff = { val actual = this.update() @@ -1402,10 +1430,10 @@ package object schema { case Removed(name) => AliasRemoved(name) } - // 7. Pipeline + // 7. Default Pipeline val pipelineDiffs = scala.collection.mutable.ListBuffer[PipelineDiff]() - val actualPipeline = actual.ddlPipeline - val desiredPipeline = desiredUpdated.ddlPipeline + val actualPipeline = actual.diffPipeline + val desiredPipeline = desiredUpdated.diffPipeline actualPipeline.diff(desiredPipeline).foreach { d => pipelineDiffs += d } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index bde0b55a..dd7365a8 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -8,6 +8,7 @@ import app.softnetwork.elastic.sql.schema.{ mapper, DateIndexNameProcessor, DefaultValueProcessor, + IngestPipelineType, IngestProcessorType, PartitionDate, PrimaryKeyProcessor, @@ -989,8 +990,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) val sql = ct.ddlTable.sql println(sql) - println(ct.ddlTable.ddlPipeline.ddl) - val json = ct.ddlTable.ddlPipeline.json + println(ct.ddlTable.defaultPipeline.ddl) + val json = ct.ddlTable.defaultPipeline.json println(json) json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = ct.ddlTable.indexMappings @@ -999,7 +1000,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { val indexSettings = ct.ddlTable.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" - val pipeline = ct.ddlTable.pipeline + val pipeline = ct.ddlTable.defaultPipelineNode println(pipeline) pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex @@ -1017,8 +1018,8 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(s"""esIndex ddl -> ${ddlTable.sql}""") println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") - println(s"""esIndex pipeline -> ${ddlTable.pipeline.toString}""") - println(s"""esIndex ddl pipeline -> ${ddlTable.ddlPipeline.ddl}""") + println(s"""esIndex pipeline -> ${ddlTable.defaultPipelineNode.toString}""") + println(s"""esIndex ddl pipeline -> ${ddlTable.defaultPipeline.ddl}""") val ddlTableDiff = ddlTable.diff(ct.ddlTable) ddlTableDiff.columns.isEmpty shouldBe true ddlTableDiff.mappings.isEmpty shouldBe true @@ -1341,6 +1342,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "name") match { case Some( DefaultValueProcessor( + IngestPipelineType.Default, "DEFAULT 'anonymous'", "name", StringValue("anonymous"), @@ -1352,6 +1354,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "age") match { case Some( ScriptProcessor( + IngestPipelineType.Default, "DATE_DIFF(birthdate, CURRENT_DATE, YEAR)", "age", SQLTypes.Int, @@ -1367,6 +1370,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "ingested_at") match { case Some( DefaultValueProcessor( + IngestPipelineType.Default, "DEFAULT _ingest.timestamp", "ingested_at", IngestTimestampValue, @@ -1378,6 +1382,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "profile.seniority") match { case Some( ScriptProcessor( + IngestPipelineType.Default, "DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)", "profile.seniority", SQLTypes.Int, @@ -1393,6 +1398,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "birthdate") match { case Some( DateIndexNameProcessor( + IngestPipelineType.Default, "PARTITION BY birthdate (MONTH)", "birthdate", "M", @@ -1407,6 +1413,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.find(_.column == "_id") match { case Some( PrimaryKeyProcessor( + IngestPipelineType.Default, "PRIMARY KEY (id)", "_id", cols, @@ -1447,6 +1454,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { statements.size shouldBe 2 statements.collect { case AddPipelineProcessor(p) => p } match { case DefaultValueProcessor( + IngestPipelineType.Default, "status DEFAULT 'active'", "status", StringValue("active"), From 304f969cc82204e7e4a8996eb2513f391013d21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Dec 2025 09:39:26 +0100 Subject: [PATCH 32/95] set table default and final pipelines names --- .../app/softnetwork/elastic/sql/schema/package.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index bf091a57..eae61b37 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -1339,6 +1339,12 @@ package object schema { ) } + def setDefaultPipelineName(pipelineName: String): Table = { + this.copy( + settings = this.settings + ("default_pipeline" -> StringValue(pipelineName)) + ) + } + lazy val defaultPipelineName: Option[String] = settings .get("default_pipeline") @@ -1348,6 +1354,12 @@ package object schema { case _ => None } + def setFinalPipelineName(pipelineName: String): Table = { + this.copy( + settings = this.settings + ("final_pipeline" -> StringValue(pipelineName)) + ) + } + lazy val finalPipelineName: Option[String] = settings .get("final_pipeline") From 2c9655aa784e03d1f345934fabd3bed94c2fee24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Dec 2025 10:51:39 +0100 Subject: [PATCH 33/95] add pipeline api with Pipeline statement --- .../client/ElasticClientDelegator.scala | 16 ++ .../elastic/client/PipelineApi.scala | 163 ++++++++++-------- .../client/metrics/MetricsElasticClient.scala | 21 ++- 3 files changed, 128 insertions(+), 72 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 9642d312..9457fecc 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -23,6 +23,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} import app.softnetwork.elastic.sql.schema.TableAlias import com.typesafe.config.Config @@ -1424,6 +1425,21 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { // ==================== PipelineApi (delegate) ==================== + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def pipeline(sql: String): ElasticResult[Boolean] = + delegate.pipeline(sql) + + override private[client] def pipeline( + statement: query.PipelineStatement + ): ElasticResult[Boolean] = + delegate.pipeline(statement) + override def createPipeline( pipelineName: String, pipelineDefinition: String diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index a467fdda..bad60055 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -23,7 +23,12 @@ import app.softnetwork.elastic.client.result.{ ElasticSuccess } import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{AlterPipeline, CreatePipeline, DropPipeline} +import app.softnetwork.elastic.sql.query.{ + AlterPipeline, + CreatePipeline, + DropPipeline, + PipelineStatement +} import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline} trait PipelineApi extends ElasticClientHelpers { _: VersionApi => @@ -46,76 +51,8 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => case Right(statement) => statement match { - case ddl: CreatePipeline => - val elasticVersion = { - this.version match { - case ElasticSuccess(v) => v - case ElasticFailure(error) => - logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") - return ElasticResult.failure(error) - } - } - if (ElasticsearchVersion.isEs6(elasticVersion)) { - val pipeline = ddl.ddlPipeline.copy( - processors = ddl.ddlPipeline.processors.map { processor => - GenericProcessor( - processorType = processor.processorType, - properties = processor.properties.filterNot(_._1 == "description") - ) - } - ) - createPipeline(ddl.name, pipeline.json) - } else - createPipeline(ddl.name, ddl.ddlPipeline.json) - case ddl: DropPipeline => - deletePipeline(ddl.name, ifExists = ddl.ifExists) - case ddl: AlterPipeline => - getPipeline(ddl.name) match { - case ElasticSuccess(Some(existing)) => - val existingPipeline = IngestPipeline(name = ddl.name, json = existing) - val updatingPipeline = existingPipeline.merge(ddl.statements) - val elasticVersion = { - this.version match { - case ElasticSuccess(v) => v - case ElasticFailure(error) => - logger.error( - s"❌ Failed to retrieve Elasticsearch version: ${error.message}" - ) - return ElasticResult.failure(error) - } - } - if (ElasticsearchVersion.isEs6(elasticVersion)) { - val pipeline = updatingPipeline.copy( - processors = updatingPipeline.processors.map { processor => - GenericProcessor( - processorType = processor.processorType, - properties = processor.properties.filterNot(_._1 == "description") - ) - } - ) - updatePipeline(ddl.name, pipeline.json) - } else - updatePipeline(ddl.name, updatingPipeline.json) - case ElasticSuccess(None) if !ddl.ifExists => - val error = - ElasticError( - message = s"Pipeline with name '${ddl.name}' not found", - statusCode = Some(404), - operation = Some("pipeline") - ) - logger.error(s"❌ ${error.message}") - ElasticResult.failure(error) - case ElasticSuccess(None) if ddl.ifExists => - logger.info( - s"ℹ️ Pipeline with name '${ddl.name}' not found, skipping update as 'ifExists' is true" - ) - ElasticSuccess(false) - case failure @ ElasticFailure(error) => - logger.error( - s"❌ Failed to retrieve pipeline with name '${ddl.name}': ${error.message}" - ) - failure - } + case ddl: PipelineStatement => + pipeline(ddl) case _ => val error = ElasticError( @@ -141,6 +78,90 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => } } + private[client] def pipeline(statement: PipelineStatement): ElasticResult[Boolean] = { + statement match { + case ddl: CreatePipeline => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = ddl.ddlPipeline.copy( + processors = ddl.ddlPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "description") + ) + } + ) + createPipeline(ddl.name, pipeline.json) + } else + createPipeline(ddl.name, ddl.ddlPipeline.json) + case ddl: DropPipeline => + deletePipeline(ddl.name, ifExists = ddl.ifExists) + case ddl: AlterPipeline => + getPipeline(ddl.name) match { + case ElasticSuccess(Some(existing)) => + val existingPipeline = IngestPipeline(name = ddl.name, json = existing) + val updatingPipeline = existingPipeline.merge(ddl.statements) + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve Elasticsearch version: ${error.message}" + ) + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = updatingPipeline.copy( + processors = updatingPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "description") + ) + } + ) + updatePipeline(ddl.name, pipeline.json) + } else + updatePipeline(ddl.name, updatingPipeline.json) + case ElasticSuccess(None) if !ddl.ifExists => + val error = + ElasticError( + message = s"Pipeline with name '${ddl.name}' not found", + statusCode = Some(404), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + case ElasticSuccess(None) if ddl.ifExists => + logger.info( + s"ℹ️ Pipeline with name '${ddl.name}' not found, skipping update as 'ifExists' is true" + ) + ElasticSuccess(false) + case failure @ ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve pipeline with name '${ddl.name}': ${error.message}" + ) + failure + } + case _ => + val error = + ElasticError( + message = s"Unsupported pipeline DDL statement: $statement", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + } + /** Create a new ingest pipeline * * @param pipelineName diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index dc660cbb..2474c9f5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -31,8 +31,8 @@ import app.softnetwork.elastic.client.{ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.schema import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement} import app.softnetwork.elastic.sql.schema.TableAlias import org.json4s.Formats @@ -1096,6 +1096,25 @@ class MetricsElasticClient( // ==================== PipelineApi (delegate) ==================== + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def pipeline(sql: String): ElasticResult[Boolean] = + measureResult("pipeline") { + delegate.pipeline(sql) + } + + override private[client] def pipeline( + statement: query.PipelineStatement + ): ElasticResult[Boolean] = + measureResult("pipeline") { + delegate.pipeline(statement) + } + override def createPipeline( pipelineName: String, pipelineDefinition: String From 28363538a7f28383a49c183f5cacf38c703f56d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Thu, 25 Dec 2025 11:00:55 +0100 Subject: [PATCH 34/95] add table statement trait --- .../app/softnetwork/elastic/sql/query/package.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index b23656fd..c3942c38 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -381,6 +381,8 @@ package object query { } } + sealed trait TableStatement extends DdlStatement + case class CreateTable( table: String, ddl: Either[DqlStatement, List[Column]], @@ -389,7 +391,7 @@ package object query { primaryKey: List[String] = Nil, partitionBy: Option[PartitionDate] = None, options: Map[String, Value[_]] = Map.empty - ) extends DdlStatement { + ) extends TableStatement { lazy val partitioned: Boolean = partitionBy.isDefined @@ -465,7 +467,7 @@ package object query { } case class AlterTable(table: String, ifExists: Boolean, statements: List[AlterTableStatement]) - extends DdlStatement { + extends TableStatement { override def sql: String = { val ifExistsClause = if (ifExists) " IF EXISTS " else "" val parenthesesNeeded = statements.size > 1 @@ -679,7 +681,7 @@ package object query { } case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) - extends DdlStatement { + extends TableStatement { override def sql: String = { val ifExistsClause = if (ifExists) "IF EXISTS " else "" val cascadeClause = if (cascade) " CASCADE" else "" @@ -687,7 +689,7 @@ package object query { } } - case class TruncateTable(table: String) extends DdlStatement { + case class TruncateTable(table: String) extends TableStatement { override def sql: String = s"TRUNCATE TABLE $table" } } From caa88d1ac82b7bdcb67d75992f1eac1c153042c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 27 Dec 2025 12:39:06 +0100 Subject: [PATCH 35/95] implements isIndexClosed, truncateIndex and deleteByQuery - fix retrieving index information that will be converted to Table objects --- .../elastic/sql/bridge/package.scala | 6 +- .../client/ElasticClientDelegator.scala | 37 +++ .../elastic/client/ElasticsearchVersion.scala | 6 + .../elastic/client/IndicesApi.scala | 312 ++++++++++++++++++ .../elastic/client/NopeClientApi.scala | 9 + .../client/metrics/MetricsElasticClient.scala | 34 +- .../elastic/client/AliasApiSpec.scala | 41 +++ .../elastic/client/IndicesApiSpec.scala | 146 ++++++++ .../elastic/client/MappingApiSpec.scala | 20 ++ .../elastic/client/SettingsApiSpec.scala | 146 ++++++++ documentation/client/indices.md | 185 +++++++++++ .../elastic/sql/bridge/package.scala | 6 +- .../client/jest/JestClientHelpers.scala | 10 +- .../elastic/client/jest/JestIndicesApi.scala | 51 +++ .../client/rest/RestHighLevelClientApi.scala | 62 ++++ .../client/rest/RestHighLevelClientApi.scala | 62 ++++ .../elastic/client/java/JavaClientApi.scala | 41 +++ .../elastic/client/java/JavaClientApi.scala | 46 +++ .../softnetwork/elastic/schema/package.scala | 17 +- .../elastic/sql/query/package.scala | 4 +- .../elastic/client/ElasticClientSpec.scala | 275 ++++++++++++++- 21 files changed, 1498 insertions(+), 18 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 36cacfc2..b6e767dc 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -478,7 +478,11 @@ package object bridge { case _ => nestedWithoutCriteriaQuery.getOrElse(matchAllQuery()) } - } sourceFiltering (fields, excludes) + } + + if (!request.deleteByQuery && !request.updateByQuery) { + _search = _search sourceFiltering (fields, excludes) + } _search = if (allAggregations.nonEmpty) { _search aggregations { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 9457fecc..76660097 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -161,6 +161,33 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = delegate.indexExists(index, pattern) + /** Truncate an index by deleting all its documents. + * + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + override def truncateIndex(index: String): ElasticResult[Long] = + delegate.truncateIndex(index) + + /** Delete documents by query from an index. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + override def deleteByQuery(index: String, query: String, refresh: Boolean): ElasticResult[Long] = + delegate.deleteByQuery(index, query, refresh) + + override def isIndexClosed(index: String): ElasticResult[Boolean] = + delegate.isIndexClosed(index) + override private[client] def executeCreateIndex( index: String, settings: String, @@ -192,6 +219,16 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = delegate.executeIndexExists(index) + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = + delegate.executeDeleteByQuery(index, query, refresh) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + delegate.executeIsIndexClosed(index) + // ==================== AliasApi ==================== /** Add an alias to an index. diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 2fabb9f4..8bc35f59 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -136,4 +136,10 @@ object ElasticsearchVersion { def requiresDocTypeWrapper(version: String): Boolean = { !isAtLeast(version, 6, 8) } + + /** Check if deletion by query on closed indices is supported (ES >= 7.5) + */ + def supportsDeletionByQueryOnClosedIndices(version: String): Boolean = { + isAtLeast(version, 7, 5) + } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index abb773f6..2dfa2a09 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -18,6 +18,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{Delete, From, SingleSearch} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode @@ -566,6 +568,308 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w } } + def isIndexClosed(index: String): ElasticResult[Boolean] = { + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("isIndexClosed"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${err.message}" + ) + ) + // 2. Check index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("isIndexClosed") + ) + ) + case ElasticFailure(err) => Left(err) + } + // 3. Retrieve index status + closed <- executeIsIndexClosed(index).toEither + + } yield closed + + result match { + case Right(closed) => + val statusStr = if (closed) "closed" else "open" + logger.info(s"✅ Index '$index' is $statusStr") + ElasticSuccess(closed) + case Left(err) => + logger.error(s"❌ Failed to check if index '$index' is closed: ${err.message}") + return ElasticFailure(err) + } + } + + /** Truncate an index by deleting all its documents. + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + def truncateIndex(index: String): ElasticResult[Long] = + deleteByQuery(index, """{"query": {"match_all": {}}}""") + + /** Delete documents by query from an index. + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + def deleteByQuery( + index: String, + query: String, + refresh: Boolean = true + ): ElasticResult[Long] = { + + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${err.message}" + ) + ) + + // 2. Parse query (SQL or JSON) + jsonQuery <- parseQueryForDeletion(index, query) + + // 3. Check index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("deleteByQuery") + ) + ) + case ElasticFailure(err) => Left(err) + } + + // 4. Open index if needed + tuple <- openIfNeeded(index) + (_, restore) = tuple + + // 5. Execute delete-by-query + deleted <- executeDeleteByQuery(index, jsonQuery, refresh).toEither + + // 6. Restore state + _ <- restore().toEither.left.map { restoreErr => + logger.warn(s"❌ Failed to restore index state for '$index': ${restoreErr.message}") + restoreErr + } + + } yield deleted + + result match { + case Right(count) => + logger.info(s"✅ Deleted $count documents from index '$index'") + ElasticSuccess(count) + case Left(err) => + logger.error(s"❌ Failed to delete by query on index '$index': ${err.message}") + ElasticFailure(err) + } + } + + /** Parse a query for deletion, determining if it's SQL or JSON. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query (SQL or JSON) + * @return + * the validated JSON query + */ + private def parseQueryForDeletion(index: String, query: String): Either[ElasticError, String] = { + val trimmed = query.trim.toUpperCase + val isSql = trimmed.startsWith("SELECT") || + trimmed.startsWith("DELETE") || + trimmed.startsWith("WITH") + if (isSql) parseSqlQueryForDeletion(index, query) + else parseJsonQueryForDeletion(index, query) + } + + /** Validate a JSON query for deletion. + * + * @param index + * - the name of the index to delete from + * @param query + * - the JSON query + * @return + * the validated JSON query + */ + private def parseJsonQueryForDeletion( + index: String, + query: String + ): Either[ElasticError, String] = { + validateJson("deleteByQuery", query) match { + case None => + logger.info(s"Processing JSON query for deleteByQuery on index '$index': $query") + Right(query) + + case Some(err) => + Left( + err.copy( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid JSON query: ${err.message}" + ) + ) + } + } + + /** Parse an SQL query for deletion and convert it to Elasticsearch JSON. + * + * @param index + * - the name of the index to delete from + * @param query + * - the SQL query + * @return + * the Elasticsearch JSON query + */ + private def parseSqlQueryForDeletion( + index: String, + query: String + ): Either[ElasticError, String] = { + logger.info(s"Processing SQL query for deleteByQuery on index '$index': $query") + + Parser(query) match { + case Left(err) => + Left( + ElasticError( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL query: ${err.msg}" + ) + ) + + case Right(statement) => + statement match { + + case deleteStmt: Delete => + if (deleteStmt.table.name != index) + Left( + sqlErrorForDeletion( + index = index, + message = + s"SQL query index '${deleteStmt.table.name}' does not match provided index '$index'" + ) + ) + else + deleteStmt.where match { + case None => + logger.info( + s"SQL delete query has no WHERE clause, deleting all documents from index '$index'" + ) + Right("""{"query": {"match_all": {}}}""") + + case Some(where) => + implicit val timestamp: Long = System.currentTimeMillis() + val search: String = + SingleSearch( + from = From(tables = Seq(deleteStmt.table)), + where = Some(where), + deleteByQuery = true + ) + logger.info(s"✅ Converted SQL delete query to search for deleteByQuery: $search") + Right(search) + } + + case search: SingleSearch => + val tables = search.from.tables + if (tables.size != 1 || tables.head.name != index) + Left( + sqlErrorForDeletion( + index = index, + message = + s"SQL query index '${tables.map(_.name).mkString(",")}' does not match provided index '$index'" + ) + ) + else { + implicit val timestamp: Long = System.currentTimeMillis() + val query: String = search.copy(deleteByQuery = true) + logger.info(s"✅ Converted SQL search query to search for deleteByQuery: $query") + Right(query) + } + + case _ => + Left( + sqlErrorForDeletion( + index = index, + message = s"Invalid SQL query for deleteByQuery" + ) + ) + } + } + } + + private def openIfNeeded( + index: String + ): Either[ElasticError, (Boolean, () => ElasticResult[Boolean])] = { + for { + // Detect initial state + isClosed <- isIndexClosed(index).toEither + + // Open only if needed + _ <- if (isClosed) openIndex(index).toEither else Right(()) + + } yield { + val restore = () => + if (isClosed) closeIndex(index) + else ElasticSuccess(true) + + (isClosed, restore) + } + } + + private def sqlErrorForDeletion(index: String, message: String): ElasticError = + ElasticError( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = message + ) + + // ================================================================================ + // IMPLICIT CONVERSIONS + // ================================================================================ + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== @@ -593,4 +897,12 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w ): ElasticResult[(Boolean, Option[Long])] private[client] def executeIndexExists(index: String): ElasticResult[Boolean] + + private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] + + private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index b68bf812..c8ef851c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -311,4 +311,13 @@ trait NopeClientApi extends ElasticClientApi { templateName: String ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ElasticSuccess(0L) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ElasticSuccess(false) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 2474c9f5..e204094b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -25,7 +25,6 @@ import app.softnetwork.elastic.client.{ ElasticQueries, ElasticQuery, ElasticResponse, - JSONQuery, SingleValueAggregateResult } import app.softnetwork.elastic.client.bulk._ @@ -149,6 +148,39 @@ class MetricsElasticClient( } } + /** Truncate an index by deleting all its documents. + * + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + override def truncateIndex(index: String): ElasticResult[Long] = + measureResult("truncate", Some(index)) { + delegate.truncateIndex(index) + } + + /** Delete documents by query from an index. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + override def deleteByQuery(index: String, query: String, refresh: Boolean): ElasticResult[Long] = + measureResult("deleteByQuery", Some(index)) { + delegate.deleteByQuery(index, query, refresh) + } + + override def isIndexClosed(index: String): ElasticResult[Boolean] = + measureResult("isIndexClosed", Some(index)) { + delegate.isIndexClosed(index) + } + // ==================== AliasApi ==================== override def addAlias(index: String, alias: String): ElasticResult[Boolean] = { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index 6cbf6c19..b9c47554 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for AliasApi @@ -113,6 +114,25 @@ class AliasApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? } var aliasApi: TestAliasApi = _ @@ -1728,6 +1748,27 @@ class AliasApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index 7c3907b2..d86f1cd4 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for IndicesApi @@ -99,6 +100,25 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? } var indicesApi: TestIndicesApi = _ @@ -749,6 +769,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -828,6 +869,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1062,6 +1124,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1125,6 +1208,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1188,6 +1292,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1251,6 +1376,27 @@ class IndicesApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index dd4fc402..ec51e455 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -6,6 +6,7 @@ import org.scalatest.BeforeAndAfterEach import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for MappingApi @@ -126,6 +127,25 @@ class MappingApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? } var mappingApi: TestMappingApi = _ diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index 378dc045..6f67aac0 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias import com.google.gson.JsonParser @@ -94,6 +95,25 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? } var settingsApi: TestSettingsApi = _ @@ -806,6 +826,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -887,6 +928,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -965,6 +1027,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1559,6 +1642,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1629,6 +1733,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When @@ -1699,6 +1824,27 @@ class SettingsApiSpec ): ElasticResult[Option[String]] = ??? override private[client] def executeVersion(): ElasticResult[String] = ??? + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query + * serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = ??? + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ??? + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ??? } // When diff --git a/documentation/client/indices.md b/documentation/client/indices.md index df278727..0d549535 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -71,6 +71,22 @@ val defaultSettings: String = """ --- +## 🔧 Index State Detection and Automatic State Restoration + +SoftClient4ES ensures that operations requiring an **open index** (such as `deleteByQuery` and `truncateIndex`) behave safely and consistently, even when the target index is **closed**. + +Elasticsearch does **not** allow `_delete_by_query` on a closed index (including ES 8.x and ES 9.x). +To guarantee correct behavior, SoftClient4ES automatically: + +1. Detects whether the index is open or closed +2. Opens the index if needed +3. Executes the operation +4. Restores the original state (re‑closes the index if it was closed) + +This mechanism is fully transparent to the user. + +--- + ## Public Methods ### createIndex @@ -462,6 +478,175 @@ existenceChecks.foreach { --- +### isIndexClosed + +Checks whether an index is currently **closed**. + +**Signature:** + +```scala +def isIndexClosed(index: String): ElasticResult[Boolean] +``` + +**Parameters:** +- `index` – Name of the index to inspect + +**Returns:** +- `ElasticSuccess(true)` if the index is closed +- `ElasticSuccess(false)` if the index is open +- `ElasticFailure` if the index does not exist or the request fails + +**Behavior:** +- Uses the Elasticsearch `_cat/indices` API internally +- Supported across Jest (ES6), REST HL (ES6/7), Java API Client (ES8/ES9) + +**Examples:** + +```scala +client.isIndexClosed("archive-2023") match { + case ElasticSuccess(true) => println("Index is closed") + case ElasticSuccess(false) => println("Index is open") + case ElasticFailure(err) => println(s"Error: ${err.message}") +} +``` + +--- + +### Automatic Index Opening and State Restoration + +Some operations require the index to be **open**. +SoftClient4ES automatically handles this through an internal helper: + +- Detect initial state (`open` or `closed`) +- Open the index if needed +- Execute the operation +- Restore the original state + +This ensures: + +- **Safety** (no accidental state changes) +- **Idempotence** (index ends in the same state it started) +- **Compatibility** with all Elasticsearch versions + +This logic is used internally by: + +- `deleteByQuery` +- `truncateIndex` + +--- + +### deleteByQuery + +Deletes documents from an index using either a JSON query or a SQL `DELETE`/`SELECT` expression. + +If the index is **closed**, SoftClient4ES will: + +1. Detect that the index is closed +2. Open it temporarily +3. Execute the delete‑by‑query +4. Re‑close it afterward + +**Signature:** + +```scala +def deleteByQuery( + index: String, + query: String, + refresh: Boolean = true +): ElasticResult[Long] +``` + +**Parameters:** +- `index` – Name of the index +- `query` – JSON or SQL delete expression +- `refresh` – Whether to refresh the index after deletion + +**Returns:** +- `ElasticSuccess[Long]` – number of deleted documents +- `ElasticFailure` – error details + +**Behavior:** +- Validates index name +- Parses SQL into JSON when needed +- Ensures index exists +- Automatically opens closed indices +- Restores original state after execution +- Uses `_delete_by_query` internally + +**Examples:** + +```scala +// JSON delete +client.deleteByQuery( + "users", + """{"query": {"term": {"active": false}}}""" +) + +// SQL delete +client.deleteByQuery( + "orders", + "DELETE FROM orders WHERE status = 'cancelled'" +) + +// SQL select (equivalent to delete) +client.deleteByQuery( + "sessions", + "SELECT * FROM sessions WHERE expired = true" +) +``` + +--- + +### truncateIndex + +Deletes **all documents** from an index while preserving its mappings, settings, and aliases. + +This is implemented as: + +```scala +deleteByQuery(index, """{"query": {"match_all": {}}}""") +``` + +If the index is closed, SoftClient4ES will automatically: + +- Open it +- Execute the truncate +- Restore the closed state + +**Signature:** + +```scala +def truncateIndex(index: String): ElasticResult[Long] +``` + +**Parameters:** +- `index` – Name of the index to truncate + +**Returns:** +- `ElasticSuccess[Long]` – number of deleted documents +- `ElasticFailure` – error details + +**Examples:** + +```scala +// Remove all documents +client.truncateIndex("logs-2024") + +// Safe truncate with existence check +for { + exists <- client.indexExists("cache") + deleted <- if (exists) client.truncateIndex("cache") + else ElasticResult.success(0L) +} yield deleted +``` + +**Notes:** +- Index structure is preserved +- Operation is irreversible +- Works even if the index is initially closed + +--- + ## Implementation Requirements The following methods must be implemented by each client-specific trait: diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index ebed4abe..4d616455 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -472,7 +472,11 @@ package object bridge { case _ => nestedWithoutCriteriaQuery.getOrElse(matchAllQuery()) } - } sourceFiltering (fields, excludes) + } + + if (!request.deleteByQuery && !request.updateByQuery) { + _search = _search sourceFiltering (fields, excludes) + } _search = if (allAggregations.nonEmpty) { _search aggregations { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala index 3b85ae2f..f5a2c666 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala @@ -18,7 +18,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.ElasticClientHelpers import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} -import io.searchbox.action.Action +import io.searchbox.action.{AbstractAction, Action} import io.searchbox.client.JestResult import scala.concurrent.Promise @@ -71,7 +71,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( transformer: R => T ): ElasticResult[T] = { @@ -166,7 +166,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] ): ElasticResult[Boolean] = { executeJestAction[R, Boolean](operation, index, retryable)(action)(_.isSucceeded) } @@ -208,7 +208,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( extractor: com.google.gson.JsonObject => T ): ElasticResult[T] = { @@ -259,7 +259,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( parser: String => T ): ElasticResult[T] = { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 2e8c3013..581027c4 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult +import io.searchbox.core.{Cat, CatResult, DeleteByQuery} import io.searchbox.indices.{CloseIndex, CreateIndex, DeleteIndex, IndicesExists, OpenIndex} import io.searchbox.indices.reindex.Reindex @@ -164,4 +165,54 @@ trait JestIndicesApi extends IndicesApi with JestClientHelpers { new IndicesExists.Builder(index).build() } } + + private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJestAction[JestResult, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + ) { + val builder = new DeleteByQuery.Builder(jsonQuery) + .addIndex(index) + + builder.setParameter("conflicts", "proceed") + + if (refresh) + builder.setParameter("refresh", true) + + builder.build() + } { result => + val deleted = Try { + result.getJsonObject.get("deleted").getAsLong + }.getOrElse(0L) + + deleted + } + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJestAction[CatResult, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + ) { + new Cat.IndicesBuilder() + .addIndex(index) + .setParameter("format", "json") + .build() + } { result => + val json = result.getJsonObject + val arr = json.getAsJsonArray("result") + + if (arr == null || arr.size() == 0) + false + else { + val entry = arr.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } } diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 03e8fc2e..2126e386 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -297,6 +297,68 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH identity ) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeRestAction[Request, Response, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_delete_by_query?refresh=$refresh&conflicts=proceed" + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonObject + + // ES6/ES7 return "deleted" + val deleted = + if (json.has("deleted")) json.get("deleted").getAsLong + else 0L + + deleted + } + ) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeRestAction[Request, Response, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + request = { + val req = new Request("GET", s"/_cat/indices/$index?format=json") + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonArray + + if (json.size() == 0) + false + else { + val entry = json.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } + ) } /** Alias management API for RestHighLevelClient diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 31354fd0..3e424902 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -310,6 +310,68 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH identity ) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeRestAction[Request, Response, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_delete_by_query?refresh=$refresh&conflicts=proceed" + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonObject + + // ES6/ES7 return "deleted" + val deleted = + if (json.has("deleted")) json.get("deleted").getAsLong + else 0L + + deleted + } + ) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeRestAction[Request, Response, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + request = { + val req = new Request("GET", s"/_cat/indices/$index?format=json") + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonArray + + if (json.size() == 0) + false + else { + val entry = json.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } + ) } /** Alias management API for RestHighLevelClient diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index c188a0cf..4257d69d 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -35,6 +35,7 @@ import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ + Conflicts, FieldSort, FieldValue, Refresh, @@ -43,6 +44,7 @@ import co.elastic.clients.elasticsearch._types.{ SortOrder, Time } +import co.elastic.clients.elasticsearch.cat.IndicesRequest import co.elastic.clients.elasticsearch.core.bulk.{ BulkOperation, DeleteOperation, @@ -264,6 +266,45 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { ) )(_.value()) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJavaAction( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + apply().deleteByQuery( + new DeleteByQueryRequest.Builder() + .index(index) + .withJson(new StringReader(jsonQuery)) // JSON query as raw string + .refresh(refresh) + .conflicts(Conflicts.Proceed) + .build() + ) + )(_.deleted()) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJavaAction( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + apply() + .cat() + .indices( + new IndicesRequest.Builder() + .index(index) + .build() + ) + ) { response => + response.valueBody().asScala.headOption match { + case Some(info) => info.status() == "close" + case None => false + } + } } /** Elasticsearch client implementation of Alias API using the Java Client diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index ef0dc96c..f1f52474 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -35,6 +35,7 @@ import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ + Conflicts, FieldSort, FieldValue, Refresh, @@ -42,6 +43,7 @@ import co.elastic.clients.elasticsearch._types.{ SortOrder, Time } +import co.elastic.clients.elasticsearch.cat.IndicesRequest import co.elastic.clients.elasticsearch.core.bulk.{ BulkOperation, DeleteOperation, @@ -256,6 +258,50 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { ) )(_.value()) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJavaAction( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + apply().deleteByQuery( + new DeleteByQueryRequest.Builder() + .index(index) + .withJson(new StringReader(jsonQuery)) // JSON query as raw string + .refresh(refresh) + .conflicts(Conflicts.Proceed) + .build() + ) + )(_.deleted()) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJavaAction( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + apply() + .cat() + .indices( + new IndicesRequest.Builder() + .index(index) + .build() + ) + ) { response => + val entries = response.indices().asScala.headOption + if (entries.isEmpty) false + else { + val entry = entries.head + entry.status() match { + case "close" => true + case _ => false + } + } + } } /** Elasticsearch client implementation of Alias API using the Java Client diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 33eab9e4..273755f2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -145,9 +145,12 @@ package object schema { object IndexMappings { def apply(root: JsonNode): IndexMappings = { - val mappings = root.path("mappings") - val fields = Option(mappings.get("properties")) - .orElse(Option(mappings.path("_doc").get("properties"))) + if (root.has("mappings")) { + val mappingsNode = root.path("mappings") + return apply(mappingsNode) + } + val fields = Option(root.get("properties")) + .orElse(Option(root.path("_doc").get("properties"))) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue @@ -155,7 +158,7 @@ package object schema { }.toList) .getOrElse(Nil) - val options = extractObject(mappings, ignoredKeys = Set("properties", "_doc")) + val options = extractObject(root, ignoredKeys = Set("properties", "_doc")) val meta = options.get("_meta") val primaryKey: List[String] = meta .map { @@ -208,7 +211,11 @@ package object schema { object IndexSettings { def apply(settings: JsonNode): IndexSettings = { - val index = settings.path("settings").path("index") + if (settings.has("settings")) { + val settingsNode = settings.path("settings") + return apply(settingsNode) + } + val index = settings.path("index") val options = extractObject(index) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index c3942c38..d98b823a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -72,7 +72,9 @@ package object query { having: Option[Having] = None, orderBy: Option[OrderBy] = None, limit: Option[Limit] = None, - score: Option[Double] = None + score: Option[Double] = None, + deleteByQuery: Boolean = false, + updateByQuery: Boolean = false ) extends DqlStatement { override def sql: String = s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 6214a823..4060f33c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -27,7 +27,7 @@ import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit import app.softnetwork.elastic.schema.{Index, IndexAlias} import app.softnetwork.elastic.sql.query.SelectStatement -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.schema.{Table, TableAlias} import app.softnetwork.persistence._ import app.softnetwork.persistence.person.model.Person import com.fasterxml.jackson.core.JsonParseException @@ -140,6 +140,13 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M | } | } | } + | }, + | "_meta": { + | "primary_key": ["uuid"], + | "partition_by": { + | "column": "birthDate", + | "granularity": "M" + | } | } |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") @@ -161,9 +168,20 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M index match { case Some(idx) => - val indexAliases = idx.aliases.keys - indexAliases should contain("create_mappings_aliases_alias1") - indexAliases should contain("create_mappings_aliases_alias2") + val table: Table = idx.asTable + log.info(table.sql) + table.columns.size shouldBe 3 + table.columns.map(_.name) should contain allOf ("uuid", "name", "birthDate") + table.primaryKey should contain("uuid") + table.partitionBy match { + case Some(partitionBy) => + partitionBy.column shouldBe "birthDate" + partitionBy.granularity.sql shouldBe "MONTH" + case None => fail("Partition by not found") + } + val tableAliases = table.aliases.keys + tableAliases should contain("create_mappings_aliases_alias1") + tableAliases should contain("create_mappings_aliases_alias2") case None => fail("Index not found") } @@ -171,6 +189,10 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "create_mappings_aliases" should not(beCreated()) } + "Checking for an alias that does not exist" should "return false" in { + pClient.aliasExists("person_alias").get shouldBe false + } + "Adding an alias and then removing it" should "work" in { pClient.addAlias("person", "person_alias") @@ -1375,4 +1397,249 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M scrollResult.children.map(_.parentId) should contain only "A16" } } + + "Truncate index" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_truncate") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => refresh(index).getStatusLine.getStatusCode < 400) shouldBe true + + indices should contain only "person_to_truncate" + + blockUntilCount(3, "person_to_truncate") + + "person_to_truncate" should haveCount(3) + + val truncateResult = pClient.truncateIndex("person_to_truncate").get + truncateResult shouldBe 3L + + "person_to_truncate" should haveCount(0) + } + + "Delete by query on closed index" should "work depending on ES version" in { + pClient.createIndex("closed_index_test").get shouldBe true + + pClient.closeIndex("closed_index_test").get shouldBe true + + val deleted = pClient + .deleteByQuery( + "closed_index_test", + """{"query": {"match_all": {}}}""" + ) + .get + + deleted shouldBe 0L // index is empty anyway + } + + "Delete by query with invalid JSON" should "fail" in { + pClient.createIndex("delete_by_query_invalid_json").get shouldBe true + val result = pClient.deleteByQuery("idx", "{not valid json}") + result.isFailure shouldBe true + } + + "Delete by query with invalid SQL" should "fail" in { + pClient.createIndex("delete_by_query_invalid_sql").get shouldBe true + val result = pClient.deleteByQuery("delete_by_query_invalid_sql", "DELETE FROM") + result.isFailure shouldBe true + } + + "Delete by query with SQL referencing wrong index" should "fail" in { + pClient.createIndex("index_a").get shouldBe true + pClient.createIndex("index_b").get shouldBe true + val result = pClient.deleteByQuery( + "index_a", + "DELETE FROM index_b WHERE x = 1" + ) + result.isFailure shouldBe true + } + + "Delete by query with valid JSON" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_json_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => refresh(index).getStatusLine.getStatusCode < 400) shouldBe true + + indices should contain only "person_to_delete_by_json_query" + + blockUntilCount(3, "person_to_delete_by_json_query") + + "person_to_delete_by_json_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery("person_to_delete_by_json_query", """{"query": {"match_all": {}}}""") + .get + deleteByQuery shouldBe 3L + + "person_to_delete_by_json_query" should haveCount(0) + } + + "Delete by query with SQL DELETE" should "work" in { + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + log.info(s"mapping: $mapping") + pClient + .createIndex("person_to_delete_by_sql_delete_query", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_sql_delete_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => pClient.refresh(index).get) + + indices should contain only "person_to_delete_by_sql_delete_query" + + blockUntilCount(3, "person_to_delete_by_sql_delete_query") + + "person_to_delete_by_sql_delete_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery( + "person_to_delete_by_sql_delete_query", + """DELETE FROM person_to_delete_by_sql_delete_query WHERE uuid = 'A16'""" + ) + .get + deleteByQuery shouldBe 1L + + "person_to_delete_by_sql_delete_query" should haveCount(2) + + pClient + .deleteByQuery( + "person_to_delete_by_sql_delete_query", + "DELETE FROM person_to_delete_by_sql_delete_query" + ) + .get shouldBe 2L + + "person_to_delete_by_sql_delete_query" should haveCount(0) + } + + "Delete by query with SQL SELECT" should "work" in { + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + log.info(s"mapping: $mapping") + pClient + .createIndex("person_to_delete_by_sql_select_query", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_sql_select_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => pClient.refresh(index).get) + + indices should contain only "person_to_delete_by_sql_select_query" + + blockUntilCount(3, "person_to_delete_by_sql_select_query") + + "person_to_delete_by_sql_select_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery( + "person_to_delete_by_sql_select_query", + """SELECT * FROM person_to_delete_by_sql_select_query WHERE uuid = 'A16'""" + ) + .get + deleteByQuery shouldBe 1L + + "person_to_delete_by_sql_select_query" should haveCount(2) + + pClient + .deleteByQuery( + "person_to_delete_by_sql_select_query", + "SELECT * FROM person_to_delete_by_sql_select_query" + ) + .get shouldBe 2L + + "person_to_delete_by_sql_select_query" should haveCount(0) + } } From aaeae3b4c1a0f7ed104334a0d360a7f7293fc5a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 27 Dec 2025 14:22:38 +0100 Subject: [PATCH 36/95] fix alias exists api with es6, add waitForShards for es6 ensuring index fully operational after reopening it when needed --- .../softnetwork/elastic/client/AliasApi.scala | 4 + .../elastic/client/IndicesApi.scala | 13 +- documentation/client/indices.md | 123 ++++++++++++++++++ .../elastic/client/jest/JestIndicesApi.scala | 16 ++- .../client/jest/actions/WaitForShards.scala | 20 +++ .../client/rest/RestHighLevelClientApi.scala | 24 ++++ 6 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala index 7c596bb5..f58a533b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala @@ -249,6 +249,10 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => success case failure @ ElasticFailure(error) => + if (error.statusCode.getOrElse(0) == 404) { + logger.info(s"✅ Alias '$alias' does not exist") + return ElasticResult.success(false) + } logger.error(s"❌ Failed to check existence of alias '$alias': ${error.message}") failure } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 2dfa2a09..6cb73472 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -608,7 +608,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w ElasticSuccess(closed) case Left(err) => logger.error(s"❌ Failed to check if index '$index' is closed: ${err.message}") - return ElasticFailure(err) + ElasticFailure(err) } } @@ -837,6 +837,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w // Open only if needed _ <- if (isClosed) openIndex(index).toEither else Right(()) + _ <- if (isClosed) waitForShards(index).toEither else Right(()) } yield { val restore = () => @@ -905,4 +906,14 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w ): ElasticResult[Long] private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] + + private[client] def waitForShards( + index: String, + status: String = "yellow", + timeout: Int = 30 + ): ElasticResult[Unit] = { + // Default implementation does nothing + ElasticSuccess(()) + } + } diff --git a/documentation/client/indices.md b/documentation/client/indices.md index 0d549535..9d93b688 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -87,6 +87,129 @@ This mechanism is fully transparent to the user. --- +## 🔧 Internal Shard Readiness Handling (`waitForShards`) + +Some Elasticsearch operations require the index to be **fully operational**, meaning all primary shards must be allocated and the index must reach at least **yellow** cluster health. +This is especially important for Elasticsearch **6.x**, where reopening a closed index does **not** guarantee immediate shard availability. + +SoftClient4ES includes an internal mechanism called **`waitForShards`**, which ensures that the index is ready before executing operations that depend on shard availability. + +This mechanism is: + +- **fully automatic** +- **transparent to the user** +- **only applied when necessary** +- **client‑specific** (Jest ES6, REST HL ES6) +- a **no‑op** for Elasticsearch 7, 8, and 9 + +--- + +### When is `waitForShards` used? + +`waitForShards` is invoked automatically after reopening an index inside the internal `openIfNeeded` workflow. + +It is used by operations that require: + +- an **open** index +- **allocated** shards +- a **searchable** state + +Specifically: + +- `deleteByQuery` +- `truncateIndex` + +These operations internally perform: + +1. Detect whether the index is closed +2. Open it if needed +3. **Wait for shards to reach the required health status** +4. Execute the operation +5. Restore the original index state (re‑close if necessary) + +This ensures consistent behavior across all Elasticsearch versions. + +--- + +### Why is this needed? + +Elasticsearch 6.x has a known behavior: + +- After reopening a closed index, shards may remain in `INITIALIZING` state for a short period. +- Executing `_delete_by_query` during this window results in: + + ``` + 503 Service Unavailable + search_phase_execution_exception + all shards failed + ``` + +Elasticsearch 7+ no longer exhibits this issue. + +SoftClient4ES abstracts this difference by automatically waiting for shard readiness on ES6. + +--- + +### How does `waitForShards` work? + +Internally, the client performs: + +``` +GET /_cluster/health/?wait_for_status=yellow&timeout=30s +``` + +This ensures: + +- primary shards are allocated +- the index is searchable +- the cluster is ready to process delete‑by‑query operations + +### Client‑specific behavior: + +| Client | ES Version | Behavior | +|---------------------|-------------|-------------------------------------------------------------| +| **Jest** | 6.x | Uses a custom Jest action to call `_cluster/health` | +| **REST HL** | 6.x | Uses the low‑level client to call `_cluster/health` | +| **Java API Client** | 8.x / 9.x | No‑op (Elasticsearch handles shard readiness automatically) | +| **REST HL** | 7.x | No‑op | +| **Jest** | 7.x | No‑op | + +--- + +### Transparency for the user + +`waitForShards` is **not part of the public API**. +It is an internal mechanism that ensures: + +- consistent behavior across ES6, ES7, ES8, ES9 +- predictable delete‑by‑query semantics +- correct handling of closed indices +- no need for users to manually manage shard allocation or cluster health + +Users do **not** need to call or configure anything. + +--- + +### Example (internal workflow) + +When calling: + +```scala +client.deleteByQuery("my_index", """{"query": {"match_all": {}}}""") +``` + +SoftClient4ES internally performs: + +1. Check if `my_index` is closed +2. If closed → open it +3. **Wait for shards to reach yellow** (ES6 only) +4. Execute `_delete_by_query` +5. Restore original state (re‑close if needed) + +This guarantees reliable behavior even on older Elasticsearch clusters. + +--- + ## Public Methods ### createIndex diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 581027c4..689b0255 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.IndicesApi -import app.softnetwork.elastic.client.jest.actions.GetIndex +import app.softnetwork.elastic.client.jest.actions.{GetIndex, WaitForShards} import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} import com.fasterxml.jackson.databind.node.ObjectNode @@ -215,4 +215,18 @@ trait JestIndicesApi extends IndicesApi with JestClientHelpers { status == "close" } } + + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = { + executeJestBooleanAction( + operation = "waitForShards", + index = Some(index), + retryable = true + ) { + new WaitForShards.Builder(index = index, status = status, timeout = timeout).build() + }.map(_ => ()) + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala new file mode 100644 index 00000000..5b42f534 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala @@ -0,0 +1,20 @@ +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.action.{AbstractAction, GenericResultAbstractAction} +import io.searchbox.client.config.ElasticsearchVersion + +object WaitForShards { + class Builder(var index: String, var status: String = "yellow", var timeout: Int = 30) + extends AbstractAction.Builder[WaitForShards, WaitForShards.Builder] { + override def build = new WaitForShards(this) + } +} + +class WaitForShards protected (builder: WaitForShards.Builder) + extends GenericResultAbstractAction(builder) { + override def getRestMethodName = "GET" + + override def buildURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_cluster/health/${builder.index}?wait_for_status=${builder.status}&timeout=${builder.timeout}s" + +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 2126e386..84bcc67d 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -359,6 +359,30 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } } ) + + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = { + executeRestAction[Request, Response, Unit]( + operation = "waitForShards", + index = Some(index.toString), + retryable = true + )( + request = { + val req = new Request( + "GET", + s"/_cluster/health/${index}?wait_for_status=${status}&timeout=${timeout}s" + ) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = _ => () + ) + } } /** Alias management API for RestHighLevelClient From abb7f45a25d755ac451e51d09c127e706598d9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 27 Dec 2025 15:04:48 +0100 Subject: [PATCH 37/95] fix index creation with mappings for es6 --- .../elastic/client/IndicesApi.scala | 28 ++++++++++++++----- .../elastic/client/MappingComparator.scala | 4 +-- .../softnetwork/elastic/schema/package.scala | 7 +++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 6cb73472..d1a2522e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -169,19 +169,33 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion)) { mappings match { case Some(m) => - val node: JsonNode = m - if (node.has("properties")) { + val root = mapper.readTree(m).asInstanceOf[ObjectNode] + + if (root.has("properties") || root.has("_meta")) { logger.info(s"Wrapping mappings with '_doc' type for ES version $elasticVersion") + val doc = mapper.createObjectNode() - val properties = node.get("properties") - doc.set("properties", properties) - val root: ObjectNode = node.asInstanceOf[ObjectNode] - root.remove("properties") - root.set[ObjectNode]("_doc", doc) + + // Move properties + if (root.has("properties")) { + doc.set("properties", root.get("properties")) + root.remove("properties") + } + + // Move _meta + if (root.has("_meta")) { + doc.set("_meta", root.get("_meta")) + root.remove("_meta") + } + + // Wrap into _doc + root.set("_doc", doc) + Some(root.toString) } else { Some(m) } + case None => None } } else { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala index badb9962..d18fe4c2 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala @@ -26,8 +26,8 @@ object MappingComparator extends StrictLogging { private def parseJsonToMap(jsonString: String): Map[String, JsonElement] = { Try( - new JsonParser() - .parse(jsonString) + JsonParser + .parseString(jsonString) .getAsJsonObject .get("properties") .getAsJsonObject diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 273755f2..09015897 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -149,8 +149,11 @@ package object schema { val mappingsNode = root.path("mappings") return apply(mappingsNode) } + if (root.has("_doc")) { + val docNode = root.path("_doc") + return apply(docNode) + } val fields = Option(root.get("properties")) - .orElse(Option(root.path("_doc").get("properties"))) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue @@ -158,7 +161,7 @@ package object schema { }.toList) .getOrElse(Nil) - val options = extractObject(root, ignoredKeys = Set("properties", "_doc")) + val options = extractObject(root, ignoredKeys = Set("properties")) val meta = options.get("_meta") val primaryKey: List[String] = meta .map { From 1c8ebaa6af7c2f7330fd964510ab1b6b8ab1dd96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 28 Dec 2025 10:00:01 +0100 Subject: [PATCH 38/95] add support for updateByQuery --- .../client/ElasticClientDelegator.scala | 53 ++- .../elastic/client/IndicesApi.scala | 311 +++++++++++++++++- .../client/metrics/MetricsElasticClient.scala | 40 +++ .../elastic/client/AliasApiSpec.scala | 14 + .../elastic/client/IndicesApiSpec.scala | 49 +++ .../elastic/client/MappingApiSpec.scala | 7 + .../elastic/client/SettingsApiSpec.scala | 49 +++ .../elastic/client/jest/JestIndicesApi.scala | 30 +- .../client/rest/RestHighLevelClientApi.scala | 35 ++ .../client/rest/RestHighLevelClientApi.scala | 35 ++ .../elastic/client/java/JavaClientApi.scala | 25 ++ .../elastic/client/java/JavaClientApi.scala | 25 ++ .../elastic/sql/query/package.scala | 13 + .../elastic/sql/schema/package.scala | 10 + .../elastic/client/ElasticClientSpec.scala | 290 +++++++++++++++- 15 files changed, 973 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 76660097..398b592b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -31,7 +31,7 @@ import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} import scala.concurrent.{ExecutionContext, Future} -import scala.language.implicitConversions +import scala.language.{dynamics, implicitConversions} import scala.reflect.ClassTag trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { @@ -188,6 +188,27 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def isIndexClosed(index: String): ElasticResult[Boolean] = delegate.isIndexClosed(index) + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + override def updateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + delegate.updateByQuery(index, query, pipelineId, refresh) + override private[client] def executeCreateIndex( index: String, settings: String, @@ -229,6 +250,21 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = delegate.executeIsIndexClosed(index) + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = + delegate.waitForShards(index, status, timeout) + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + delegate.executeUpdateByQuery(index, query, pipelineId, refresh) + // ==================== AliasApi ==================== /** Add an alias to an index. @@ -1484,6 +1520,21 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.createPipeline(pipelineName, pipelineDefinition) } + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def updatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + delegate.updatePipeline(pipelineName, pipelineDefinition) + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { delegate.deletePipeline(pipelineName, ifExists = ifExists) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index d1a2522e..0ddf6db5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -19,10 +19,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{Delete, From, SingleSearch} -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.query.{Delete, From, SingleSearch, Table, Update} +import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, TableAlias} import app.softnetwork.elastic.sql.serialization._ -import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode /** Index management API. @@ -708,6 +707,206 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w } } + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + def updateByQuery( + index: String, + query: String, + pipelineId: Option[String] = None, + refresh: Boolean = true + ): ElasticResult[Long] = { + + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("updateByQuery"), + statusCode = Some(400), + index = Some(index) + ) + ) + + // 2. Parse SQL or JSON + parsed <- parseQueryForUpdate(index, query).toEither + + // 3. Extract SQL pipeline (optional) + sqlPipeline = parsed match { + case Left(u: Update) if u.values.nonEmpty => Some(u.customPipeline) + case _ => None + } + + // 4. Extract SQL WHERE → JSON query + jsonQuery = parsed match { + case Left(u: Update) => + u.where match { + case None => + logger.info( + s"SQL update query has no WHERE clause, updating all documents from index '$index'" + ) + """{"query": {"match_all": {}}}""" + + case Some(where) => + implicit val timestamp: Long = System.currentTimeMillis() + val search: String = + SingleSearch( + from = From(tables = Seq(Table(u.table))), + where = Some(where), + updateByQuery = true + ) + logger.info(s"✅ Converted SQL update query to search for updateByQuery: $search") + search + } + + case _ => query // JSON passthrough + } + + // 5. Load user pipeline if provided + userPipeline <- pipelineId match { + case Some(id) => + getPipeline(id).toEither.flatMap { + case Some(json) => Right(Some(IngestPipeline(id, json))) + case None => + Left( + ElasticError( + message = s"Pipeline '$id' not found", + index = Some(index), + operation = Some("updateByQuery"), + statusCode = Some(404) + ) + ) + } + case None => Right(None) + } + + // 6. Resolve final pipeline (merge if needed) + elasticVersion <- this.version.toEither + + resolved <- resolveFinalPipeline(userPipeline, sqlPipeline, elasticVersion).toEither + (finalPipelineId, mustDelete) = resolved + + // 7. Ensure index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("updateByQuery") + ) + ) + case ElasticFailure(err) => Left(err) + } + + // 8. Open index if needed + tuple <- openIfNeeded(index) + (_, restore) = tuple + + // 9. Execute update-by-query + updated <- executeUpdateByQuery(index, jsonQuery, finalPipelineId, refresh).toEither + + // 10. Cleanup temporary pipeline + _ <- + if (mustDelete && finalPipelineId.isDefined) + deletePipeline(finalPipelineId.get, ifExists = true).toEither + else Right(()) + + // 11. Restore index state + _ <- restore().toEither + + } yield updated + + result match { + case Right(count) => ElasticSuccess(count) + case Left(err) => ElasticFailure(err) + } + } + + private def parseQueryForUpdate( + index: String, + query: String + ): ElasticResult[Either[Update, String]] = { + val trimmed = query.trim.toUpperCase + val isSql = trimmed.startsWith("UPDATE") + if (isSql) { + logger.info(s"Processing SQL query for updateByQuery on index '$index': $query") + + Parser(query) match { + case Left(err) => + ElasticFailure( + ElasticError( + operation = Some("updateByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL query: ${err.msg}" + ) + ) + + case Right(statement) => + statement match { + + case updateStmt: Update => + if (updateStmt.table != index) + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = + s"SQL query index '${updateStmt.table}' does not match provided index '$index'" + ) + ) + else + ElasticSuccess(Left(updateStmt)) + + case search: SingleSearch => + val tables = search.from.tables + if (tables.size != 1 || tables.head.name != index) + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = + s"SQL query index '${tables.map(_.name).mkString(",")}' does not match provided index '$index'" + ) + ) + else { + implicit val timestamp: Long = System.currentTimeMillis() + val query: String = search.copy(deleteByQuery = false) + logger.info(s"✅ Converted SQL search query to JSON for updateByQuery: $query") + ElasticSuccess(Right(query)) + } + + case _ => + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = s"Invalid SQL query for updateByQuery" + ) + ) + } + } + } else { + logger.info(s"Processing JSON query for updateByQuery on index '$index': $query") + ElasticSuccess(Right(query)) + } + } + /** Parse a query for deletion, determining if it's SQL or JSON. * * @param index @@ -788,7 +987,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w case deleteStmt: Delete => if (deleteStmt.table.name != index) Left( - sqlErrorForDeletion( + sqlErrorFor( + operation = "deleteByQuery", index = index, message = s"SQL query index '${deleteStmt.table.name}' does not match provided index '$index'" @@ -818,7 +1018,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w val tables = search.from.tables if (tables.size != 1 || tables.head.name != index) Left( - sqlErrorForDeletion( + sqlErrorFor( + operation = "deleteByQuery", index = index, message = s"SQL query index '${tables.map(_.name).mkString(",")}' does not match provided index '$index'" @@ -833,7 +1034,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w case _ => Left( - sqlErrorForDeletion( + sqlErrorFor( + operation = "deleteByQuery", index = index, message = s"Invalid SQL query for deleteByQuery" ) @@ -862,9 +1064,96 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w } } - private def sqlErrorForDeletion(index: String, message: String): ElasticError = + private def resolveFinalPipeline( + user: Option[IngestPipeline], + sql: Option[IngestPipeline], + elasticVersion: String + ): ElasticResult[(Option[String], Boolean)] = { + + (user, sql) match { + + // No pipeline + case (None, None) => + ElasticSuccess((None, false)) + + // User pipeline only + case (Some(u), None) => + ElasticSuccess((Some(u.name), false)) + + // Only SQL pipeline → temporary pipeline + case (None, Some(sqlPipe)) => + val tmpId = s"_tmp_update_${System.nanoTime()}" + val json = + if (ElasticsearchVersion.isEs6(elasticVersion)) { + sqlPipe + .copy( + processors = sqlPipe.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = + processor.properties.filterNot(_._1 == "description").filterNot(_._1 == "if") + ) + } + ) + .json + } else { + sqlPipe + .copy( + processors = sqlPipe.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "if") + ) + } + ) + .json + } + logger.info(s"Creating temporary pipeline for updateByQuery: $json") + createPipeline(tmpId, json) match { + case ElasticSuccess(_) => ElasticSuccess((Some(tmpId), true)) + case ElasticFailure(e) => ElasticFailure(e) + } + + // Merge user + SQL pipeline → temporary pipeline + case (Some(u), Some(sqlPipe)) => + val merged = u.merge(sqlPipe) + val tmpId = s"_tmp_update_${System.nanoTime()}" + val json = + if (ElasticsearchVersion.isEs6(elasticVersion)) { + merged + .copy( + processors = merged.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = + processor.properties.filterNot(_._1 == "description").filterNot(_._1 == "if") + ) + } + ) + .json + } else { + merged + .copy( + processors = merged.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "if") + ) + } + ) + .json + } + logger.info(s"Creating merged temporary pipeline for updateByQuery: $json") + createPipeline(tmpId, json) match { + case ElasticSuccess(_) => ElasticSuccess((Some(tmpId), true)) + case ElasticFailure(e) => ElasticFailure(e) + } + } + } + + private def sqlErrorFor(operation: String, index: String, message: String): ElasticError = ElasticError( - operation = Some("deleteByQuery"), + operation = Some(operation), statusCode = Some(400), index = Some(index), message = message @@ -930,4 +1219,10 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w ElasticSuccess(()) } + private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index e204094b..b7603a82 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -181,6 +181,29 @@ class MetricsElasticClient( delegate.isIndexClosed(index) } + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + override def updateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + measureResult("updateByQuery", Some(index)) { + delegate.updateByQuery(index, query, pipelineId, refresh) + } + // ==================== AliasApi ==================== override def addAlias(index: String, alias: String): ElasticResult[Boolean] = { @@ -1155,6 +1178,23 @@ class MetricsElasticClient( delegate.createPipeline(pipelineName, pipelineDefinition) } + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def updatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + measureResult("updatePipeline") { + delegate.updatePipeline(pipelineName, pipelineDefinition) + } + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = measureResult("deletePipeline") { delegate.deletePipeline(pipelineName, ifExists = ifExists) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index b9c47554..b01aceed 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -133,6 +133,13 @@ class AliasApiSpec ): ElasticResult[Long] = ??? override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } var aliasApi: TestAliasApi = _ @@ -1769,6 +1776,13 @@ class AliasApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index d86f1cd4..a0261657 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -119,6 +119,13 @@ class IndicesApiSpec ): ElasticResult[Long] = ??? override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } var indicesApi: TestIndicesApi = _ @@ -790,6 +797,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -890,6 +904,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1145,6 +1166,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1229,6 +1257,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1313,6 +1348,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1397,6 +1439,13 @@ class IndicesApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index ec51e455..82ffe04b 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -146,6 +146,13 @@ class MappingApiSpec ): ElasticResult[Long] = ??? override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } var mappingApi: TestMappingApi = _ diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index 6f67aac0..cca9b114 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -114,6 +114,13 @@ class SettingsApiSpec ): ElasticResult[Long] = ??? override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } var settingsApi: TestSettingsApi = _ @@ -847,6 +854,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -949,6 +963,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1048,6 +1069,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1663,6 +1691,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1754,6 +1789,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When @@ -1845,6 +1887,13 @@ class SettingsApiSpec override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = ??? } // When diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 689b0255..c4095a65 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -22,7 +22,7 @@ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult -import io.searchbox.core.{Cat, CatResult, DeleteByQuery} +import io.searchbox.core.{Cat, CatResult, DeleteByQuery, UpdateByQuery, UpdateByQueryResult} import io.searchbox.indices.{CloseIndex, CreateIndex, DeleteIndex, IndicesExists, OpenIndex} import io.searchbox.indices.reindex.Reindex @@ -229,4 +229,32 @@ trait JestIndicesApi extends IndicesApi with JestClientHelpers { new WaitForShards.Builder(index = index, status = status, timeout = timeout).build() }.map(_ => ()) } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + executeJestAction[UpdateByQueryResult, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + ) { + val builder = new UpdateByQuery.Builder(jsonQuery) + .addIndex(index) + + if (refresh) + builder.setParameter("refresh", true) + + pipelineId.foreach { id => + builder.setParameter("pipeline", id) + } + + builder.build() + } { result => + val json = result.getJsonObject + json.get("updated").getAsLong + } + } } diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 84bcc67d..3831e3ec 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -383,6 +383,41 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH transformer = _ => () ) } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeRestAction[Request, Response, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_update_by_query?refresh=$refresh" + + pipelineId.map(id => s"&pipeline=$id").getOrElse("") + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString( + scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + ) + .getAsJsonObject + json.get("updated").getAsLong + } + ) + } } /** Alias management API for RestHighLevelClient diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 3e424902..3154ef0d 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -372,6 +372,41 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } } ) + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeRestAction[Request, Response, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_update_by_query?refresh=$refresh" + + pipelineId.map(id => s"&pipeline=$id").getOrElse("") + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString( + scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + ) + .getAsJsonObject + json.get("updated").getAsLong + } + ) + } } /** Alias management API for RestHighLevelClient diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 4257d69d..25e09b20 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -305,6 +305,31 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { case None => false } } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeJavaAction( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + apply().updateByQuery( + new UpdateByQueryRequest.Builder() + .index(index) + .refresh(refresh) + .pipeline(pipelineId.orNull) + .withJson(new StringReader(jsonQuery)) + .build() + ) + ) { response => + response.updated() + } + } } /** Elasticsearch client implementation of Alias API using the Java Client diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index f1f52474..51b70f7e 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -302,6 +302,31 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { } } } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeJavaAction( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + apply().updateByQuery( + new UpdateByQueryRequest.Builder() + .index(index) + .refresh(refresh) + .pipeline(pipelineId.orNull) + .withJson(new StringReader(jsonQuery)) + .build() + ) + ) { response => + response.updated() + } + } } /** Elasticsearch client implementation of Alias API using the Java Client diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index d98b823a..2cf32d11 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -306,6 +306,19 @@ package object query { override def sql: String = s"UPDATE $table SET ${values .map { case (k, v) => s"$k = ${v.value}" } .mkString(", ")}${where.map(w => s" ${w.sql}").getOrElse("")}" + + lazy val customPipeline: IngestPipeline = IngestPipeline( + s"update-$table-${Instant.now}", + IngestPipelineType.Custom, + values.map { case (k, v) => + DefaultValueProcessor( + sql = s"SET DEFAULT $k = ${v.value}", + column = k, + value = v + ) + }.toSeq + ) + } case class Delete(table: Table, where: Option[Where]) extends DmlStatement { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index eae61b37..43a71e4e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -509,6 +509,16 @@ package object schema { } } } + + def merge(other: IngestPipeline): IngestPipeline = { + other.processors.foldLeft(this) { (current, processor) => + current.copy(processors = + current.processors.filterNot(p => + p.processorType == processor.processorType && p.column == processor.column + ) :+ processor + ) + } + } } object IngestPipeline { diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 4060f33c..1c6d7c8f 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -25,7 +25,7 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.model.{Binary, Child, Parent, Sample} import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -import app.softnetwork.elastic.schema.{Index, IndexAlias} +import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query.SelectStatement import app.softnetwork.elastic.sql.schema.{Table, TableAlias} import app.softnetwork.persistence._ @@ -1539,7 +1539,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices - indices.forall(index => pClient.refresh(index).get) + indices.forall(index => pClient.refresh(index).get) shouldBe true indices should contain only "person_to_delete_by_sql_delete_query" @@ -1615,7 +1615,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices - indices.forall(index => pClient.refresh(index).get) + indices.forall(index => pClient.refresh(index).get) shouldBe true indices should contain only "person_to_delete_by_sql_select_query" @@ -1642,4 +1642,288 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "person_to_delete_by_sql_select_query" should haveCount(0) } + + "Update by query with missing index" should "fail with 404" in { + val result = pClient.updateByQuery( + "person_update_with_missing_index", + """{"query": {"match_all": {}}}""" + ) + + result.isFailure shouldBe true + result.toEither.left.get.statusCode shouldBe Some(404) + } + + "Update by query with missing pipeline" should "fail with 404" in { + pClient.createIndex("person_update_with_missing_pipeline").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_missing_pipeline", + """{"query": {"match_all": {}}}""", + pipelineId = Some("does-not-exist") + ) + + result.isFailure shouldBe true + result.toEither.left.get.statusCode shouldBe Some(404) + } + + "Update by query with invalid SQL" should "fail" in { + pClient.createIndex("person_update_with_invalid_sql").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_invalid_sql", + """UPDATE person_update_sql_where WHERE uuid = 'A16'""" + ) + + result.isFailure shouldBe true + } + + "Update by query with invalid JSON" should "fail" in { + pClient.createIndex("person_update_with_invalid_json").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_invalid_json", + """{ invalid json }""" + ) + + result.isFailure shouldBe true + } + + "Update by query with SQL UPDATE and WHERE" should "update only matching documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_sql_where", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_sql_where") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + + blockUntilCount(3, "person_update_sql_where") + "person_update_sql_where" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_sql_where", + """UPDATE person_update_sql_where SET name = 'Another Name' WHERE uuid = 'A16'""" + ) + .get + + updated shouldBe 1L + + pClient.refresh("person_update_sql_where").get shouldBe true + + pClient.getAs[Person]("A16", Some("person_update_sql_where")).get match { + case Some(doc) => doc.name shouldBe "Another Name" + case None => fail("Document A16 not found") + } + } + + "Update by query with SQL UPDATE without WHERE" should "update all documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_all_sql", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_all_sql") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some("uuid") + ) + .get + + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + + blockUntilCount(3, "person_update_all_sql") + "person_update_all_sql" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_all_sql", + """UPDATE person_update_all_sql SET birthDate = '1972-12-26'""" + ) + .get + + updated shouldBe 3L + + pClient.refresh("person_update_all_sql").get shouldBe true + + val updatedPersons = pClient.searchDocuments("SELECT * FROM person_update_all_sql") + updatedPersons.size shouldBe 3 + updatedPersons.forall(person => person.birthDate == "1972-12-26") shouldBe true + } + + "Update by query with JSON query and user pipeline" should "apply the pipeline to all documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_json", mappings = Some(mapping)) + .get shouldBe true + + // User Pipeline + val userPipelineJson = + """{ + | "processors": [ + | { "set": { "field": "birthDate", "value": "1972-12-26" } } + | ] + |} + """.stripMargin + + pClient.createPipeline("set-birthdate-1972-12-26", userPipelineJson).get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_json") + pClient + .bulk[String](persons, identity, idKey = Some("uuid")) + .get + + blockUntilCount(3, "person_update_json") + "person_update_json" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_json", + """{"query": {"match_all": {}}}""", + pipelineId = Some("set-birthdate-1972-12-26") + ) + .get + + updated shouldBe 3L + + pClient.refresh("person_update_json").get shouldBe true + + val updatedPersons = pClient.searchDocuments("SELECT * FROM person_update_json") + updatedPersons.size shouldBe 3 + updatedPersons.forall(person => person.birthDate == "1972-12-26") shouldBe true + } + + "Update by query with SQL UPDATE and user pipeline" should "merge user and SQL pipelines" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_merge_pipeline", mappings = Some(mapping)) + .get shouldBe true + + val userPipelineJson = + """{ + | "processors": [ + | { "set": { "field": "name", "value": "UPDATED_NAME" } } + | ] + |} + """.stripMargin + + pClient.createPipeline("user-update-name", userPipelineJson).get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_merge_pipeline") + pClient + .bulk[String](persons, identity, idKey = Some("uuid")) + .get + + blockUntilCount(3, "person_update_merge_pipeline") + "person_update_merge_pipeline" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_merge_pipeline", + """UPDATE person_update_merge_pipeline SET birthDate = '1972-12-26' WHERE uuid = 'A16'""", + pipelineId = Some("user-update-name") + ) + .get + + updated shouldBe 1L + + pClient.refresh("person_update_merge_pipeline").get shouldBe true + + pClient.getAs[Person]("A16", Some("person_update_merge_pipeline")).get match { + case Some(doc) => + doc.name shouldBe "UPDATED_NAME" + doc.birthDate shouldBe "1972-12-26" + case None => fail("Document A16 not found") + } + } + + "Update by query on a closed index" should "open, update, and restore closed state" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_closed_index", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_closed_index") + pClient + .bulk[String](persons, identity, idKey = Some("uuid")) + .get + + blockUntilCount(3, "person_update_closed_index") + "person_update_closed_index" should haveCount(3) + + pClient.closeIndex("person_update_closed_index").get shouldBe true + pClient.isIndexClosed("person_update_closed_index").get shouldBe true + + val updated = pClient + .updateByQuery( + "person_update_closed_index", + """UPDATE person_update_closed_index SET birthDate = '1972-12-26' WHERE uuid = 'A16'""" + ) + .get + + updated shouldBe 1L + + pClient.isIndexClosed("person_update_closed_index").get shouldBe true + } } From 4a95927b0ab713a03486f77b45ded5b8d4140910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 28 Dec 2025 10:07:26 +0100 Subject: [PATCH 39/95] add documentation for updateByQuery --- documentation/client/indices.md | 198 ++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/documentation/client/indices.md b/documentation/client/indices.md index 9d93b688..7f34e98e 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -210,6 +210,204 @@ This guarantees reliable behavior even on older Elasticsearch clusters. --- +### 🔧 updateByQuery + +`updateByQuery` updates documents in an index using either a **JSON query** or a **SQL UPDATE statement**. +It supports ingest pipelines, SQL‑driven SET clauses, and automatic pipeline merging. + +SoftClient4ES ensures consistent behavior across Elasticsearch 6, 7, 8, and 9, including: + +- automatic index opening and state restoration +- shard readiness handling (ES6 only) +- temporary pipeline creation and cleanup +- SQL → JSON query translation +- SQL → ingest pipeline generation + +--- + +#### SQL UPDATE Support + +SoftClient4ES accepts SQL UPDATE statements of the form: + +```sql +UPDATE SET field = value [, field2 = value2 ...] [WHERE ] +``` + +#### ✔️ Supported: + +- simple literal values (`string`, `number`, `boolean`, `date`) +- multiple assignments in the SET clause +- WHERE clause with any supported SQL predicate +- UPDATE without WHERE (updates all documents) +- automatic conversion to: + - a JSON query (`WHERE` → `query`) + - an ingest pipeline (`SET` → processors) + +#### ✖️ Not supported: + +- painless scripts +- complex expressions in SET +- joins or multi‑table updates + +--- + +#### Automatic Pipeline Generation (SQL SET → Ingest Pipeline) + +When using SQL UPDATE, the `SET` clause is automatically converted into an ingest pipeline: + +```json +{ + "processors": [ + { "set": { "field": "name", "value": "Homer" } }, + { "set": { "field": "childrenCount", "value": 3 } } + ] +} +``` + +This pipeline is created **only for the duration of the update**, unless the user explicitly provides a pipeline ID. + +--- + +#### Pipeline Resolution and Merging + +`updateByQuery` supports three pipeline sources: + +| Source | Description | +|-------------------|--------------------------------| +| **User pipeline** | Provided via `pipelineId` | +| **SQL pipeline** | Generated from SQL SET clause | +| **No pipeline** | JSON update without processors | + +#### Pipeline resolution rules: + +| User pipeline | SQL pipeline | Result | +|----------------|---------------|--------------------------------------| +| None | None | No pipeline | +| Some | None | Use user pipeline | +| None | Some | Create temporary pipeline | +| Some | Some | Merge both into a temporary pipeline | + +#### Pipeline merging + +Processors are merged deterministically: + +- processors with the same `(type, field)` → SQL processor overrides user processor +- order is preserved +- merged pipeline is temporary and deleted after execution + +--- + +#### JSON Query Support + +If the query is not SQL, it is treated as a raw JSON `_update_by_query` request: + +```json +{ + "query": { + "term": { "uuid": "A16" } + } +} +``` + +No pipeline is generated unless the user provides one. + +--- + +#### Index State Handling + +`updateByQuery` uses the same robust index‑state workflow as `deleteByQuery`: + +1. Detect whether the index is open or closed +2. Open it if needed +3. **ES6 only:** wait for shards to reach `yellow` +4. Execute update‑by‑query +5. Restore the original state (re‑close if needed) + +This ensures safe, predictable behavior across all Elasticsearch versions. + +--- + +#### Signature + +```scala +def updateByQuery( + index: String, + query: String, + pipelineId: Option[String] = None, + refresh: Boolean = true +): ElasticResult[Long] +``` + +#### Parameters + +| Name | Type | Description | +|--------------|------------------|-------------------------------------------| +| `index` | `String` | Target index | +| `query` | `String` | SQL UPDATE or JSON query | +| `pipelineId` | `Option[String]` | Optional ingest pipeline to apply | +| `refresh` | `Boolean` | Whether to refresh the index after update | + +#### Returns + +- `ElasticSuccess[Long]` → number of updated documents +- `ElasticFailure` → error details + +--- + +#### Examples + +#### SQL UPDATE with WHERE + +```scala +client.updateByQuery( + "person", + """UPDATE person SET name = 'Another Name' WHERE uuid = 'A16'""" +) +``` + +#### SQL UPDATE without WHERE (update all) + +```scala +client.updateByQuery( + "person", + """UPDATE person SET birthDate = '1972-12-26'""" +) +``` + +#### JSON query with user pipeline + +```scala +client.updateByQuery( + "person", + """{"query": {"match_all": {}}}""", + pipelineId = Some("set-birthdate-1972-12-26") +) +``` + +#### SQL UPDATE + user pipeline (merged) + +```scala +client.updateByQuery( + "person", + """UPDATE person SET birthDate = '1972-12-26' WHERE uuid = 'A16'""", + pipelineId = Some("user-update-name") +) +``` + +--- + +#### Behavior Summary + +- SQL UPDATE is fully supported +- SET clause → ingest pipeline +- WHERE clause → JSON query +- Pipelines are merged when needed +- Temporary pipelines are cleaned up automatically +- Index state is preserved +- Works consistently across ES6, ES7, ES8, ES9 + +--- + ## Public Methods ### createIndex From db049e65b6c27a55472f3b08d8d2430b39a9c80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Dec 2025 00:03:25 +0100 Subject: [PATCH 40/95] add support for indexByQuery --- .../softnetwork/elastic/client/BulkApi.scala | 32 +- .../client/ElasticClientDelegator.scala | 32 +- .../elastic/client/IndicesApi.scala | 316 ++++++++++- .../elastic/client/NopeClientApi.scala | 59 +- .../elastic/client/ScrollApi.scala | 64 ++- .../elastic/client/SearchApi.scala | 14 +- .../elastic/client/TemplateConverter.scala | 16 + .../client/metrics/MetricsElasticClient.scala | 31 +- .../elastic/client/result/package.scala | 42 +- .../elastic/client/AliasApiSpec.scala | 155 +---- .../elastic/client/IndicesApiSpec.scala | 477 +--------------- .../elastic/client/MappingApiSpec.scala | 51 +- .../elastic/client/SettingsApiSpec.scala | 534 +----------------- documentation/client/indices.md | 273 +++++++++ .../elastic/client/jest/JestIndicesApi.scala | 7 +- .../elastic/client/jest/JestPipelineApi.scala | 16 + .../elastic/client/jest/JestTemplateApi.scala | 16 + .../client/jest/actions/GetIndex.scala | 16 + .../client/jest/actions/Pipeline.scala | 16 + .../client/jest/actions/Template.scala | 16 + .../client/jest/actions/WaitForShards.scala | 16 + .../client/JestClientInsertByQuerySpec.scala | 10 + .../client/rest/RestHighLevelClientApi.scala | 2 + ...RestHighLevelClientInsertByQuerySpec.scala | 8 + .../client/rest/RestHighLevelClientApi.scala | 2 + ...RestHighLevelClientInsertByQuerySpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 2 + .../client/JavaClientInsertByQuerySpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 6 +- .../client/JavaClientInsertByQuerySpec.scala | 8 + .../elastic/sql/parser/Parser.scala | 33 +- .../elastic/sql/query/package.scala | 80 ++- .../elastic/sql/schema/package.scala | 3 +- .../elastic/sql/parser/ParserSpec.scala | 35 +- .../elastic/client/BulkApiSpec.scala | 14 +- .../elastic/client/ElasticClientSpec.scala | 46 +- .../elastic/client/EmployeeData.scala | 2 +- .../elastic/client/InsertByQuerySpec.scala | 263 +++++++++ .../elastic/client/PipelineApiSpec.scala | 16 + .../elastic/client/TemplateApiSpec.scala | 16 + 40 files changed, 1452 insertions(+), 1309 deletions(-) create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala index df9714f4..c2bdf29a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala @@ -23,12 +23,9 @@ import akka.stream.scaladsl.{Balance, Flow, GraphDSL, Merge, Sink, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.file._ import app.softnetwork.elastic.client.result.{ElasticResult, ElasticSuccess} - +import app.softnetwork.elastic.sql.schema.sqlConfig import org.apache.hadoop.conf.Configuration -import org.json4s.DefaultFormats -import org.json4s.jackson.JsonMethods.parse - import java.time.LocalDate import java.time.format.DateTimeFormatter import scala.concurrent.{ExecutionContext, Future} @@ -83,7 +80,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromFile( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -128,7 +125,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromParquet( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, callbacks: BulkCallbacks = BulkCallbacks.default, bufferSize: Int = 500, hadoopConf: Option[Configuration] = None @@ -153,7 +150,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromJson( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, callbacks: BulkCallbacks = BulkCallbacks.default, bufferSize: Int = 500, validateJson: Boolean = true, @@ -219,7 +216,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -324,7 +321,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -407,7 +404,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -602,7 +599,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { private def toBulkItem[D]( toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -610,13 +607,13 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { parentIdKey: Option[String], item: D )(implicit bulkOptions: BulkOptions): BulkItem = { - implicit val formats: DefaultFormats = org.json4s.DefaultFormats val document = toDocument(item) - val jsonMap = parse(document, useBigDecimalForDouble = false).extract[Map[String, Any]] + val jsonNode = mapper.readTree(document) + val jsonMap = mapper.convertValue(jsonNode, classOf[Map[String, Any]]) // extract id - val id = idKey.flatMap { i => - jsonMap.get(i).map(_.toString) + val id = idKey.map { keys => + keys.map(i => jsonMap.getOrElse(i, "").toString).mkString(sqlConfig.compositeKeySeparator) } // extract final index name @@ -628,8 +625,9 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { val index = suffixDateKey .flatMap { s => jsonMap.get(s).map { d => - val strDate = d.toString.substring(0, 10) - val date = LocalDate.parse(strDate, DateTimeFormatter.ofPattern("yyyy-MM-dd")) + val pattern = suffixDatePattern.getOrElse("yyyy-MM-dd") + val strDate = d.toString.substring(0, pattern.length) + val date = LocalDate.parse(strDate, DateTimeFormatter.ofPattern(pattern)) date.format( suffixDatePattern .map(DateTimeFormatter.ofPattern) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 398b592b..56481870 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -24,7 +24,12 @@ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} +import app.softnetwork.elastic.sql.query.{ + DqlStatement, + SQLAggregation, + SelectStatement, + SingleSearch +} import app.softnetwork.elastic.sql.schema.TableAlias import com.typesafe.config.Config import org.json4s.Formats @@ -209,6 +214,21 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Long] = delegate.updateByQuery(index, query, pipelineId, refresh) + /** Insert documents by query into an index. + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit + system: ActorSystem + ): Future[ElasticResult[Long]] = + delegate.insertByQuery(index, query, refresh) + override private[client] def executeCreateIndex( index: String, settings: String, @@ -1249,9 +1269,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SelectStatement, config: ScrollConfig)(implicit + override def scroll(statement: DqlStatement, config: ScrollConfig)(implicit system: ActorSystem - ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(sql, config) + ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(statement, config) /** Scroll and convert results into typed entities from an SQL query. * @@ -1347,7 +1367,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -1409,7 +1429,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -1439,7 +1459,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 0ddf6db5..17181be5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -16,14 +16,20 @@ package app.softnetwork.elastic.client +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk.BulkOptions import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{Delete, From, SingleSearch, Table, Update} +import app.softnetwork.elastic.sql.query.{Delete, From, Insert, SingleSearch, Table, Update} import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, TableAlias} import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode +import scala.concurrent.{ExecutionContext, Future} + /** Index management API. * * This implementation provides: @@ -32,7 +38,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode * - Parameter validation * - Automatic retry for transient errors */ -trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi with VersionApi => +trait IndicesApi extends ElasticClientHelpers { + _: RefreshApi with PipelineApi with BulkApi with ScrollApi with VersionApi => // ======================================================================== // PUBLIC METHODS @@ -837,6 +844,258 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w } } + /** Insert documents by query into an index. + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + def insertByQuery( + index: String, + query: String, + refresh: Boolean = true + )(implicit system: ActorSystem): Future[ElasticResult[Long]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val result = for { + // 1. Validate index + _ <- Future.fromTry( + validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index) + ) + ) + .toTry + ) + + // 2. Parse SQL INSERT + parsed <- parseInsertQuery(index, query).toFuture + + // 3. Load index metadata + idx <- Future.fromTry(getIndex(index).toEither.flatMap { + case Some(i) => Right(i.asTable) + case None => + Left(ElasticError.notFound(index, "insertByQuery")) + }.toTry) + + // 3.b Compute effective insert columns (handles INSERT ... AS SELECT without column list) + val effectiveInsertCols: Seq[String] = parsed.values match { + case Left(single: SingleSearch) => + if (parsed.cols.isEmpty) + single.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.sourceField) + } + else parsed.cols + + case _ => + parsed.cols + } + + // 3.c Validate ON CONFLICT rules + _ <- Future.fromTry { + val pk = idx.primaryKey + val doUpdate = parsed.doUpdate + val conflictTarget = parsed.conflictTarget + + val validation: Either[ElasticError, Unit] = + if (!doUpdate) { + Right(()) + } else if (pk.nonEmpty) { + // --- Case 1: PK defined in index + val conflictKey = conflictTarget.getOrElse(pk) + + // conflictTarget must match PK exactly + if (conflictKey.toSet != pk.toSet) + Left( + ElasticError( + message = s"Conflict target columns [${conflictKey + .mkString(",")}] must match primary key [${pk.mkString(",")}]", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + // INSERT must include all PK columns + else if (!pk.forall(effectiveInsertCols.contains)) + Left( + ElasticError( + message = + s"INSERT must include all primary key columns [${pk.mkString(",")}] when using ON CONFLICT DO UPDATE", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + else + Right(()) + } else { + // --- Case 2: No PK defined in index + conflictTarget match { + case None => + Left( + ElasticError( + message = + "ON CONFLICT DO UPDATE requires a conflict target when no primary key is defined in the index", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + + case Some(conflictKey) => + if (!conflictKey.forall(effectiveInsertCols.contains)) + Left( + ElasticError( + message = + s"INSERT must include all conflict target columns [${conflictKey.mkString(",")}] when using ON CONFLICT DO UPDATE", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + else + Right(()) + } + } + + validation.toTry + } + + // 3.d Validate SELECT columns for INSERT ... AS SELECT + _ <- Future.fromTry { + parsed.values match { + + // INSERT ... VALUES → rien à valider + case Right(_) => + Right(()).toTry + + // INSERT ... AS SELECT + case Left(single: SingleSearch) => + val selectCols = single.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.sourceField) + } + + // Vérifier que toutes les colonnes de l'INSERT sont présentes dans le SELECT + val missing = effectiveInsertCols.filterNot(selectCols.contains) + + if (missing.nonEmpty) + Left( + ElasticError( + message = + s"INSERT columns [${effectiveInsertCols.mkString(",")}] must all be present in SELECT output columns [${selectCols + .mkString(",")}]. Missing: ${missing.mkString(",")}", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ).toTry + else + Right(()).toTry + + case Left(_) => + Left( + ElasticError( + message = "INSERT AS SELECT requires a SELECT statement", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ).toTry + } + } + + // 4. Derive bulk options + idKey = idx.primaryKey match { + case Nil => None + case pk => Some(pk.toSet) + } + suffixKey = idx.partitionBy.map(_.column) + suffixPattern = idx.partitionBy.flatMap(_.dateFormats.headOption) + + // 5. Build source of documents + source <- Future.fromTry((parsed.values match { + + // INSERT … VALUES + case Right(_) => + parsed.toJson match { + case Some(jsonNode) => + Right(Source.single(jsonNode.toString)) + case None => + Left( + ElasticError( + message = "Invalid INSERT ... VALUES clause", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + } + + // INSERT … AS SELECT + case Left(single: SingleSearch) => + Right( + scroll(single).map { case (row, _) => + val jsonNode: JsonNode = row - "_id" - "_index" - "_score" - "_sort" + jsonNode.toString + } + ) + + case Left(_) => + Left( + ElasticError( + message = "INSERT AS SELECT requires a SELECT statement", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + }).toTry) + + // 6. Bulk insert + bulkResult <- bulkWithResult[String]( + items = source, + toDocument = identity, + indexKey = Some(index), + idKey = idKey, + suffixDateKey = suffixKey, + suffixDatePattern = suffixPattern, + update = Some(parsed.doUpdate) + )(BulkOptions(index), system) + + // 7. Refresh + _ <- + if (refresh) this.refresh(index).toFuture else Future.successful(true) + + } yield bulkResult.successCount.toLong + + result.map(ElasticSuccess(_)).recover { + case e: ElasticError => + ElasticFailure( + e.copy( + operation = Some("insertByQuery"), + index = Some(index) + ) + ) + case e => + ElasticFailure( + ElasticError( + message = e.getMessage, + operation = Some("insertByQuery"), + index = Some(index) + ) + ) + } + } + private def parseQueryForUpdate( index: String, query: String @@ -1151,6 +1410,59 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi with PipelineApi w } } + def parseInsertQuery( + index: String, + query: String + ): ElasticResult[Insert] = { + + Parser(query) match { + case Left(err) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL: ${err.msg}" + ) + ) + + case Right(insert: Insert) => + if (insert.table != index) + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"SQL table '${insert.table}' does not match index '$index'" + ) + ) + else + insert.validate() match { + case Left(msg) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = msg + ) + ) + case Right(_) => + ElasticSuccess(insert) + } + + case Right(_) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = "Only INSERT statements are allowed" + ) + ) + } + } + private def sqlErrorFor(operation: String, index: String, message: String): ElasticError = ElasticError( operation = Some(operation), diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index c8ef851c..2a42e26c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -18,7 +18,14 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.actor.ActorSystem -import akka.stream.scaladsl.Source +import akka.stream.scaladsl.{Flow, Source} +import app.softnetwork.elastic.client.bulk.{ + BulkElasticAction, + BulkItem, + BulkOptions, + FailedDocument, + SuccessfulDocument +} import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.query.SQLAggregation import app.softnetwork.elastic.client.result._ @@ -320,4 +327,54 @@ trait NopeClientApi extends ElasticClientApi { override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + ElasticSuccess(0L) + + override type BulkActionType = this.type + + override type BulkResultType = this.type + + override private[client] def toBulkAction(bulkItem: BulkItem): BulkActionType = + throw new UnsupportedOperationException + + override private[client] implicit def toBulkElasticAction(a: BulkActionType): BulkElasticAction = + throw new UnsupportedOperationException + + /** Basic flow for executing a bulk action. This method must be implemented by concrete classes + * depending on the Elasticsearch version and client used. + * + * @param bulkOptions + * configuration options + * @return + * Flow transforming bulk actions into results + */ + override private[client] def bulkFlow(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[BulkActionType], BulkResultType, NotUsed] = + throw new UnsupportedOperationException + + /** Convert a BulkResultType into individual results. This method must extract the successes and + * failures from the ES response. + * + * @param result + * raw result from the bulk + * @return + * sequence of Right(id) for success or Left(failed) for failure + */ + override private[client] def extractBulkResults( + result: BulkResultType, + originalBatch: Seq[BulkItem] + ): Seq[Either[FailedDocument, SuccessfulDocument]] = + throw new UnsupportedOperationException + + /** Conversion BulkActionType -> BulkItem */ + override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = + throw new UnsupportedOperationException } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 19680764..efcc43f9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -29,7 +29,13 @@ import app.softnetwork.elastic.client.scroll.{ UseSearchAfter } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement, SingleSearch} +import app.softnetwork.elastic.sql.query.{ + DqlStatement, + MultiSearch, + SQLAggregation, + SelectStatement, + SingleSearch +} import org.json4s.{Formats, JNothing} import org.json4s.jackson.JsonMethods.parse @@ -117,34 +123,60 @@ trait ScrollApi extends ElasticClientHelpers { /** Create a scrolling source with automatic strategy selection */ def scroll( - sql: SelectStatement, + statement: DqlStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { implicit def timestamp: Long = System.currentTimeMillis() - sql.statement match { - case Some(single: SingleSearch) => + statement match { + case select: SelectStatement => + select.statement match { + case Some(single: SingleSearch) => + if (single.windowFunctions.nonEmpty) + return scrollWithWindowEnrichment(select.score, single, config) + + val sqlRequest = single.copy(score = select.score) + val elasticQuery = + ElasticQuery(sqlRequest, collection.immutable.Seq(sqlRequest.sources: _*)) + scrollWithMetrics( + elasticQuery, + sqlRequest.fieldAliases, + sqlRequest.sqlAggregations, + config, + single.sorts.nonEmpty + ) + + case Some(_) => + Source.failed( + new UnsupportedOperationException( + "Scrolling is not supported for multi-search queries" + ) + ) + + case None => + Source.failed( + new IllegalArgumentException("SQL query does not contain a valid search request") + ) + } + case single: SingleSearch => if (single.windowFunctions.nonEmpty) - return scrollWithWindowEnrichment(sql, single, config) + return scrollWithWindowEnrichment(None, single, config) - val sqlRequest = single.copy(score = sql.score) val elasticQuery = - ElasticQuery(sqlRequest, collection.immutable.Seq(sqlRequest.sources: _*)) + ElasticQuery(single, collection.immutable.Seq(single.sources: _*)) scrollWithMetrics( elasticQuery, - sqlRequest.fieldAliases, - sqlRequest.sqlAggregations, + single.fieldAliases, + single.sqlAggregations, config, single.sorts.nonEmpty ) - - case Some(_) => + case _: MultiSearch => Source.failed( new UnsupportedOperationException("Scrolling is not supported for multi-search queries") ) - - case None => + case _ => Source.failed( - new IllegalArgumentException("SQL query does not contain a valid search request") + new IllegalArgumentException("Scrolling is only supported for SELECT statements") ) } } @@ -377,7 +409,7 @@ trait ScrollApi extends ElasticClientHelpers { /** Scroll with window function enrichment */ private def scrollWithWindowEnrichment( - sql: SelectStatement, + score: Option[Double], request: SingleSearch, config: ScrollConfig )(implicit @@ -394,7 +426,7 @@ trait ScrollApi extends ElasticClientHelpers { Future(executeWindowAggregations(request)) // Create base query without window functions - val baseQuery = createBaseQuery(sql, request) + val baseQuery = createBaseQuery(score, request) // Stream and enrich Source diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 297dfe90..98c36534 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -75,7 +75,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { sql = Some(sql.query) ) if (single.windowFunctions.exists(_.isWindowing) && single.groupBy.isEmpty) - searchWithWindowEnrichment(sql, single) + searchWithWindowEnrichment(sql.score, single) else singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) @@ -1105,7 +1105,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * functions) 3. Enrich results with window values */ private def searchWithWindowEnrichment( - sql: SelectStatement, + score: Option[Double], request: SingleSearch )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { @@ -1116,7 +1116,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { windowCache <- executeWindowAggregations(request) // Step 2: Execute base query (without window functions) - baseResponse <- executeBaseQuery(sql, request) + baseResponse <- executeBaseQuery(score, request) // Step 3: Enrich results enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) @@ -1219,11 +1219,11 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Execute base query without window functions */ private def executeBaseQuery( - sql: SelectStatement, + score: Option[Double], request: SingleSearch )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { - val baseQuery = createBaseQuery(sql, request) + val baseQuery = createBaseQuery(score, request) logger.info(s"🔍 Executing base query without window functions ${baseQuery.sql}") @@ -1241,7 +1241,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Create base query by removing window functions from SELECT */ protected def createBaseQuery( - sql: SelectStatement, + score: Option[Double], request: SingleSearch ): SingleSearch = { @@ -1253,7 +1253,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { .copy( select = request.select.copy(fields = baseFields) ) - .copy(score = sql.score) + .copy(score = score) .update() baseRequest diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala index 4f6eaedd..35e63649 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client import app.softnetwork.elastic.sql.serialization.JacksonConfig diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index b7603a82..84191f30 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -32,7 +32,7 @@ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement} +import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} import app.softnetwork.elastic.sql.schema.TableAlias import org.json4s.Formats @@ -204,6 +204,25 @@ class MetricsElasticClient( delegate.updateByQuery(index, query, pipelineId, refresh) } + /** Insert documents by query into an index. + * + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit + system: ActorSystem + ): Future[ElasticResult[Long]] = { + measureAsync("insertByQuery", Some(index)) { + delegate.insertByQuery(index, query, refresh) + }(system.dispatcher) + } + // ==================== AliasApi ==================== override def addAlias(index: String, alias: String): ElasticResult[Boolean] = { @@ -958,12 +977,12 @@ class MetricsElasticClient( /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SelectStatement, config: ScrollConfig)(implicit + override def scroll(statement: DqlStatement, config: ScrollConfig)(implicit system: ActorSystem ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { // Note: For streams, we measure at the beginning but not every element val startTime = System.currentTimeMillis() - val source = delegate.scroll(sql, config) + val source = delegate.scroll(statement, config) source.watchTermination() { (_, done) => done.onComplete { result => @@ -1028,7 +1047,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -1057,7 +1076,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -1098,7 +1117,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 23861a22..1d214f4a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -48,6 +48,13 @@ package object result { /** Converts to Either */ def toEither: Either[ElasticError, T] + /** Converts to Future */ + def toFuture(implicit + ec: scala.concurrent.ExecutionContext + ): scala.concurrent.Future[T] = { + scala.concurrent.Future.fromTry(this.toEither.toTry) + } + /** Fold pattern matching */ def fold[U](onFailure: ElasticError => U, onSuccess: T => U): U @@ -116,7 +123,9 @@ package object result { /** Represents a failed operation. */ - case class ElasticFailure(elasticError: ElasticError) extends ElasticResult[Nothing] { + case class ElasticFailure(elasticError: ElasticError) + extends Throwable(elasticError.message, elasticError) + with ElasticResult[Nothing] { override def isSuccess: Boolean = false override def map[U](f: Nothing => U): ElasticResult[U] = this @@ -151,7 +160,7 @@ package object result { statusCode: Option[Int] = None, index: Option[String] = None, operation: Option[String] = None - ) { + ) extends Throwable(message, cause.orNull) { /** Complete message with context */ def fullMessage: String = { @@ -174,6 +183,35 @@ package object result { } } + object ElasticError { + + /** Creates an ElasticError from an exception */ + def fromThrowable( + ex: Throwable, + statusCode: Option[Int] = None, + index: Option[String] = None, + operation: Option[String] = None + ): ElasticError = { + ElasticError( + ex.getMessage, + Some(ex), + statusCode, + index, + operation + ) + } + + /** Creates a not found error */ + def notFound(index: String, operation: String): ElasticError = { + ElasticError( + s"Resource not found in index '$index' during operation '$operation'", + statusCode = Some(404), + index = Some(index), + operation = Some(operation) + ) + } + } + /** Companion object with utility methods. */ object ElasticResult { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index b01aceed..cf247a1f 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -7,8 +7,6 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ -import app.softnetwork.elastic.sql.query -import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for AliasApi */ @@ -23,13 +21,7 @@ class AliasApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestAliasApi - extends AliasApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + class TestAliasApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -81,65 +73,6 @@ class AliasApiSpec executeGetIndexResult } - // Other required methods - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } var aliasApi: TestAliasApi = _ @@ -1684,12 +1617,7 @@ class AliasApiSpec "not call execute methods when validation fails" in { // Given var executeCalled = false - val validatingApi = new AliasApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeAddAlias( @@ -1704,85 +1632,6 @@ class AliasApiSpec ElasticSuccess(true) } - override private[client] def executeRemoveAlias( - index: String, - alias: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = - ??? - override private[client] def executeGetAliases(index: String): ElasticResult[String] = ??? - override private[client] def executeSwapAlias( - oldIndex: String, - newIndex: String, - alias: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index a0261657..d665a82c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -7,7 +7,6 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ -import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for IndicesApi @@ -23,12 +22,7 @@ class IndicesApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestIndicesApi - extends IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + class TestIndicesApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables for each operation @@ -85,47 +79,6 @@ class IndicesApiSpec executeRefreshResult } - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } var indicesApi: TestIndicesApi = _ @@ -724,11 +677,7 @@ class IndicesApiSpec "fail when target index does not exist" in { // Given var callCount = 0 - val checkingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val checkingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -737,73 +686,6 @@ class IndicesApiSpec else ElasticSuccess(false) // target doesn't exist } - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -831,11 +713,7 @@ class IndicesApiSpec "fail when target existence check fails" in { // Given var callCount = 0 - val checkingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val checkingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -844,73 +722,6 @@ class IndicesApiSpec else ElasticFailure(ElasticError("Connection error")) // target check fails } - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1094,11 +905,7 @@ class IndicesApiSpec "validate index name before calling execute methods" in { // Given var executeCalled = false - val validatingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( @@ -1111,68 +918,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1185,11 +930,7 @@ class IndicesApiSpec "validate settings after index name validation" in { // Given var executeCalled = false - val validatingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( @@ -1202,68 +943,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1276,11 +955,7 @@ class IndicesApiSpec "validate both indices in reindex before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1288,73 +963,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1367,11 +975,7 @@ class IndicesApiSpec "check source and target are different before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1379,73 +983,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index 82ffe04b..75f3558b 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -6,7 +6,6 @@ import org.scalatest.BeforeAndAfterEach import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.slf4j.Logger import app.softnetwork.elastic.client.result._ -import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for MappingApi @@ -29,14 +28,7 @@ class MappingApiSpec """{"properties":{"name":{"type":"text"},"age":{"type":"integer"}}}""" // Concrete implementation for testing - class TestMappingApi - extends MappingApi - with SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + class TestMappingApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -112,47 +104,6 @@ class MappingApiSpec settings: String ): ElasticResult[Boolean] = ElasticSuccess(true) - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } var mappingApi: TestMappingApi = _ diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index cca9b114..d59965f9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -24,13 +24,7 @@ class SettingsApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestSettingsApi - extends SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + class TestSettingsApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -60,67 +54,6 @@ class SettingsApiSpec executeOpenIndexResult } - // Other required methods - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } var settingsApi: TestSettingsApi = _ @@ -765,12 +698,7 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -793,74 +721,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -877,12 +737,7 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -902,74 +757,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -984,12 +771,7 @@ class SettingsApiSpec // Given var openCalled = false - val workflowApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1008,74 +790,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1612,12 +1326,7 @@ class SettingsApiSpec "validate index name before calling executeCloseIndex" in { // Given var closeCalled = false - val validatingApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1625,79 +1334,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1710,12 +1346,7 @@ class SettingsApiSpec "validate settings after index name" in { // Given var closeCalled = false - val validatingApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1723,79 +1354,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When @@ -1808,12 +1366,7 @@ class SettingsApiSpec "validate index name before calling executeLoadSettings" in { // Given var loadCalled = false - val validatingApi = new SettingsApi - with IndicesApi - with RefreshApi - with PipelineApi - with VersionApi - with SerializationApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { @@ -1821,79 +1374,6 @@ class SettingsApiSpec ElasticSuccess("{}") } - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeCreateIndex( - index: String, - settings: String, - mappings: Option[String], - aliases: Seq[TableAlias] - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetIndex( - index: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean, - pipeline: Option[String] - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? - - override private[client] def executeCreatePipeline( - pipelineName: String, - pipelineDefinition: String - ): ElasticResult[Boolean] = ??? - - override private[client] def executeDeletePipeline( - pipelineName: String, - ifExists: Boolean - ): ElasticResult[Boolean] = ??? - - override private[client] def executeGetPipeline( - pipelineName: String - ): ElasticResult[Option[String]] = ??? - - override private[client] def executeVersion(): ElasticResult[String] = ??? - - /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query - * serialization. - * - * @param sqlSearch - * the SQL search request to convert - * @return - * JSON string representation of the query - */ - override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: query.SingleSearch - )(implicit timestamp: Long): String = ??? - - override private[client] def executeDeleteByQuery( - index: String, - query: String, - refresh: Boolean - ): ElasticResult[Long] = ??? - - override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = - ??? - - override private[client] def executeUpdateByQuery( - index: String, - query: String, - pipelineId: Option[String], - refresh: Boolean - ): ElasticResult[Long] = ??? } // When diff --git a/documentation/client/indices.md b/documentation/client/indices.md index 7f34e98e..db99ff36 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -408,6 +408,279 @@ client.updateByQuery( --- +### insertByQuery — SQL‑Driven Bulk Insert & Upsert for Elasticsearch + +`insertByQuery` executes SQL `INSERT` statements and writes documents into an Elasticsearch index. +It supports: + +- `INSERT … VALUES` +- `INSERT … AS SELECT` +- `ON CONFLICT DO UPDATE` +- Primary keys (simple or composite) +- Partitioning (`partition_by`) +- Column aliasing +- Automatic mapping between SELECT output and INSERT columns +- Strict SQL validation before execution + +This API is designed for ETL pipelines, migrations, and SQL‑driven ingestion workflows. + +--- + +#### Supported SQL Syntax + +**INSERT … VALUES** + +```sql +INSERT INTO index_name (col1, col2) +VALUES (v1, v2) +``` + +**INSERT … AS SELECT** + +```sql +INSERT INTO index_name (col1, col2) +AS SELECT a AS col1, b AS col2 FROM other_index +``` + +**INSERT without column list** + +```sql +INSERT INTO index_name +SELECT a, b, c FROM other_index +``` + +**ON CONFLICT DO UPDATE** + +```sql +INSERT INTO index_name (...) +VALUES (...) +ON CONFLICT DO UPDATE +``` + +or with explicit conflict target: + +```sql +INSERT INTO index_name (...) +VALUES (...) +ON CONFLICT (col1, col2) DO UPDATE +``` + +--- + +#### Primary Key Semantics + +Primary keys are defined in the index mapping under `_meta.primary_key`: + +``` +"_meta": { + "primary_key": ["order_id", "customer_id"] +} +``` + +Rules: + +- PK may be simple or composite. +- PK determines the Elasticsearch `_id`. +- All PK columns must be present in the INSERT. +- For `INSERT … AS SELECT`, the SELECT must produce all PK columns. + +--- + +#### Conflict Handling + +**ON CONFLICT DO UPDATE** + +Triggers an **upsert**. + +Rules when PK exists: + +- If conflictTarget omitted → PK is used. +- If conflictTarget provided → must match PK exactly. +- All PK columns must be present in the INSERT. + +Rules when PK does not exist: + +- conflictTarget is mandatory. +- All conflictTarget columns must be present in the INSERT. + +--- + +#### INSERT … VALUES + +Direct insertion of literal values. + +Validation: + +- Column count must match value count. +- All PK columns must be present if `ON CONFLICT` is used. + +--- + +#### INSERT … AS SELECT + +Inserts documents produced by a SELECT query. + +Behavior: + +- Executes a scroll query. +- Removes Elasticsearch metadata (`_id`, `_index`, `_score`, `_sort`). +- Maps SELECT output to INSERT columns **by name**. +- Supports aliasing. +- Supports INSERT without column list. + +--- + +#### Column Mapping Rules + +**Mapping is always by name, never by position.** + +Example: + +```sql +SELECT foo AS bar +``` + +→ INSERT column `bar` receives the value of `foo`. + +--- + +#### Validation Rules + +**General** + +- INSERT column count must match VALUES count. +- conflictTarget ⊆ INSERT columns. +- INSERT must include all PK columns if PK exists. +- If PK does not exist → conflictTarget is mandatory. + +**For DO UPDATE** + +- conflictTarget must match PK exactly when PK exists. + +**For INSERT … AS SELECT** + +- All INSERT columns must be present in SELECT output. +- SELECT metadata fields are ignored. +- Aliases are resolved correctly. + +**For INSERT without column list** + +- INSERT columns = SELECT output columns. + +--- + +#### Composite Primary Keys + +Composite PKs are fully supported: + +``` +"_meta": { + "primary_key": ["order_id", "customer_id"] +} +``` + +The Elasticsearch `_id` is constructed from all PK columns, e.g.: + +``` +O1001|C001 +``` + +This ensures deterministic conflict detection. + +--- + +#### Partitioning + +If the index defines: + +``` +"_meta": { + "partition_by": { + "column": "order_date", + "granularity": "d" + } +} +``` + +Then: + +- Documents are routed to partitioned indices (e.g., `orders-2024-02-01`). +- The partition key is extracted from the INSERT or SELECT row. + +--- + +#### Error Handling + +`insertByQuery` returns: + +- `ElasticSuccess(count)` on success +- `ElasticFailure(error)` on validation or execution error + +Errors include: + +- Missing PK columns +- conflictTarget mismatch +- Missing SELECT columns +- Invalid SQL syntax +- Unsupported INSERT form +- Elasticsearch bulk failures + +`ON CONFLICT DO NOTHING` never raises a conflict error. + +--- + +#### Examples + +**Insert with VALUES** + +```sql +INSERT INTO customers (customer_id, name, email) +VALUES ('C010', 'Bob', 'bob@example.com') +``` + +**Upsert with PK** + +```sql +INSERT INTO products (sku, name, price) +VALUES ('SKU-001', 'Laptop Pro', 1499.99) +ON CONFLICT DO UPDATE +``` + +**Insert‑or‑ignore with DO NOTHING** + +```sql +INSERT INTO products (sku, name, price) +VALUES ('SKU-001', 'Laptop Pro', 1499.99) +ON CONFLICT DO NOTHING +``` + +**Insert from SELECT with alias mapping** + +```sql +INSERT INTO orders (order_id, customer_id, total) +AS SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders +``` + +**Insert from SELECT without column list** + +```sql +INSERT INTO orders +SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders +``` + +**Upsert with composite PK** + +```sql +INSERT INTO orders (order_id, customer_id, total) +AS SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders_updates +ON CONFLICT (order_id, customer_id) DO UPDATE +``` + +--- + ## Public Methods ### createIndex diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index c4095a65..dcfbe003 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -33,7 +33,12 @@ import scala.util.Try * [[IndicesApi]] for generic API documentation */ trait JestIndicesApi extends IndicesApi with JestClientHelpers { - _: JestRefreshApi with JestPipelineApi with JestVersionApi with JestClientCompanion => + _: JestRefreshApi + with JestPipelineApi + with JestScrollApi + with JestBulkApi + with JestVersionApi + with JestClientCompanion => /** Create an index with the given settings. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala index 4bd621d1..35d392de 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.jest.actions.Pipeline diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala index aec2ceb6..c7c86c3b 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.jest.actions.Template diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala index 6e413508..9f109c96 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest.actions import io.searchbox.action.AbstractAction diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala index 733f1549..1e0f168d 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest.actions import io.searchbox.client.config.ElasticsearchVersion diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala index 24036909..1c38c2fa 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest.actions import io.searchbox.client.config.ElasticsearchVersion diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala index 5b42f534..dcfc5c43 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client.jest.actions import io.searchbox.action.{AbstractAction, GenericResultAbstractAction} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala new file mode 100644 index 00000000..06ab8d14 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class JestClientInsertByQuerySpec extends InsertByQuerySpec with EmbeddedElasticTestKit { + override def client: ElasticClientApi = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 3831e3ec..0ec9825e 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -145,6 +145,8 @@ trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientH trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { _: RestHighLevelClientRefreshApi with RestHighLevelClientPipelineApi + with RestHighLevelClientScrollApi + with RestHighLevelClientBulkApi with RestHighLevelClientVersionApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala new file mode 100644 index 00000000..2f81b747 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with EmbeddedElasticTestKit { + override lazy val client: ElasticClientApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 3154ef0d..9fe9316d 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -154,6 +154,8 @@ trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientH trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { _: RestHighLevelClientRefreshApi with RestHighLevelClientPipelineApi + with RestHighLevelClientScrollApi + with RestHighLevelClientBulkApi with RestHighLevelClientVersionApi with RestHighLevelClientCompanion => diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala new file mode 100644 index 00000000..5dbbd428 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 25e09b20..037b85b2 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -123,6 +123,8 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { _: JavaClientRefreshApi with JavaClientPipelineApi + with JavaClientScrollApi + with JavaClientBulkApi with JavaClientVersionApi with JavaClientCompanion => override private[client] def executeCreateIndex( diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala new file mode 100644 index 00000000..94e8a1d0 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 51b70f7e..ed5cb31f 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -116,7 +116,11 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * [[IndicesApi]] for index management operations */ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { - _: JavaClientRefreshApi with JavaClientPipelineApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientPipelineApi + with JavaClientScrollApi + with JavaClientBulkApi + with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, settings: String, diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala new file mode 100644 index 00000000..94e8a1d0 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index d57c9bfa..b4eb3a2b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -61,9 +61,9 @@ object Parser with LimitParser { def single: PackratParser[SingleSearch] = { - phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.?) ^^ { - case s ~ f ~ w ~ g ~ h ~ o ~ l => - SingleSearch(s, f, w, g, h, o, l).update() + phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.? ~ onConflict.?) ^^ { + case s ~ f ~ w ~ g ~ h ~ o ~ l ~ oc => + SingleSearch(s, f, w, g, h, o, l, onConflict = oc).update() } } @@ -519,12 +519,30 @@ object Parser truncateTable | dropPipeline + def onConflict: PackratParser[OnConflict] = + ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { + case target ~ action => + OnConflict(target, action == "UPDATE") + } + + def conflictTarget: PackratParser[List[String]] = + start ~> repsep(ident, separator) <~ end + /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ def insert: PackratParser[Insert] = ("INSERT" ~ "INTO") ~ ident ~ opt(start ~> repsep(ident, separator) <~ end) ~ (("VALUES" ~ start ~> repsep(value, separator) <~ end) ^^ { vs => Right(vs) } - | dqlStatement ^^ { q => Left(q) }) ^^ { case _ ~ table ~ colsOpt ~ vals => - Insert(table, colsOpt.getOrElse(Nil), vals) + | "AS".? ~> dqlStatement ^^ { q => Left(q) }) ~ opt(onConflict) ^^ { + case _ ~ table ~ colsOpt ~ vals ~ conflict => + conflict match { + case Some(c) => Insert(table, colsOpt.getOrElse(Nil), vals, Some(c)) + case _ => + vals match { + case Left(q: SingleSearch) => + Insert(table, colsOpt.getOrElse(Nil), vals, q.onConflict) + case _ => Insert(table, colsOpt.getOrElse(Nil), vals) + } + } } /** UPDATE table SET col1 = v1, col2 = v2 [WHERE ...] */ @@ -743,7 +761,10 @@ trait Parser "last_value", "ltrim", "rtrim", - "replace" + "replace", + "on", + "conflict", + "do" ) private val identifierRegexStr = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 2cf32d11..ef709565 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -31,6 +31,8 @@ import app.softnetwork.elastic.sql.schema.{ Table => DdlTable } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode import java.time.Instant @@ -74,10 +76,11 @@ package object query { limit: Option[Limit] = None, score: Option[Double] = None, deleteByQuery: Boolean = false, - updateByQuery: Boolean = false + updateByQuery: Boolean = false, + onConflict: Option[OnConflict] = None ) extends DqlStatement { override def sql: String = - s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" + s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}${asString(onConflict)}" lazy val fieldAliases: Map[String, String] = select.fieldAliases lazy val tableAliases: Map[String, String] = from.tableAliases @@ -269,17 +272,34 @@ package object query { sealed trait DmlStatement extends Statement + case class OnConflict(target: Option[Seq[String]], doUpdate: Boolean) extends Token { + override def sql: String = { + val targetSql = + target match { + case Some(t) => if (t.isEmpty) " () " else s" (${t.mkString(", ")}) " + case None => " " + } + val actionSql = if (doUpdate) "DO UPDATE" else "DO NOTHING" + s" ON CONFLICT$targetSql$actionSql" + } + } + case class Insert( table: String, cols: Seq[String], - values: Either[DqlStatement, Seq[Value[_]]] + values: Either[DqlStatement, Seq[Value[_]]], + onConflict: Option[OnConflict] = None ) extends DmlStatement { + lazy val conflictTarget: Option[Seq[String]] = onConflict.flatMap(_.target) + + lazy val doUpdate: Boolean = onConflict.exists(_.doUpdate) + override def sql: String = { values match { case Left(query) if cols.isEmpty => - s"INSERT INTO $table ${query.sql}" + s"INSERT INTO $table ${query.sql}${asString(onConflict)}" case Left(query) => - s"INSERT INTO $table (${cols.mkString(",")}) ${query.sql}" + s"INSERT INTO $table (${cols.mkString(",")}) ${query.sql}${asString(onConflict)}" case Right(vs) => val valuesSql = vs .map { @@ -292,11 +312,53 @@ package object query { } override def validate(): Either[String, Unit] = { + for { + _ <- values match { + case Left(query) => query.validate() + case Right(vs) if cols.size != vs.size => + Left(s"Number of columns (${cols.size}) does not match number of values (${vs.size})") + case _ => + Right(()) + } + _ <- conflictTarget match { + case Some(target) => + values match { + case Left(query: SingleSearch) => + val queryFields = + query.select.fields.map(f => f.fieldAlias.map(_.alias).getOrElse(f.sourceField)) + if (!target.forall(queryFields.contains)) + Left( + s"Conflict target columns (${target.mkString(",")}) must be part of the inserted columns from SELECT (${queryFields + .mkString(",")})" + ) + else Right(()) + case _ => + if (!target.forall(cols.contains)) + Left( + s"Conflict target columns (${target.mkString(",")}) must be part of the inserted columns (${cols + .mkString(",")})" + ) + else Right(()) + + } + case _ => Right(()) + } + } yield () + } + + def toJson: Option[JsonNode] = { values match { - case Right(vs) if cols.size != vs.size => - Left(s"Number of columns (${cols.size}) does not match number of values (${vs.size})") - case _ => - Right(()) + case Right(vs) if cols.size == vs.size => + val map: Map[String, Value[_]] = + cols + .zip(vs) + .map { case (k, v) => + k -> v + } + .toMap + val json: JsonNode = ObjectValue(map) + Some(json) + case _ => None } } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 43a71e4e..c1e63f65 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -398,7 +398,8 @@ package object schema { PrimaryKeyProcessor( sql = s"PRIMARY KEY (${primaryKey.mkString(", ")})", column = "_id", - value = primaryKey.toSet + value = primaryKey.toSet, + separator = sqlConfig.compositeKeySeparator ) ) } else { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index dd7365a8..2f399b51 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1483,12 +1483,25 @@ class ParserSpec extends AnyFlatSpec with Matchers { // --- DML --- it should "parse INSERT INTO ... VALUES" in { - val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice')" + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO NOTHING" val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case Insert("users", cols, Right(values)) => + case Insert("users", cols, Right(values), Some(OnConflict(None, false))) => + cols should contain inOrder ("id", "name") + values.map(_.value) should contain inOrder (1, "Alice") + case _ => fail("Expected Insert with values") + } + } + + it should "parse INSERT INTO ... VALUES with DO UPDATE" in { + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO UPDATE" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", cols, Right(values), Some(OnConflict(None, true))) => cols should contain inOrder ("id", "name") values.map(_.value) should contain inOrder (1, "Alice") case _ => fail("Expected Insert with values") @@ -1496,13 +1509,25 @@ class ParserSpec extends AnyFlatSpec with Matchers { } it should "parse INSERT INTO ... SELECT" in { - val sql = "INSERT INTO users SELECT id, name FROM old_users" + val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT DO NOTHING" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", Nil, Left(sel: DqlStatement), Some(OnConflict(None, false))) => + sel.sql should include("SELECT id, name FROM old_users ON CONFLICT DO NOTHING") + case _ => fail("Expected Insert with select") + } + } + + it should "parse INSERT INTO ... SELECT with ON CONFLICT" in { + val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE" val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get stmt match { - case Insert("users", Nil, Left(sel: DqlStatement)) => - sel.sql should include("SELECT id, name FROM old_users") + case Insert("users", Nil, Left(sel: DqlStatement), Some(OnConflict(Some(Seq("id")), true))) => + sel.sql should include("SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE") case _ => fail("Expected Insert with select") } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala index 7ba465d4..c594194c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala @@ -74,7 +74,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .futureValue @@ -100,7 +100,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -119,7 +119,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -141,7 +141,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -162,7 +162,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -199,7 +199,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val result = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -224,7 +224,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 1c6d7c8f..b9633cba 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -294,7 +294,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M ) shouldBe false implicit val bulkOptions: BulkOptions = BulkOptions("person_mapping") - val result = pClient.bulk[String](persons, identity, idKey = Some("uuid")).get + val result = pClient.bulk[String](persons, identity, idKey = Some(Set("uuid"))).get result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices @@ -376,7 +376,11 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M implicit val bulkOptions: BulkOptions = BulkOptions("person_migration") val result = pClient - .bulk[String](Source.fromIterator(() => persons.iterator), identity, idKey = Some("uuid")) + .bulk[String]( + Source.fromIterator(() => persons.iterator), + identity, + idKey = Some(Set("uuid")) + ) .get result.failedCount shouldBe 0 result.successCount shouldBe persons.size @@ -482,7 +486,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Bulk index valid json with an id key but no suffix key" should "work" in { implicit val bulkOptions: BulkOptions = BulkOptions("person2") - val result = pClient.bulk[String](persons, identity, idKey = Some("uuid")).get + val result = pClient.bulk[String](persons, identity, idKey = Some(Set("uuid"))).get result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices @@ -521,7 +525,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), suffixDateKey = Some("birthDate") ) .get @@ -574,7 +578,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -615,7 +619,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), suffixDateKey = Some("birthDate"), update = Some(true) ) @@ -661,7 +665,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -697,7 +701,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -754,7 +758,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -784,7 +788,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -996,7 +1000,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -1265,7 +1269,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -1405,7 +1409,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get result.failedCount shouldBe 0 @@ -1469,7 +1473,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get result.failedCount shouldBe 0 @@ -1533,7 +1537,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get result.failedCount shouldBe 0 @@ -1609,7 +1613,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get result.failedCount shouldBe 0 @@ -1707,7 +1711,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get @@ -1756,7 +1760,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .get @@ -1811,7 +1815,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M implicit val bulkOptions: BulkOptions = BulkOptions("person_update_json") pClient - .bulk[String](persons, identity, idKey = Some("uuid")) + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) .get blockUntilCount(3, "person_update_json") @@ -1862,7 +1866,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M implicit val bulkOptions: BulkOptions = BulkOptions("person_update_merge_pipeline") pClient - .bulk[String](persons, identity, idKey = Some("uuid")) + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) .get blockUntilCount(3, "person_update_merge_pipeline") @@ -1906,7 +1910,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M implicit val bulkOptions: BulkOptions = BulkOptions("person_update_closed_index") pClient - .bulk[String](persons, identity, idKey = Some("uuid")) + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) .get blockUntilCount(3, "person_update_closed_index") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala index a98d3cba..e4ce7e38 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala @@ -52,7 +52,7 @@ trait EmployeeData { _: Suite => implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = Source.fromIterator(() => list.iterator) - client.bulk[String](employees, identity, idKey = Some("id")) match { + client.bulk[String](employees, identity, idKey = Some(Set("id"))) match { case ElasticSuccess(response) => println(s"✅ Bulk indexing completed:") println(s" - Total items: ${response.metrics.totalDocuments}") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala new file mode 100644 index 00000000..667d5e75 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala @@ -0,0 +1,263 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk.BulkOptions +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.ExecutionContextExecutor +import scala.language.implicitConversions + +trait InsertByQuerySpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: ElasticClientApi + + // --------------------------------------------------------------------------- + // FIXTURES : MAPPINGS + // --------------------------------------------------------------------------- + + private val customersMapping: String = + """{ + |"properties":{ + | "customer_id":{"type":"keyword"}, + | "name":{"type":"text"}, + | "email":{"type":"keyword"}, + | "country":{"type":"keyword"} + |}, + |"_meta":{"primary_key":["customer_id"]} + |}""".stripMargin.replaceAll("\\s+", "") + + private val productsMapping: String = + """{ + |"properties":{ + | "sku":{"type":"keyword"}, + | "name":{"type":"text"}, + | "price":{"type":"double"}, + | "category":{"type":"keyword"} + |}, + |"_meta":{"primary_key":["sku"]} + |}""".stripMargin.replaceAll("\\s+", "") + + private val ordersMapping: String = + """{ + |"properties":{ + | "order_id":{"type":"keyword"}, + | "customer_id":{"type":"keyword"}, + | "order_date":{"type":"date"}, + | "total":{"type":"double"}, + | "items":{ + | "type":"nested", + | "properties":{ + | "sku":{"type":"keyword"}, + | "qty":{"type":"integer"} + | } + | } + |}, + |"_meta":{ + | "primary_key":["order_id","customer_id"], + | "partition_by":{"column":"order_date","granularity":"d"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + private val stagingOrdersMapping: String = + """{ + |"properties":{ + | "id":{"type":"keyword"}, + | "cust":{"type":"keyword"}, + | "date":{"type":"date"}, + | "amount":{"type":"double"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + private val stagingCustomersMapping: String = + """{ + |"properties":{ + | "id":{"type":"keyword"}, + | "fullname":{"type":"text"}, + | "email":{"type":"keyword"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + // --------------------------------------------------------------------------- + // FIXTURES : DATASETS JSON PURS + // --------------------------------------------------------------------------- + + private val customers: List[String] = List( + """{"customer_id":"C001","name":"Alice","email":"alice@example.com","country":"FR"}""", + """{"customer_id":"C002","name":"Bob","email":"bob@example.com","country":"US"}""", + """{"customer_id":"C003","name":"Charlie","email":"charlie@example.com","country":"DE"}""" + ) + + private val products: List[String] = List( + """{"sku":"SKU-001","name":"Laptop","price":1299.99,"category":"electronics"}""", + """{"sku":"SKU-002","name":"Mouse","price":29.99,"category":"electronics"}""", + """{"sku":"SKU-003","name":"Desk Chair","price":199.99,"category":"furniture"}""" + ) + + private val orders: List[String] = List( + """{"order_id":"O1001","customer_id":"C001","order_date":"2024-01-10","total":1299.99,"items":[{"sku":"SKU-001","qty":1}]}""", + """{"order_id":"O1002","customer_id":"C002","order_date":"2024-01-11","total":29.99,"items":[{"sku":"SKU-002","qty":1}]}""" + ) + + private val stagingOrders: List[String] = List( + """{"id":"O2001","cust":"C001","date":"2024-02-01","amount":1299.99}""", + """{"id":"O2002","cust":"C003","date":"2024-02-02","amount":199.99}""", + """{"id":"O2003","cust":"C002","date":"2024-02-03","amount":29.99}""" + ) + + private val stagingOrdersUpdates: List[String] = List( + """{"id":"O1001","cust":"C001","date":"2024-01-10","amount":1499.99}""", + """{"id":"O1002","cust":"C002","date":"2024-01-11","amount":39.99}""" + ) + + private val stagingCustomers: List[String] = List( + """{"id":"C010","fullname":"Bob Martin","email":"bob.martin@example.com"}""", + """{"id":"C011","fullname":"Jane Doe","email":"jane.doe@example.com"}""" + ) + + // --------------------------------------------------------------------------- + // FIXTURE : INITIALISATION DES INDEXES + // --------------------------------------------------------------------------- + + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + + private def initIndex(name: String, mapping: String, docs: List[String]): Unit = { + client.createIndex(name, mappings = Some(mapping)) + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = name) + client.bulk[String](docs, identity, Some(name)) + client.refresh(name) + } + + // --------------------------------------------------------------------------- + // TESTS D’INTÉGRATION COMPLETS + // --------------------------------------------------------------------------- + + behavior of "insertByQuery" + + it should "init all indices" in { + initIndex("customers", customersMapping, customers) + initIndex("products", productsMapping, products) + initIndex("orders", ordersMapping, orders) + initIndex("staging_orders", stagingOrdersMapping, stagingOrders) + initIndex("staging_orders_updates", stagingOrdersMapping, stagingOrdersUpdates) + initIndex("staging_customers", stagingCustomersMapping, stagingCustomers) + } + + it should "insert a customer with INSERT ... VALUES" in { + val sql = + """INSERT INTO customers (customer_id, name, email, country) + |VALUES ('C010', 'Bob', 'bob@example.com', 'FR')""".stripMargin + + val result = client.insertByQuery("customers", sql).futureValue + result shouldBe ElasticSuccess(1L) + } + + it should "upsert a product with ON CONFLICT DO UPDATE" in { + val sql = + """INSERT INTO products (sku, name, price, category) + |VALUES ('SKU-001', 'Laptop Pro', 1499.99, 'electronics') + |ON CONFLICT DO UPDATE""".stripMargin.replaceAll("\\s+", " ") + + val result = client.insertByQuery("products", sql).futureValue + result shouldBe ElasticSuccess(1L) + } + + it should "insert orders from a SELECT with alias mapping" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |AS SELECT + | id AS order_id, + | cust AS customer_id, + | date AS order_date, + | amount AS total + |FROM staging_orders""".stripMargin.replaceAll("\\s+", " ") + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe ElasticSuccess(3L) + } + + it should "upsert orders with composite PK using ON CONFLICT DO UPDATE" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |AS SELECT + | id AS order_id, + | cust AS customer_id, + | date AS order_date, + | amount AS total + |FROM staging_orders_updates + |ON CONFLICT (order_id, customer_id) DO UPDATE""".stripMargin + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe ElasticSuccess(2L) + } + + it should "fail when conflictTarget does not match PK" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |VALUES ('O1001', 'C001', '2024-01-10', 1299.99) + |ON CONFLICT (order_id) DO UPDATE""".stripMargin + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe a[ElasticFailure] + } + + it should "fail when SELECT does not provide all INSERT columns" in { + val sql = + """INSERT INTO customers (customer_id, name, email) + |AS SELECT id AS customer_id, fullname AS name + |FROM staging_customers""".stripMargin + + val result = client.insertByQuery("customers", sql).futureValue + result shouldBe a[ElasticFailure] + } + + it should "fail when ON CONFLICT DO UPDATE is used without PK or conflictTarget" in { + val sql = + """INSERT INTO staging_orders (id, cust, date, amount) + |VALUES ('X', 'Y', '2024-01-01', 10.0) + |ON CONFLICT DO UPDATE""".stripMargin + + val result = client.insertByQuery("staging_orders", sql).futureValue + result shouldBe a[ElasticFailure] + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala index 9164f64f..5753e451 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client import akka.actor.ActorSystem diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala index b16ed8d3..40d18793 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.client import akka.actor.ActorSystem From 88f54db27aace6d6d0020c13357c8d7731d859f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 29 Dec 2025 06:36:12 +0100 Subject: [PATCH 41/95] fix double refresh --- .../scala/app/softnetwork/elastic/client/IndicesApi.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 17181be5..39e48a52 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -1069,11 +1069,7 @@ trait IndicesApi extends ElasticClientHelpers { suffixDateKey = suffixKey, suffixDatePattern = suffixPattern, update = Some(parsed.doUpdate) - )(BulkOptions(index), system) - - // 7. Refresh - _ <- - if (refresh) this.refresh(index).toFuture else Future.successful(true) + )(BulkOptions(defaultIndex = index, disableRefresh = !refresh), system) } yield bulkResult.successCount.toLong From 4dc2d667917ea29e1d7cf10b536246bf406a6111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 31 Dec 2025 11:09:19 +0100 Subject: [PATCH 42/95] init SQL gateway --- .../softnetwork/elastic/client/AliasApi.scala | 169 ++- .../elastic/client/ElasticClientApi.scala | 1 + .../client/ElasticClientDelegator.scala | 102 +- .../elastic/client/IndicesApi.scala | 64 +- .../elastic/client/MappingApi.scala | 92 +- .../elastic/client/NopeClientApi.scala | 3 +- .../elastic/client/ScrollApi.scala | 40 +- .../elastic/client/SearchApi.scala | 84 +- .../elastic/client/SqlGateway.scala | 1078 +++++++++++++++++ .../client/metrics/MetricsElasticClient.scala | 52 +- .../elastic/client/result/package.scala | 32 + .../elastic/client/AliasApiSpec.scala | 55 +- .../elastic/client/jest/JestAliasApi.scala | 13 +- .../elastic/client/jest/JestMappingApi.scala | 7 +- .../client/rest/RestHighLevelClientApi.scala | 29 +- .../client/rest/RestHighLevelClientApi.scala | 29 +- .../elastic/client/java/JavaClientApi.scala | 23 +- .../elastic/client/java/JavaClientApi.scala | 23 +- .../softnetwork/elastic/schema/package.scala | 5 +- .../app/softnetwork/elastic/sql/package.scala | 30 +- .../elastic/sql/query/package.scala | 18 +- .../elastic/sql/schema/MappingsRules.scala | 38 + .../elastic/sql/schema/SettingsRules.scala | 43 + .../elastic/sql/schema/TableDiff.scala | 93 +- .../elastic/sql/schema/package.scala | 51 +- .../elastic/sql/type/SQLTypeUtils.scala | 61 + .../elastic/sql/parser/ParserSpec.scala | 16 +- .../elastic/client/ElasticClientSpec.scala | 4 +- .../elastic/client/InsertByQuerySpec.scala | 10 +- .../elastic/client/MockElasticClientApi.scala | 5 +- 30 files changed, 2024 insertions(+), 246 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala create mode 100644 sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala index f58a533b..3aad94bf 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala @@ -22,7 +22,9 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } -import com.google.gson.JsonParser +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode import scala.jdk.CollectionConverters._ import scala.util.Try @@ -60,7 +62,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * @param index * the index name * @param alias - * the alias name to add + * the alias to add * @return * ElasticSuccess(true) if added, ElasticFailure otherwise * @@ -78,7 +80,40 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * An index can have multiple aliases */ //format:on + @deprecated("Use addAlias(TableAlias) instead", "0.15.0") def addAlias(index: String, alias: String): ElasticResult[Boolean] = { + addAlias(TableAlias(index, alias)) + } + + //format:off + /** Add an alias to an index. + * + * This operation: + * 1. Validates the index and alias names 2. Checks that the index exists 3. Adds the alias + * + * @param alias + * the TableAlias to add + * @return + * ElasticSuccess(true) if added, ElasticFailure otherwise + * + * @example + * {{{ + * val alias = TableAlias(table = "my-index-2024", alias = "my-index-current") + * addAlias(alias) match { + * case ElasticSuccess(_) => println("Alias added") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + * + * @note + * An alias can point to multiple indexes (useful for searches) + * @note + * An index can have multiple aliases + */ + //format:on + def addAlias(alias: TableAlias): ElasticResult[Boolean] = { + val index = alias.table + val aliasName = alias.alias // Validation... validateIndexName(index) match { case Some(error) => @@ -92,7 +127,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case None => // OK } - validateAliasName(alias) match { + validateAliasName(aliasName) match { case Some(error) => return ElasticFailure( error.copy( @@ -104,7 +139,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case None => // OK } - if (index == alias) { + if (index == aliasName) { return ElasticFailure( ElasticError( message = s"Index and alias cannot have the same name: '$index'", @@ -115,7 +150,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => ) } - indexExists(index, false) match { + indexExists(index, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -129,15 +164,15 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case _ => // OK } - logger.debug(s"Adding alias '$alias' to index '$index'") + logger.debug(s"Adding alias '$aliasName' to index '$index'") - executeAddAlias(index, alias) match { + executeAddAlias(alias) match { case success @ ElasticSuccess(_) => - logger.info(s"✅ Alias '$alias' successfully added to index '$index'") + logger.info(s"✅ Alias '$aliasName' successfully added to index '$index'") success case failure @ ElasticFailure(error) => - logger.error(s"❌ Failed to add alias '$alias' to index '$index': ${error.message}") + logger.error(s"❌ Failed to add alias '$aliasName' to index '$index': ${error.message}") failure } } @@ -269,14 +304,14 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ //format:on - def getAliases(index: String): ElasticResult[Set[String]] = { + def getAliases(index: String): ElasticResult[Seq[TableAlias]] = { validateIndexName(index) match { case Some(error) => @@ -296,7 +331,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => // ✅ Extracting aliases from JSON ElasticResult.fromTry( Try { - JsonParser.parseString(jsonString).getAsJsonObject + mapper.readTree(jsonString) } ) match { case ElasticFailure(error) => @@ -305,32 +340,39 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case ElasticSuccess(rootObj) => if (!rootObj.has(index)) { logger.warn(s"Index '$index' not found in response") - return ElasticResult.success(Set.empty[String]) + return ElasticResult.success(Seq.empty[TableAlias]) } - val indexObj = rootObj.getAsJsonObject(index) - if (indexObj == null) { - logger.warn(s"Index '$index' is null in response") - return ElasticResult.success(Set.empty[String]) - } + val root = rootObj.get(index) - val aliasesObj = indexObj.getAsJsonObject("aliases") - if (aliasesObj == null || aliasesObj.size() == 0) { + if (!root.has("aliases")) { logger.debug(s"No aliases found for index '$index'") - ElasticResult.success(Set.empty[String]) - } else { - val aliases = aliasesObj.entrySet().asScala.map(_.getKey).toSet - logger.debug( - s"Found ${aliases.size} alias(es) for index '$index': ${aliases.mkString(", ")}" - ) - ElasticResult.success(aliases) + return ElasticResult.success(Seq.empty[TableAlias]) } + + val aliasesNode = root.path("aliases") + val aliases: Map[String, JsonNode] = + if (aliasesNode != null && aliasesNode.isObject) { + aliasesNode + .properties() + .asScala + .map { entry => + val aliasName = entry.getKey + val aliasValue = entry.getValue + aliasName -> aliasValue + } + .toMap + } else { + Map.empty + } + + ElasticSuccess(aliases.toSeq.map(alias => TableAlias(index, alias._1, alias._2))) } } match { case success @ ElasticSuccess(aliases) => if (aliases.nonEmpty) logger.info( - s"✅ Found ${aliases.size} alias(es) for index '$index': ${aliases.mkString(", ")}" + s"✅ Found ${aliases.size} alias(es) for index '$index': ${aliases.map(_.alias).sorted.mkString(", ")}" ) else logger.info(s"✅ No aliases found for index '$index'") @@ -441,11 +483,80 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => } } + //format:off + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * + * @example + * {{{ + * setAliases("my-index", Seq(TableAlias("my-index", "alias1"), TableAlias("my-index", "alias2"))) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + //format:on + def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = { + getAliases(index).flatMap { existingAliases => + val notIndexAliases = aliases.filter(_.table != index) + if (notIndexAliases.nonEmpty) { + return ElasticFailure( + ElasticError( + message = s"All aliases must belong to the index '$index': ${notIndexAliases + .map(_.alias) + .mkString(", ")}", + cause = None, + statusCode = Some(400), + operation = Some("setAliases") + ) + ) + } + + val existingAliasNames = existingAliases.map(_.alias).toSet + val aliasesMap = aliases.map(alias => alias.alias -> alias).toMap + + val toSet = aliasesMap.keys.toSet.diff(existingAliasNames) ++ aliasesMap.keys.toSet.intersect( + existingAliasNames + ) + val toRemove = existingAliasNames.diff(aliasesMap.keys.toSet) + + val setResults = toSet.map(alias => addAlias(aliasesMap(alias))) + val removeResults = toRemove.map(alias => removeAlias(index, alias)) + + val allResults = setResults ++ removeResults + + val failures = allResults.collect { case ElasticFailure(error) => error } + + if (failures.nonEmpty) { + ElasticFailure( + ElasticError( + message = + s"Failed to set aliases for index '$index': ${failures.map(_.message).mkString(", ")}", + cause = None, + statusCode = Some(500), + operation = Some("setAliases") + ) + ) + } else { + ElasticResult.success(true) + } + } + } + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== - private[client] def executeAddAlias(index: String, alias: String): ElasticResult[Boolean] + private[client] def executeAddAlias(alias: TableAlias): ElasticResult[Boolean] private[client] def executeRemoveAlias(index: String, alias: String): ElasticResult[Boolean] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index c2aad4f5..13b9f2c1 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -49,6 +49,7 @@ trait ElasticClientApi with SerializationApi with PipelineApi with TemplateApi + with SqlGateway with ClientCompanion { protected def logger: Logger diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 56481870..0f597ba7 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -30,7 +30,7 @@ import app.softnetwork.elastic.sql.query.{ SelectStatement, SingleSearch } -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -226,9 +226,19 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { */ override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit system: ActorSystem - ): Future[ElasticResult[Long]] = + ): Future[ElasticResult[DmlResult]] = delegate.insertByQuery(index, query, refresh) + /** Load the schema for the provided index. + * + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + override def loadSchema(index: String): ElasticResult[Schema] = + delegate.loadSchema(index) + override private[client] def executeCreateIndex( index: String, settings: String, @@ -313,6 +323,31 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def addAlias(index: String, alias: String): ElasticResult[Boolean] = delegate.addAlias(index, alias) + /** Add an alias to an index. + * + * This operation: + * 1. Validates the index and alias names 2. Checks that the index exists 3. Adds the alias + * + * @param alias + * the TableAlias to add + * @return + * ElasticSuccess(true) if added, ElasticFailure otherwise + * @example + * {{{ + * val alias = TableAlias(table = "my-index-2024", alias = "my-index-current") + * addAlias(alias) match { + * case ElasticSuccess(_) => println("Alias added") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + * @note + * An alias can point to multiple indexes (useful for searches) + * @note + * An index can have multiple aliases + */ + override def addAlias(alias: TableAlias): ElasticResult[Boolean] = + delegate.addAlias(alias) + /** Remove an alias from an index. * * @param index @@ -362,13 +397,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ - override def getAliases(index: String): ElasticResult[Set[String]] = + override def getAliases(index: String): ElasticResult[Seq[TableAlias]] = delegate.getAliases(index) /** Atomic swap of an alias between two indexes. @@ -403,11 +438,32 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.swapAlias(oldIndex, newIndex, alias) + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * @example + * {{{ + * setAliases("my-index", Set("alias1", "alias2")) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + override def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = + delegate.setAliases(index, aliases) + override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = - delegate.executeAddAlias(index, alias) + delegate.executeAddAlias(alias) override private[client] def executeRemoveAlias( index: String, @@ -554,6 +610,31 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.updateMapping(index, mapping, settings) + /** Migrate an existing index to a new mapping. + * + * Process: + * 1. Create temporary index with new mapping 2. Reindex data from original to temporary 3. + * Delete original index 4. Recreate original index with new mapping 5. Reindex data from + * temporary to original 6. Delete temporary index + */ + override private[client] def performMigration( + index: String, + tempIndex: String, + mapping: String, + settings: String, + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = + delegate.performMigration(index, tempIndex, mapping, settings, aliases) + + override private[client] def rollbackMigration( + index: String, + tempIndex: String, + originalMapping: String, + originalSettings: String, + originalAliases: Seq[TableAlias] + ): ElasticResult[Boolean] = + delegate.rollbackMigration(index, tempIndex, originalMapping, originalSettings, originalAliases) + override private[client] def executeSetMapping( index: String, mapping: String @@ -1014,7 +1095,8 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * the Elasticsearch response */ - override def search(sql: SelectStatement): ElasticResult[ElasticResponse] = delegate.search(sql) + override def search(statement: DqlStatement): ElasticResult[ElasticResponse] = + delegate.search(statement) /** Search for documents / aggregations matching the Elasticsearch query. * @@ -1059,9 +1141,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a Future containing the Elasticsearch response */ - override def searchAsync(sqlQuery: SelectStatement)(implicit + override def searchAsync(statement: DqlStatement)(implicit ec: ExecutionContext - ): Future[ElasticResult[ElasticResponse]] = delegate.searchAsync(sqlQuery) + ): Future[ElasticResult[ElasticResponse]] = delegate.searchAsync(statement) /** Asynchronous search for documents / aggregations matching the Elasticsearch query. * diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 39e48a52..13522b18 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{Delete, From, Insert, SingleSearch, Table, Update} -import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, TableAlias} +import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, Schema, TableAlias} import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.ObjectNode @@ -221,6 +221,30 @@ trait IndicesApi extends ElasticClientHelpers { } } + /** Load the schema for the provided index. + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + def loadSchema(index: String): ElasticResult[Schema] = { + getIndex(index) match { + case ElasticSuccess(Some(idx)) => + ElasticSuccess(idx.schema) + case ElasticSuccess(None) => + logger.info(s"Index '$index' not found for schema loading") + ElasticFailure( + ElasticError.notFound( + index = index, + operation = "loadSchema" + ) + ) + case ElasticFailure(error) => + logger.error(s"❌ Failed to load schema for index '$index': ${error.message}") + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } + } + /** Get an index with the provided name. * @param index * - the name of the index to get @@ -858,7 +882,7 @@ trait IndicesApi extends ElasticClientHelpers { index: String, query: String, refresh: Boolean = true - )(implicit system: ActorSystem): Future[ElasticResult[Long]] = { + )(implicit system: ActorSystem): Future[ElasticResult[DmlResult]] = { implicit val ec: ExecutionContext = system.dispatcher val result = for { @@ -882,7 +906,7 @@ trait IndicesApi extends ElasticClientHelpers { // 3. Load index metadata idx <- Future.fromTry(getIndex(index).toEither.flatMap { - case Some(i) => Right(i.asTable) + case Some(i) => Right(i.schema) case None => Left(ElasticError.notFound(index, "insertByQuery")) }.toTry) @@ -1071,25 +1095,27 @@ trait IndicesApi extends ElasticClientHelpers { update = Some(parsed.doUpdate) )(BulkOptions(defaultIndex = index, disableRefresh = !refresh), system) - } yield bulkResult.successCount.toLong + } yield bulkResult - result.map(ElasticSuccess(_)).recover { - case e: ElasticError => - ElasticFailure( - e.copy( - operation = Some("insertByQuery"), - index = Some(index) + result + .map(r => ElasticSuccess(DmlResult(inserted = r.successCount, rejected = r.failedCount))) + .recover { + case e: ElasticError => + ElasticFailure( + e.copy( + operation = Some("insertByQuery"), + index = Some(index) + ) ) - ) - case e => - ElasticFailure( - ElasticError( - message = e.getMessage, - operation = Some("insertByQuery"), - index = Some(index) + case e => + ElasticFailure( + ElasticError( + message = e.getMessage, + operation = Some("insertByQuery"), + index = Some(index) + ) ) - ) - } + } } private def parseQueryForUpdate( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index 689aac60..088d0bb0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -22,13 +22,16 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.schema.TableAlias +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import java.util.UUID /** Mapping management API. */ -trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi with RefreshApi => +trait MappingApi extends ElasticClientHelpers { + _: SettingsApi with IndicesApi with RefreshApi with VersionApi with AliasApi => // ======================================================================== // PUBLIC METHODS @@ -69,9 +72,51 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w case None => // continue } - logger.debug(s"Setting mapping for index '$index': $mapping") + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + val updatedMapping: String = + if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion)) { + val root = mapper.readTree(mapping).asInstanceOf[ObjectNode] + + if (root.has("properties") || root.has("_meta")) { + logger.info(s"Wrapping mappings with '_doc' type for ES version $elasticVersion") + + val doc = mapper.createObjectNode() - executeSetMapping(index, mapping) match { + // Move properties + if (root.has("properties")) { + doc.set("properties", root.get("properties")) + root.remove("properties") + } + + // Move _meta + if (root.has("_meta")) { + doc.set("_meta", root.get("_meta")) + root.remove("_meta") + } + + // Wrap into _doc + root.set("_doc", doc) + + root.toString + } else { + mapping + } + } else { + mapping + } + + logger.debug(s"Setting mapping for index '$index': $updatedMapping") + + executeSetMapping(index, updatedMapping) match { case success @ ElasticSuccess(true) => logger.info(s"✅ Mapping for index '$index' updated successfully") success @@ -155,7 +200,7 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w mapping: String, settings: String = defaultSettings ): ElasticResult[Boolean] = { - indexExists(index, false).flatMap { + indexExists(index, pattern = false).flatMap { case false => // Scenario 1: Index doesn't exist createIndex(index, settings, Some(mapping), Nil).flatMap { @@ -201,17 +246,19 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w val backupResult = for { originalMapping <- getMapping(index) originalSettings <- loadSettings(index) - } yield (originalMapping, originalSettings) + originalAliases <- getAliases(index) + } yield (originalMapping, originalSettings, originalAliases) backupResult match { - case ElasticSuccess((origMapping, origSettings)) => + case ElasticSuccess((origMapping, origSettings, origAliases)) => logger.info(s"✅ Backed up original mapping and settings for '$index'") val migrationResult = performMigration( index, tempIndex, newMapping, - settings + settings, + Nil ) migrationResult match { @@ -246,23 +293,21 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w * Delete original index 4. Recreate original index with new mapping 5. Reindex data from * temporary to original 6. Delete temporary index */ - private def performMigration( + private[client] def performMigration( index: String, tempIndex: String, mapping: String, - settings: String + settings: String, + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { logger.info(s"Starting migration: $index -> $tempIndex") for { // Create temp index - _ <- createIndex(tempIndex, settings, None, Nil) + _ <- createIndex(tempIndex, settings, Some(mapping), aliases) .filter(_ == true, s"❌ Failed to create temp index '$tempIndex'") - _ <- setMapping(tempIndex, mapping) - .filter(_ == true, s"❌ Failed to set mapping on temp index") - // Reindex to temp _ <- reindex(index, tempIndex, refresh = true) .filter(_._1 == true, s"❌ Failed to reindex to temp") @@ -271,13 +316,10 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w _ <- deleteIndex(index) .filter(_ == true, s"❌ Failed to delete original index") - // Recreate original with new mapping - _ <- createIndex(index, settings, None, Nil) + // Recreate original with new settings, mapping and aliases + _ <- createIndex(index, settings, Some(mapping), Nil) .filter(_ == true, s"❌ Failed to recreate original index") - _ <- setMapping(index, mapping) - .filter(_ == true, s"❌ Failed to set new mapping") - // Reindex back from temp _ <- reindex(tempIndex, index, refresh = true) .filter(_._1 == true, s"❌ Failed to reindex from temp") @@ -294,32 +336,30 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w } } - private def rollbackMigration( + private[client] def rollbackMigration( index: String, tempIndex: String, originalMapping: String, - originalSettings: String + originalSettings: String, + originalAliases: Seq[TableAlias] = Nil ): ElasticResult[Boolean] = { logger.warn(s"Rolling back migration for '$index'") for { // Check if temp index exists and has data - tempExists <- indexExists(tempIndex, false) + tempExists <- indexExists(tempIndex, pattern = false) // Delete current (potentially corrupted) index if it exists - _ <- indexExists(index, false).flatMap { + _ <- indexExists(index, pattern = false).flatMap { case true => deleteIndex(index) case false => ElasticResult.success(true) } // Recreate with original settings and mapping - _ <- createIndex(index, originalSettings, None, Nil) + _ <- createIndex(index, originalSettings, Some(originalMapping), originalAliases) .filter(_ == true, s"❌ Rollback: Failed to recreate index") - _ <- setMapping(index, originalMapping) - .filter(_ == true, s"❌ Rollback: Failed to restore mapping") - // If temp exists, reindex from it _ <- if (tempExists) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala index 2a42e26c..d74cf353 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -38,8 +38,7 @@ import scala.language.implicitConversions trait NopeClientApi extends ElasticClientApi { override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = ElasticResult.success(false) override private[client] def executeRemoveAlias( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index efcc43f9..26089bc6 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -19,7 +19,12 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.{Sink, Source} -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} import app.softnetwork.elastic.client.scroll.{ ScrollConfig, ScrollMetrics, @@ -128,38 +133,25 @@ trait ScrollApi extends ElasticClientHelpers { )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { implicit def timestamp: Long = System.currentTimeMillis() statement match { + // Select statement case select: SelectStatement => select.statement match { case Some(single: SingleSearch) => - if (single.windowFunctions.nonEmpty) - return scrollWithWindowEnrichment(select.score, single, config) + scroll(single.copy(score = select.score), config) - val sqlRequest = single.copy(score = select.score) - val elasticQuery = - ElasticQuery(sqlRequest, collection.immutable.Seq(sqlRequest.sources: _*)) - scrollWithMetrics( - elasticQuery, - sqlRequest.fieldAliases, - sqlRequest.sqlAggregations, - config, - single.sorts.nonEmpty - ) - - case Some(_) => - Source.failed( - new UnsupportedOperationException( - "Scrolling is not supported for multi-search queries" - ) - ) + case Some(multiple: MultiSearch) => + scroll(multiple, config) case None => Source.failed( new IllegalArgumentException("SQL query does not contain a valid search request") ) } + + // Single search case single: SingleSearch => if (single.windowFunctions.nonEmpty) - return scrollWithWindowEnrichment(None, single, config) + return scrollWithWindowEnrichment(single, config) val elasticQuery = ElasticQuery(single, collection.immutable.Seq(single.sources: _*)) @@ -170,10 +162,13 @@ trait ScrollApi extends ElasticClientHelpers { config, single.sorts.nonEmpty ) + + // Multi search case _: MultiSearch => Source.failed( new UnsupportedOperationException("Scrolling is not supported for multi-search queries") ) + case _ => Source.failed( new IllegalArgumentException("Scrolling is only supported for SELECT statements") @@ -409,7 +404,6 @@ trait ScrollApi extends ElasticClientHelpers { /** Scroll with window function enrichment */ private def scrollWithWindowEnrichment( - score: Option[Double], request: SingleSearch, config: ScrollConfig )(implicit @@ -426,7 +420,7 @@ trait ScrollApi extends ElasticClientHelpers { Future(executeWindowAggregations(request)) // Create base query without window functions - val baseQuery = createBaseQuery(score, request) + val baseQuery = createBaseQuery(request) // Stream and enrich Source diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 98c36534..14aa804f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.client.result.{ } import app.softnetwork.elastic.sql.macros.SQLQueryMacros import app.softnetwork.elastic.sql.query.{ + DqlStatement, MultiSearch, SQLAggregation, SelectStatement, @@ -60,26 +61,44 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Search for documents / aggregations matching the SQL query. * - * @param sql + * @param statement * the SQL query to execute * @return * the Elasticsearch response */ - def search(sql: SelectStatement): ElasticResult[ElasticResponse] = { + def search(statement: DqlStatement): ElasticResult[ElasticResponse] = { implicit def timestamp: Long = System.currentTimeMillis() - sql.statement match { - case Some(single: SingleSearch) => + val query = statement.sql + statement match { + case select: SelectStatement => + select.statement match { + case Some(statement: SingleSearch) => + search(statement.copy(score = select.score)) + case Some(statement: MultiSearch) => + search(statement) + case None => + logger.error( + s"❌ Failed to execute search for query \n${statement.sql}" + ) + ElasticResult.failure( + ElasticError( + message = s"SQL query does not contain a valid search request\n$query", + operation = Some("search") + ) + ) + } + case single: SingleSearch => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*), - sql = Some(sql.query) + sql = Some(query) ) if (single.windowFunctions.exists(_.isWindowing) && single.groupBy.isEmpty) - searchWithWindowEnrichment(sql.score, single) + searchWithWindowEnrichment(single) else singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(multiple: MultiSearch) => + case multiple: MultiSearch => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -87,17 +106,17 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { collection.immutable.Seq(query.sources: _*) ) }.toList, - sql = Some(sql.query) + sql = Some(query) ) multiSearch(elasticQueries, multiple.fieldAliases, multiple.sqlAggregations) - case None => + case _ => logger.error( - s"❌ Failed to execute search for query \n${sql.query}" + s"❌ Failed to execute search for query \n${statement.sql}" ) ElasticResult.failure( ElasticError( - message = s"SQL query does not contain a valid search request\n${sql.query}", + message = s"SQL query does not contain a valid search request\n$query", operation = Some("search") ) ) @@ -301,20 +320,40 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * a Future containing the Elasticsearch response */ def searchAsync( - sqlQuery: SelectStatement + statement: DqlStatement )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { implicit def timestamp: Long = System.currentTimeMillis() - sqlQuery.statement match { - case Some(single: SingleSearch) => + statement match { + case select: SelectStatement => + select.statement match { + case Some(statement: SingleSearch) => + searchAsync(statement.copy(score = select.score)) + case Some(statement: MultiSearch) => + searchAsync(statement) + case None => + logger.error( + s"❌ Failed to execute asynchronous search for query '${statement.sql}'" + ) + Future.successful( + ElasticResult.failure( + ElasticError( + message = s"SQL query does not contain a valid search request: ${statement.sql}", + operation = Some("searchAsync") + ) + ) + ) + } + + case single: SingleSearch => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*) ) singleSearchAsync(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(multiple: MultiSearch) => + case multiple: MultiSearch => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -325,14 +364,15 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { ) multiSearchAsync(elasticQueries, multiple.fieldAliases, multiple.sqlAggregations) - case None => + case _ => + val query = statement.sql logger.error( - s"❌ Failed to execute asynchronous search for query '${sqlQuery.query}'" + s"❌ Failed to execute asynchronous search for query '$query'" ) Future.successful( ElasticResult.failure( ElasticError( - message = s"SQL query does not contain a valid search request: ${sqlQuery.query}", + message = s"SQL query does not contain a valid search request: $query", operation = Some("searchAsync") ) ) @@ -1105,7 +1145,6 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * functions) 3. Enrich results with window values */ private def searchWithWindowEnrichment( - score: Option[Double], request: SingleSearch )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { @@ -1116,7 +1155,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { windowCache <- executeWindowAggregations(request) // Step 2: Execute base query (without window functions) - baseResponse <- executeBaseQuery(score, request) + baseResponse <- executeBaseQuery(request) // Step 3: Enrich results enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) @@ -1219,11 +1258,10 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Execute base query without window functions */ private def executeBaseQuery( - score: Option[Double], request: SingleSearch )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { - val baseQuery = createBaseQuery(score, request) + val baseQuery = createBaseQuery(request) logger.info(s"🔍 Executing base query without window functions ${baseQuery.sql}") @@ -1241,7 +1279,6 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Create base query by removing window functions from SELECT */ protected def createBaseQuery( - score: Option[Double], request: SingleSearch ): SingleSearch = { @@ -1253,7 +1290,6 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { .copy( select = request.select.copy(fields = baseFields) ) - .copy(score = score) .update() baseRequest diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala new file mode 100644 index 00000000..6ebeb4f1 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala @@ -0,0 +1,1078 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ + DdlResult, + DmlResult, + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess, + QueryResult, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + AlterTable, + CreateTable, + DdlStatement, + Delete, + DmlStatement, + DqlStatement, + DropTable, + Insert, + MultiSearch, + PipelineStatement, + SelectStatement, + SingleSearch, + Statement, + TableStatement, + TruncateTable, + Update +} +import app.softnetwork.elastic.sql.schema.{ + ColumnOptionSet, + Impossible, + IngestPipeline, + IngestPipelineType, + PipelineDiff, + Safe, + Table, + TableDiff, + UnsafeReindex +} +import app.softnetwork.elastic.sql.serialization._ +import org.slf4j.Logger + +import scala.concurrent.{ExecutionContext, Future} + +trait Executor[T <: Statement] { + def execute(statement: T)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] +} + +class DqlExecutor(api: ScrollApi with SearchApi, logger: Logger) extends Executor[DqlStatement] { + + override def execute( + statement: DqlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + + // ============================ + // SELECT ... (AST SQL) + // ============================ + case select: SelectStatement => + select.statement match { + + case Some(single: SingleSearch) => + execute(single.copy(score = select.score)) + + case Some(multiple: MultiSearch) => + execute(multiple) + + case None => + val error = ElasticError( + message = s"SELECT statement could not be translated into a search query: $statement", + statusCode = Some(400), + operation = Some("dql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + + // ============================ + // SingleSearch → SCROLL + // ============================ + case single: SingleSearch => + logger.info(s"▶ Executing scroll search on index ${single.from.tables.mkString(",")}") + Future.successful( + ElasticSuccess(QueryStream(api.scroll(single))) + ) + + // ============================ + // MultiSearch → searchAsync + // ============================ + case multiple: MultiSearch => + logger.info(s"▶ Executing multi-search on ${multiple.requests.size} indices") + api.searchAsync(multiple).map { + case ElasticSuccess(results) => + logger.info(s"✅ Multi-search returned ${results.results.size} hits.") + ElasticSuccess(QueryStructured(results)) + + case ElasticFailure(err) => + ElasticFailure(err.copy(operation = Some("dql"))) + } + + // ============================ + // Unsupported DQL + // ============================ + case _ => + val error = ElasticError( + message = s"Unsupported DQL statement: $statement", + statusCode = Some(400), + operation = Some("dql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } +} + +class DmlExecutor(api: IndicesApi, logger: Logger) extends Executor[DmlStatement] { + override def execute( + statement: DmlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + statement match { + case delete: Delete => + api.deleteByQuery(delete.table.name, delete.sql) match { + case ElasticSuccess(count) => + logger.info(s"✅ Deleted $count documents from ${delete.table.name}.") + Future.successful(ElasticResult.success(DmlResult(deleted = count))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case update: Update => + api.updateByQuery(update.table, update.sql) match { + case ElasticSuccess(count) => + logger.info(s"✅ Updated $count documents in ${update.table}.") + Future.successful(ElasticResult.success(DmlResult(updated = count))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case insert: Insert => + api.insertByQuery(insert.table, insert.sql).map { + case ElasticSuccess(res) => + logger.info(s"✅ Inserted ${res.inserted} documents into ${insert.table}.") + ElasticResult.success(res) + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } + case _ => + // unsupported DML statement + val error = + ElasticError( + message = s"Unsupported DML statement: $statement", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } +} + +trait DdlExecutor[T <: DdlStatement] extends Executor[T] + +class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[PipelineStatement] { + override def execute( + statement: PipelineStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // handle PIPELINE statement + api.pipeline(statement) match { + case ElasticSuccess(result) => + logger.info(s"✅ Executed pipeline statement: $statement.") + Future.successful(ElasticResult.success(DdlResult(result))) + case ElasticFailure(elasticError) => + logger.error(s"❌ Error executing pipeline statement: $statement. ${elasticError.message}") + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + } +} + +class TableExecutor( + api: IndicesApi with PipelineApi with TemplateApi with SettingsApi with MappingApi with AliasApi, + logger: Logger +) extends DdlExecutor[TableStatement] { + override def execute( + statement: TableStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // handle TABLE statement + statement match { + // handle CREATE TABLE statement + case create: CreateTable => + val ifNotExists = create.ifNotExists + val orReplace = create.orReplace + // will create template if partitioned + val partitioned = create.partitioned + val indexName = create.table + val single: Option[SingleSearch] = create.ddl match { + case Left(dql) => + dql match { + case s: SingleSearch => Some(s) + case _ => + val error = + ElasticError( + message = s"Only single search DQL supported in CREATE TABLE.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case Right(_) => None + } + + // check if index exists + api.indexExists(indexName, pattern = false) match { + // 1) Index exists + IF NOT EXISTS → skip creation, DdlResult(false) + case ElasticSuccess(true) if ifNotExists => + // skip creation + logger.info(s"⚠️ Index $indexName already exists, skipping creation.") + Future.successful(ElasticSuccess(DdlResult(false))) + + // 2) Index exists + no OR REPLACE specified → error + case ElasticSuccess(true) if !orReplace => + val error = + ElasticError( + message = s"Index $indexName already exists.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + + // 3) Index exists + OR REPLACE → remplacement + case ElasticSuccess(true) => + // proceed with replacement + logger.info(s"♻️ Replacing index $indexName.") + replaceExistingIndex(indexName, create, partitioned, single) + + // 4) Index not exists → creation + case ElasticSuccess(false) => + // proceed with creation + logger.info(s"✅ Creating index $indexName.") + createNonExistentIndex(indexName, create, partitioned, single) + + // 5) Error on indexExists + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + + // handle ALTER TABLE statement + case alter: AlterTable => + val ifExists = alter.ifExists + val indexName = alter.table + + // check if index exists + api.indexExists(indexName, pattern = false) match { + // 1) Index does not exist + IF EXISTS → skip alteration, DdlResult(false) + case ElasticSuccess(false) if ifExists => + // skip alteration + logger.info(s"⚠️ Index $indexName does not exist, skipping alteration.") + Future.successful(ElasticSuccess(DdlResult(false))) + + // 2) Index does not exists + no IF EXISTS → error + case ElasticSuccess(false) => + val error = + ElasticError( + message = s"Index $indexName does not exist.", + statusCode = Some(404), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + + // 3) Index exists → alteration + case ElasticSuccess(true) => + // proceed with alteration + logger.info(s"♻️ Alter index $indexName.") + alterExistingIndex(indexName, alter) + + // 5) Error on indexExists + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + + // handle TRUNCATE TABLE statement + case truncate: TruncateTable => + // handle TRUNCATE TABLE statement + val indexName = truncate.table + api.truncateIndex(indexName) match { + case ElasticSuccess(count) => + // index truncated successfully + logger.info(s"✅ Index $indexName truncated successfully ($count documents deleted).") + Future.successful(ElasticResult.success(DdlResult(true))) + case ElasticFailure(elasticError) => + // error truncating index + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("truncate")) + ) + ) + } + + // handle DROP TABLE statement + case drop: DropTable => + // handle DROP TABLE statement + val indexName = drop.table + // check if index exists - pattern indices not supported for drop + api.indexExists(indexName, pattern = false) match { + case ElasticSuccess(false) if drop.ifExists => + // index does not exist and IF EXISTS specified, skip deletion + logger.info(s"⚠️ Index $indexName does not exist, skipping deletion.") + Future.successful(ElasticResult.success(DdlResult(false))) + case ElasticSuccess(false) if !drop.ifExists => + // index does not exist and no IF EXISTS specified, return error + val error = + ElasticError( + message = s"Index $indexName does not exist.", + statusCode = Some(404), + operation = Some("drop") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + case ElasticSuccess(true) => + // proceed with deletion + api.deleteIndex(indexName) match { + case ElasticSuccess(true) => + // index deleted successfully + logger.info(s"✅ Index $indexName deleted successfully.") + Future.successful(ElasticResult.success(DdlResult(true))) + case ElasticSuccess(false) => + // index deletion failed + logger.warn(s"⚠️ Index $indexName could not be deleted.") + Future.successful(ElasticResult.success(DdlResult(false))) + case ElasticFailure(elasticError) => + // error deleting index + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("drop")) + ) + ) + } + case ElasticFailure(elasticError) => + // error checking index existence + Future.successful(ElasticFailure(elasticError.copy(operation = Some("drop")))) + } + + // handle unsupported table ddl statement + case _ => + // unsupported table ddl statement + val error = + ElasticError( + message = s"Unsupported table DDL statement: $statement", + statusCode = Some(400), + operation = Some("table") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + + /* Alter existing index + */ + private def alterExistingIndex( + indexName: String, + alter: AlterTable + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + // index exists and REPLACE specified, proceed with replacement + logger.info(s"♻️ Altering index $indexName.") + // load existing index schema + api.loadSchema(indexName) match { + case ElasticSuccess(schema) => + logger.info( + s"🔄 Merging existing index $indexName DDL with new DDL." + ) + var updatedTable: Table = schema.merge(alter.statements) + + // load default pipeline diff if needed + val defaultPipelineDiff: Option[List[PipelineDiff]] = + loadTablePipelineDiff( + updatedTable, + IngestPipelineType.Default, + updatedTable.defaultPipelineName, + schema.defaultPipelineName + ) match { + case ElasticSuccess(result) => + result + case ElasticFailure(elasticError) => + return Future.successful(ElasticFailure(elasticError)) + } + + // load final pipeline diff if needed + val finalPipelineDiff: Option[List[PipelineDiff]] = + loadTablePipelineDiff( + updatedTable, + IngestPipelineType.Final, + updatedTable.finalPipelineName, + schema.finalPipelineName + ) match { + case ElasticSuccess(result) => + result + case ElasticFailure(elasticError) => + return Future.successful(ElasticFailure(elasticError)) + } + + // compute diff + var diff = schema.diff(updatedTable) + + // handle pipeline diffs + defaultPipelineDiff match { + case Some(pipelineDiffs) => + diff = diff.copy(pipeline = + diff.pipeline.filterNot(_.pipelineType == IngestPipelineType.Default) ++ pipelineDiffs + ) + if (pipelineDiffs.nonEmpty) { + logger.warn( + s"🔄 Default ingesting pipeline for index $indexName has changes: $pipelineDiffs" + ) + logger.warn( + s"""⚠️ Default Pipeline may be used by multiple indexes. + | Modifying it may impact other indexes. + | Consider creating a dedicated pipeline if isolation is required.""".stripMargin + ) + } + case _ => + } + finalPipelineDiff match { + case Some(pipelineDiffs) => + diff = diff.copy(pipeline = + diff.pipeline.filterNot(_.pipelineType == IngestPipelineType.Final) ++ pipelineDiffs + ) + if (pipelineDiffs.nonEmpty) { + logger.warn( + s"🔄 Final ingesting pipeline for index $indexName has changes: $pipelineDiffs" + ) + logger.warn( + s"""⚠️ Final Pipeline may be used by multiple indexes. + | Modifying it may impact other indexes. + | Consider creating a dedicated pipeline if isolation is required.""".stripMargin + ) + } + case _ => + } + + if (diff.isEmpty) { + logger.info( + s"⚠️ No changes detected for index $indexName, skipping update." + ) + Future.successful(ElasticResult.success(DdlResult(false))) + } else { + logger.info(s"🔄 Updating index $indexName with changes: $diff") + diff.safety match { + + // ------------------------------------------------------------ + // SAFE → mise à jour en place + // ------------------------------------------------------------ + case Safe => + logger.info(s"🔧 Applying SAFE ALTER TABLE changes on $indexName.") + applyUpdatesInPlace(indexName, updatedTable, diff) match { + case Right(result) => Future.successful(ElasticSuccess(DdlResult(result))) + case Left(error) => + Future.successful(ElasticFailure(error.copy(operation = Some("schema")))) + } + + // ------------------------------------------------------------ + // UNSAFE → reindex + // ------------------------------------------------------------ + case UnsafeReindex => + logger.warn(s"⚠️ ALTER TABLE requires REINDEX for $indexName.") + migrateToNewSchema(indexName, schema, updatedTable, diff) match { + case Right(result) => Future.successful(ElasticSuccess(DdlResult(result))) + case Left(error) => + Future.successful(ElasticFailure(error.copy(operation = Some("schema")))) + } + + // ------------------------------------------------------------ + // IMPOSSIBLE + // ------------------------------------------------------------ + case Impossible => + val error = ElasticError( + message = s"ALTER TABLE cannot be applied to index $indexName: $diff", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + + case ElasticFailure(elasticError) => + // error retrieving existing index info + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + } + + private def replaceExistingIndex( + indexName: String, + create: CreateTable, + partitioned: Boolean, + single: Option[SingleSearch] + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + + logger.info(s"♻️ Replacing existing index $indexName.") + + // 1. Delete existing index + api.deleteIndex(indexName) match { + case ElasticFailure(err) => + logger.error(s"❌ Failed to delete index $indexName: ${err.message}") + return Future.successful(ElasticFailure(err.copy(operation = Some("schema")))) + + case ElasticSuccess(false) => + logger.warn(s"⚠️ Index $indexName could not be deleted.") + return Future.successful(ElasticSuccess(DdlResult(false))) + + case ElasticSuccess(true) => + logger.info(s"🗑️ Index $indexName deleted successfully.") + } + + // 2. Recreate index using the same logic as createNonExistentIndex + createNonExistentIndex(indexName, create, partitioned, single) + } + + /* Create index / template for non-existent index + */ + private def createNonExistentIndex( + indexName: String, + create: CreateTable, + partitioned: Boolean, + single: Option[SingleSearch] + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // index does not exist, proceed with creation + logger.info(s"✅ Creating index $indexName.") + var table: Table = + single match { + case Some(single) => + single.from.tables.map(_.name).toSet.headOption match { + case Some(from) => + api.loadSchema(from) match { + case ElasticSuccess(fromSchema) => + // we update the schema based on the DQL select clause + fromSchema.mergeWithSearch(single) + case ElasticFailure(elasticError) => + val error = ElasticError( + message = + s"Error retrieving source schema $from for DQL in CREATE TABLE: ${elasticError.message}", + statusCode = elasticError.statusCode, + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case _ => + val error = + ElasticError( + message = s"Source index not specified for DQL in CREATE TABLE.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case _ => + create.schema + } + + // create index pipeline(s) if needed + table.defaultPipeline match { + case pipeline if pipeline.processors.nonEmpty => + createIndexPipeline(indexName, pipeline) match { + case ElasticSuccess(_) => + table = table.setDefaultPipelineName(pipelineName = pipeline.name) + case ElasticFailure(elasticError) => + return Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case _ => + } + + table.finalPipeline match { + case pipeline if pipeline.processors.nonEmpty => + createIndexPipeline(indexName, pipeline) match { + case ElasticSuccess(_) => + table = table.setFinalPipelineName(pipelineName = pipeline.name) + case ElasticFailure(elasticError) => + return Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case _ => + } + + // create index / template + createIndexOrTemplate(table, partitioned = partitioned) match { + case ElasticSuccess(_) => + // index / template created successfully + single match { + case Some(single) => + // populate index based on DQL + logger.info( + s"🚚 Populating index $indexName based on DQL." + ) + val query = + s"""INSERT INTO ${create.table} AS ${single.sql} ON DUPLICATE KEY NOTHING""" + val result = api.insertByQuery( + index = indexName, + query = query + ) + result.map { + case ElasticSuccess(res) => + logger.info( + s"✅ Index $indexName populated successfully (${res.inserted} documents indexed)." + ) + ElasticResult.success(res) + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } + case _ => + // no population needed + Future.successful(ElasticResult.success(DdlResult(true))) + } + case ElasticFailure(elasticError) => + // error creating index / template + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + } + + /* Create ingesting pipeline for index + */ + private def createIndexPipeline( + indexName: String, + pipeline: IngestPipeline + ): ElasticResult[Boolean] = { + logger.info( + s"🔧 Creating ${pipeline.pipelineType.name} ingesting pipeline ${pipeline.name} for index $indexName." + ) + api.createPipeline(pipeline.name, pipeline.json) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Pipeline ${pipeline.name} created successfully.") + success + case ElasticSuccess(_) => + // pipeline creation failed + val error = + ElasticError( + message = s"Failed to create pipeline ${pipeline.name}.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(_) => + failure + } + + } + + /* Create index or template based on partitioning + */ + private def createIndexOrTemplate(table: Table, partitioned: Boolean): ElasticResult[Boolean] = { + val indexName = table.name + if (partitioned) { + // create index template + api.createTemplate(s"template_$indexName", table.indexTemplate) match { + case success @ ElasticSuccess(true) => + logger.info(s"✅ Template template_$indexName created successfully.") + success + case ElasticSuccess(_) => + // template creation failed + val error = + ElasticError( + message = s"Failed to create template template_$indexName.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(_) => + failure + } + } else { + // create index + api.createIndex( + index = indexName, + settings = table.indexSettings, + mappings = Some(table.indexMappings), + aliases = table.indexAliases + ) match { + case success @ ElasticSuccess(true) => + // index created successfully + logger.info(s"✅ Index $indexName created successfully.") + success + case ElasticSuccess(_) => + // index creation failed + val error = + ElasticError( + message = s"Failed to create index $indexName.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(_) => + failure + } + } + } + + private def loadTablePipelineDiff( + table: Table, + pipelineType: IngestPipelineType, + updatedPipelineName: Option[String], + existingPipelineName: Option[String] + ): ElasticResult[Option[List[PipelineDiff]]] = { + updatedPipelineName match { + case Some(pipelineName) if pipelineName != existingPipelineName.getOrElse("") => + logger.info( + s"🔄 ${pipelineType.name} ingesting pipeline for index ${table.name} has been updated to $pipelineName." + ) + // load new pipeline + api.getPipeline(pipelineName) match { + case ElasticSuccess(maybePipeline) if maybePipeline.isDefined => + val pipeline = IngestPipeline( + name = pipelineName, + json = maybePipeline.get, + pipelineType = Some(IngestPipelineType.Default) + ) + // compute diff for pipeline update + val pipelineDiff: List[PipelineDiff] = pipeline.diff(table.defaultPipeline) + ElasticSuccess(Some(pipelineDiff)) + case ElasticSuccess(_) => + val error = + ElasticError( + message = + s"${pipelineType.name} ingesting pipeline $pipelineName for index ${table.name} not found.", + statusCode = Some(404), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case ElasticFailure(elasticError) => + val error = + ElasticError( + message = + s"Error retrieving ${pipelineType.name} ingesting pipeline $pipelineName for index ${table.name}: ${elasticError.message}", + statusCode = elasticError.statusCode, + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + } + case _ => ElasticSuccess(None) + } + } + + private def applyUpdatesInPlace( + indexName: String, + updated: Table, + diff: TableDiff + ): Either[ElasticError, Boolean] = { + + val mappingUpdate = + if (diff.mappings.nonEmpty || diff.columns.exists(_.isInstanceOf[ColumnOptionSet])) + api.setMapping(indexName, updated.indexMappings) + else ElasticSuccess(true) + + val settingsUpdate = + if (diff.settings.nonEmpty) + api.updateSettings(indexName, updated.indexSettings) + else ElasticSuccess(true) + + val aliasUpdate = + if (diff.aliases.nonEmpty) + api.setAliases(indexName, updated.indexAliases) + else ElasticSuccess(true) + + val defaultPipelineCreateOrUpdate = + if (diff.defaultPipeline.nonEmpty) { + updated.defaultPipelineName match { + case Some(pipelineName) => + logger.info( + s"🔧 Updating default ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Default)) + case None => + val pipelineName = updated.defaultPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Default) + ) + } + } else ElasticSuccess(true) + + val finalPipelineCreateOrUpdate = + if (diff.finalPipeline.nonEmpty) { + updated.finalPipelineName match { + case Some(pipelineName) => + logger.info( + s"🔧 Updating final ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Final)) + case None => + val pipelineName = updated.finalPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Final) + ) + } + } else ElasticSuccess(true) + + for { + _ <- mappingUpdate.toEither + _ <- settingsUpdate.toEither + _ <- aliasUpdate.toEither + _ <- defaultPipelineCreateOrUpdate.toEither + last <- finalPipelineCreateOrUpdate.toEither + } yield last + } + + private def migrateToNewSchema( + indexName: String, + oldSchema: Table, + newSchema: Table, + diff: TableDiff + ): Either[ElasticError, Boolean] = { + + val tmpIndex = s"${indexName}_tmp_${System.currentTimeMillis()}" + + // migrate index with updated schema + val migrate: ElasticResult[Boolean] = + api.performMigration( + index = indexName, + tempIndex = tmpIndex, + mapping = newSchema.indexMappings, + settings = newSchema.indexSettings, + aliases = newSchema.indexAliases + ) match { + case ElasticFailure(err) => + logger.error(s"❌ Failed to perform migration for index $indexName: ${err.message}") + api.rollbackMigration( + index = indexName, + tempIndex = tmpIndex, + originalMapping = oldSchema.indexMappings, + originalSettings = oldSchema.indexSettings, + originalAliases = oldSchema.indexAliases + ) match { + case ElasticSuccess(_) => + logger.info(s"✅ Rollback of migration for index $indexName completed successfully.") + ElasticFailure(err.copy(operation = Some("schema"))) + case ElasticFailure(rollbackErr) => + logger.error( + s"❌ Failed to rollback migration for index $indexName: ${rollbackErr.message}" + ) + ElasticFailure( + ElasticError( + message = + s"Migration failed: ${err.message}. Rollback failed: ${rollbackErr.message}", + statusCode = Some(500), + operation = Some("schema") + ) + ) + } + + case success @ ElasticSuccess(_) => + logger.info( + s"🔄 Migration performed successfully for index $indexName to temporary index $tmpIndex." + ) + success + } + + val defaultPipelineCreateOrUpdate: ElasticResult[Boolean] = + if (diff.defaultPipeline.nonEmpty) { + newSchema.defaultPipelineName match { + case Some(pipelineName) => + logger.info( + s"🔧 Updating default ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Default)) + case None => + val pipelineName = newSchema.defaultPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Default) + ) + } + } else ElasticSuccess(true) + + val finalPipelineCreateOrUpdate: ElasticResult[Boolean] = + if (diff.finalPipeline.nonEmpty) { + newSchema.finalPipelineName match { + case Some(pipelineName) => + logger.info( + s"🔧 Updating final ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Final)) + case None => + val pipelineName = newSchema.finalPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Final) + ) + } + } else ElasticSuccess(true) + + for { + _ <- migrate.toEither + _ <- defaultPipelineCreateOrUpdate.toEither + last <- finalPipelineCreateOrUpdate.toEither + } yield last + } +} + +class DdlRouterExecutor( + pipelineExec: PipelineExecutor, + tableExec: TableExecutor +) extends Executor[DdlStatement] { + + override def execute( + statement: DdlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = statement match { + + case p: PipelineStatement => pipelineExec.execute(p) + case t: TableStatement => tableExec.execute(t) + + case _ => + Future.successful( + ElasticFailure( + ElasticError(s"Unsupported DDL statement: $statement", statusCode = Some(400)) + ) + ) + } +} + +trait SqlGateway extends ElasticClientHelpers { + _: IndicesApi + with PipelineApi + with MappingApi + with SettingsApi + with AliasApi + with TemplateApi + with SearchApi + with ScrollApi + with VersionApi => + + lazy val dqlExecutor = new DqlExecutor( + api = this, + logger = logger + ) + + lazy val dmlExecutor = new DmlExecutor( + api = this, + logger = logger + ) + + lazy val pipelineExecutor = new PipelineExecutor( + api = this, + logger = logger + ) + + lazy val tableExecutor = new TableExecutor( + api = this, + logger = logger + ) + + lazy val ddlExecutor = new DdlRouterExecutor( + pipelineExec = pipelineExecutor, + tableExec = tableExecutor + ) + + // ======================================================================== + // SQL GATEWAY API + // ======================================================================== + + def run(sql: String)(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + ElasticResult.attempt(Parser(sql)) match { + case ElasticSuccess(parsedStatement) => + parsedStatement match { + case Right(statement) => + run(statement) + case Left(l) => + // parsing error + val error = + ElasticError( + message = s"Error parsing schema DDL statement: ${l.msg}", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + + case ElasticFailure(elasticError) => + // parsing error + Future.successful(ElasticFailure(elasticError.copy(operation = Some("schema")))) + } + + } + + def run( + statement: Statement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + statement match { + + case dql: DqlStatement => + dqlExecutor.execute(dql) + + // handle DML statements + case dml: DmlStatement => + dmlExecutor.execute(dml) + + // handle DDL statements + case ddl: DdlStatement => + ddlExecutor.execute(ddl) + + case _ => + // unsupported SQL statement + val error = + ElasticError( + message = s"Unsupported SQL statement: $statement", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 84191f30..b6e09136 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -33,7 +33,7 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} -import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -217,12 +217,24 @@ class MetricsElasticClient( */ override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit system: ActorSystem - ): Future[ElasticResult[Long]] = { + ): Future[ElasticResult[DmlResult]] = { measureAsync("insertByQuery", Some(index)) { delegate.insertByQuery(index, query, refresh) }(system.dispatcher) } + /** Load the schema for the provided index. + * + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + override def loadSchema(index: String): ElasticResult[Schema] = + measureResult("loadSchema", Some(index.indices.mkString(","))) { + delegate.loadSchema(index) + } + // ==================== AliasApi ==================== override def addAlias(index: String, alias: String): ElasticResult[Boolean] = { @@ -267,13 +279,13 @@ class MetricsElasticClient( * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ - override def getAliases(index: String): ElasticResult[Set[String]] = + override def getAliases(index: String): ElasticResult[Seq[TableAlias]] = measureResult("getAliases", Some(index)) { delegate.getAliases(index) } @@ -312,6 +324,30 @@ class MetricsElasticClient( delegate.swapAlias(oldIndex, newIndex, alias) } + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * @example + * {{{ + * setAliases("my-index", Set("alias1", "alias2")) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + override def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = + measureResult("setAliases", Some(index)) { + delegate.setAliases(index, aliases) + } + // ==================== SettingsApi ==================== override def updateSettings(index: String, settings: String): ElasticResult[Boolean] = { @@ -731,9 +767,9 @@ class MetricsElasticClient( * @return * the Elasticsearch response */ - override def search(sql: SelectStatement): ElasticResult[ElasticResponse] = + override def search(statement: DqlStatement): ElasticResult[ElasticResponse] = measureResult("search") { - delegate.search(sql) + delegate.search(statement) } /** Asynchronous search for documents / aggregations matching the SQL query. @@ -744,10 +780,10 @@ class MetricsElasticClient( * a Future containing the Elasticsearch response */ override def searchAsync( - sqlQuery: SelectStatement + statement: DqlStatement )(implicit ec: ExecutionContext): Future[ElasticResult[ElasticResponse]] = measureAsync("searchAsync") { - delegate.searchAsync(sqlQuery) + delegate.searchAsync(statement) } /** Searches and converts results into typed entities from an SQL query. diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 1d214f4a..d1231101 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -16,6 +16,10 @@ package app.softnetwork.elastic.client +import akka.NotUsed +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.scroll.ScrollMetrics + import scala.util.control.NonFatal package object result { @@ -368,4 +372,32 @@ package object result { } } } + + sealed trait QueryResult + + // -------------------- + // DQL (SELECT) + // -------------------- + case class QueryRows(rows: Seq[Map[String, Any]]) extends QueryResult + + case class QueryStream( + stream: Source[(Map[String, Any], ScrollMetrics), NotUsed] + ) extends QueryResult + + case class QueryStructured(response: ElasticResponse) extends QueryResult + + // -------------------- + // DML (INSERT / UPDATE / DELETE) + // -------------------- + case class DmlResult( + inserted: Long = 0L, + updated: Long = 0L, + deleted: Long = 0L, + rejected: Long = 0L + ) extends QueryResult + + // -------------------- + // DDL (CREATE / ALTER / DROP / TRUNCATE) + // -------------------- + case class DdlResult(success: Boolean) extends QueryResult } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index cf247a1f..2cd72c1c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for AliasApi */ @@ -36,8 +37,7 @@ class AliasApiSpec var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = { executeAddAliasResult } @@ -509,10 +509,10 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1", "alias2") + result.get.map(_.alias).toSet shouldBe Set("alias1", "alias2") verify(mockLogger).debug("Getting aliases for index 'my-index'") - verify(mockLogger).debug("Found 2 alias(es) for index 'my-index': alias1, alias2") +// verify(mockLogger).debug("Found 2 alias(es) for index 'my-index': alias1, alias2") verify(mockLogger).info("✅ Found 2 alias(es) for index 'my-index': alias1, alias2") } @@ -526,9 +526,10 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty - verify(mockLogger).debug("No aliases found for index 'my-index'") + verify(mockLogger).debug("Getting aliases for index 'my-index'") +// verify(mockLogger).debug("No aliases found for index 'my-index'") verify(mockLogger).info("✅ No aliases found for index 'my-index'") } @@ -542,7 +543,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "return empty set when index not found in response" in { @@ -554,7 +555,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty verify(mockLogger).warn("Index 'my-index' not found in response") } @@ -620,7 +621,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1", "alias2", "alias3") + result.get.map(_.alias).toSet shouldBe Set("alias1", "alias2", "alias3") } "handle single alias" in { @@ -633,7 +634,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("single-alias") + result.get.map(_.alias).toSet shouldBe Set("single-alias") verify(mockLogger).info(contains("Found 1 alias(es)")) } @@ -655,7 +656,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1") + result.get.map(_.alias).toSet shouldBe Set("alias1") } } @@ -903,7 +904,7 @@ class AliasApiSpec add1.isSuccess shouldBe true add2.isSuccess shouldBe true getResult.isSuccess shouldBe true - getResult.get should contain allOf ("alias1", "alias2") + getResult.get.map(_.alias) should contain allOf ("alias1", "alias2") } "verify alias existence before and after removal" in { @@ -972,7 +973,7 @@ class AliasApiSpec val result = for { _ <- aliasApi.addAlias("my-index", "my-alias") aliases <- aliasApi.getAliases("my-index") - } yield aliases.contains("my-alias") + } yield aliases.map(_.alias).contains("my-alias") // Then result shouldBe ElasticSuccess(true) @@ -1099,7 +1100,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "handle JSON with unexpected structure" in { @@ -1111,7 +1112,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "handle multiple consecutive swapAlias calls" in { @@ -1445,7 +1446,8 @@ class AliasApiSpec aliasApi.getAliases("my-index") // Then - verify(mockLogger).debug(contains("Found 2 alias(es)")) + verify(mockLogger).debug("Getting aliases for index 'my-index'") +// verify(mockLogger).debug(contains("Found 2 alias(es)")) verify(mockLogger).info(contains("✅ Found 2 alias(es)")) } @@ -1621,8 +1623,7 @@ class AliasApiSpec override protected def logger: Logger = mockLogger override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) @@ -1667,7 +1668,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("filtered-alias") + result.get.map(_.alias) should contain("filtered-alias") } "correctly parse aliases with routing" in { @@ -1689,7 +1690,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("routed-alias") + result.get.map(_.alias) should contain("routed-alias") } "correctly parse aliases with search and index routing" in { @@ -1712,7 +1713,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("complex-alias") + result.get.map(_.alias) should contain("complex-alias") } "handle null values in JSON" in { @@ -1725,7 +1726,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("my-alias") + result.get.map(_.alias) should contain("my-alias") } "handle empty aliases object" in { @@ -1738,7 +1739,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get.map(_.alias) shouldBe Seq.empty } "handle malformed JSON gracefully" in { @@ -1772,7 +1773,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("my-alias") + result.get.map(_.alias) should contain("my-alias") } "handle deeply nested JSON structure" in { @@ -1801,7 +1802,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("alias1") + result.get.map(_.alias) should contain("alias1") } "handle JSON with unicode characters" in { @@ -1921,7 +1922,7 @@ class AliasApiSpec // Then addCurrent.isSuccess shouldBe true addMonth.isSuccess shouldBe true - getAliases.get should contain allOf ("logs-current", "logs-january") + getAliases.get.map(_.alias) should contain allOf ("logs-current", "logs-january") } "support filtered alias for multi-tenancy" in { @@ -2118,7 +2119,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty result.get.isEmpty shouldBe true } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala index 997a7869..7d84a327 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.AliasApi import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.schema.TableAlias import io.searchbox.client.JestResult import io.searchbox.indices.aliases.{AddAliasMapping, GetAliases, ModifyAliases, RemoveAliasMapping} @@ -34,14 +35,20 @@ trait JestAliasApi extends AliasApi with JestClientHelpers { * @see * [[AliasApi.addAlias]] */ - private[client] def executeAddAlias(index: String, alias: String): ElasticResult[Boolean] = + private[client] def executeAddAlias(alias: TableAlias): ElasticResult[Boolean] = executeJestBooleanAction( operation = "addAlias", - index = Some(s"$index -> $alias"), + index = Some(s"${alias.table} -> ${alias.alias}"), retryable = false // Aliases operations can not be retried ) { + val builder = new AddAliasMapping.Builder(alias.table, alias.alias) + if (alias.filter.nonEmpty) + builder.setFilter(alias.filter.asJava.asInstanceOf[java.util.Map[String, AnyRef]]) + alias.routing.foreach(builder.addRouting) + alias.indexRouting.foreach(builder.addIndexRouting) + alias.searchRouting.foreach(builder.addSearchRouting) new ModifyAliases.Builder( - new AddAliasMapping.Builder(index, alias).build() + builder.build() ).build() } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index d18f8897..fe53c9c3 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -33,7 +33,12 @@ import scala.util.Try * [[MappingApi]] for generic API documentation */ trait JestMappingApi extends MappingApi with JestClientHelpers { - _: JestSettingsApi with JestIndicesApi with JestRefreshApi with JestClientCompanion => + _: JestSettingsApi + with JestIndicesApi + with JestRefreshApi + with JestVersionApi + with JestAliasApi + with JestClientCompanion => /** Set the mapping for an index. * @see diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 0ec9825e..8f3f4273 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -430,20 +430,29 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false )( - request = new IndicesAliasesRequest() - .addAliasAction( - new AliasActions(AliasActions.Type.ADD) - .index(index) - .alias(alias) - ) + request = { + val aliasAction = new AliasActions(AliasActions.Type.ADD) + .index(alias.table) + .alias(alias.alias) + if (alias.isWriteIndex) { + aliasAction.writeIndex(true) + } + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + aliasAction.filter(filterNode.toString) + } + alias.routing.foreach(aliasAction.routing) + alias.indexRouting.foreach(aliasAction.indexRouting) + alias.searchRouting.foreach(aliasAction.searchRouting) + new IndicesAliasesRequest().addAliasAction(aliasAction) + } )( executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) @@ -559,6 +568,8 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH _: RestHighLevelClientSettingsApi with RestHighLevelClientIndicesApi with RestHighLevelClientRefreshApi + with RestHighLevelClientVersionApi + with RestHighLevelClientAliasApi with RestHighLevelClientCompanion => override private[client] def executeSetMapping( index: String, diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 9fe9316d..ef4666d7 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -419,20 +419,29 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false )( - request = new IndicesAliasesRequest() - .addAliasAction( - new AliasActions(AliasActions.Type.ADD) - .index(index) - .alias(alias) - ) + request = { + val aliasAction = new AliasActions(AliasActions.Type.ADD) + .index(alias.table) + .alias(alias.alias) + if (alias.isWriteIndex) { + aliasAction.writeIndex(true) + } + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + aliasAction.filter(filterNode.toString) + } + alias.routing.foreach(aliasAction.routing) + alias.indexRouting.foreach(aliasAction.indexRouting) + alias.searchRouting.foreach(aliasAction.searchRouting) + new IndicesAliasesRequest().addAliasAction(aliasAction) + } )( executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) @@ -547,6 +556,8 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH _: RestHighLevelClientSettingsApi with RestHighLevelClientIndicesApi with RestHighLevelClientRefreshApi + with RestHighLevelClientVersionApi + with RestHighLevelClientAliasApi with RestHighLevelClientCompanion => override private[client] def executeSetMapping( diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 037b85b2..3f0b2e55 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -342,26 +342,31 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false - )( + ) { + val node = alias.node + node.put("index", alias.table) + node.put("alias", alias.alias) + + val action = mapper.createObjectNode() + action.set("add", node) + + val addAction = new Action.Builder().withJson(new StringReader(action)) apply() .indices() .updateAliases( new UpdateAliasesRequest.Builder() .actions( - new Action.Builder() - .add(new AddAction.Builder().index(index).alias(alias).build()) - .build() + addAction.build() ) .build() ) - )(_.acknowledged()) + }(_.acknowledged()) override private[client] def executeRemoveAlias( index: String, @@ -490,6 +495,8 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { _: JavaClientSettingsApi with JavaClientIndicesApi with JavaClientRefreshApi + with JavaClientVersionApi + with JavaClientAliasApi with JavaClientCompanion => override private[client] def executeSetMapping( diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index ed5cb31f..75d6fbfb 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -341,26 +341,31 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false - )( + ) { + val node = alias.node + node.put("index", alias.table) + node.put("alias", alias.alias) + + val action = mapper.createObjectNode() + action.set("add", node) + + val addAction = new Action.Builder().withJson(new StringReader(action)) apply() .indices() .updateAliases( new UpdateAliasesRequest.Builder() .actions( - new Action.Builder() - .add(new AddAction.Builder().index(index).alias(alias).build()) - .build() + addAction.build() ) .build() ) - )(_.acknowledged()) + }(_.acknowledged()) override private[client] def executeRemoveAlias( index: String, @@ -489,6 +494,8 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { _: JavaClientSettingsApi with JavaClientIndicesApi with JavaClientRefreshApi + with JavaClientVersionApi + with JavaClientAliasApi with JavaClientCompanion => override private[client] def executeSetMapping( diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 09015897..b2ca0768 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -26,6 +26,7 @@ import app.softnetwork.elastic.sql.schema.{ IngestProcessor, PartitionDate, PrimaryKeyProcessor, + Schema, ScriptProcessor, Table } @@ -351,7 +352,7 @@ package object schema { defaultProcessors ++ finalProcessors } - lazy val asTable: Table = { + lazy val schema: Schema = { // 1. Columns from the mapping val initialCols: Map[String, Column] = esMappings.fields.map { field => @@ -399,7 +400,7 @@ package object schema { } - // 4. Final construction of the DdlTable + // 4. Final construction of the Table Table( name = name, columns = enrichedCols.values.toList.sortBy(_.name), diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 6b8c41a2..78d3169d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -21,6 +21,7 @@ import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.function.time.CurrentFunction import app.softnetwork.elastic.sql.parser.{Validation, Validator} import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.Column import com.fasterxml.jackson.databind.JsonNode import java.security.MessageDigest @@ -1065,7 +1066,8 @@ package object sql { fieldAlias: Option[String] = None, bucket: Option[Bucket] = None, nestedElement: Option[NestedElement] = None, - bucketPath: String = "" + bucketPath: String = "", + col: Option[Column] = None ) extends Identifier { def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) @@ -1076,6 +1078,8 @@ package object sql { id } + override def baseType: SQLType = col.map(_.dataType).getOrElse(super.baseType) + def update(request: SingleSearch): Identifier = { val bucketPath: String = request.groupBy match { @@ -1100,27 +1104,31 @@ package object sql { case Some(unnest) => Some(request.toNestedElement(unnest)) case None => None } + val colName = parts.tail.mkString(".") this .copy( tableAlias = Some(tableAlias), - name = s"${tuple._2._1}.${parts.tail.mkString(".")}", + name = s"${tuple._2._1}.$colName", nested = true, limit = tuple._2._2, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), nestedElement = nestedElement, - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)) ) .withFunctions(this.updateFunctions(request)) case Some(tuple) if nested => + val colName = parts.tail.mkString(".") this .copy( tableAlias = Some(tableAlias), - name = s"${tuple._2._1}.${parts.tail.mkString(".")}", + name = s"${tuple._2._1}.$colName", limit = tuple._2._2, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)) ) .withFunctions(this.updateFunctions(request)) case None if nested => @@ -1129,16 +1137,19 @@ package object sql { tableAlias = Some(tableAlias), fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(name)) ) .withFunctions(this.updateFunctions(request)) case _ => + val colName = parts.tail.mkString(".") this.copy( tableAlias = Some(tableAlias), - name = parts.tail.mkString("."), + name = colName, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)) ) } } else { @@ -1146,7 +1157,8 @@ package object sql { .copy( fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(name)) ) .withFunctions(this.updateFunctions(request)) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index ef709565..c2022bf5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -27,6 +27,7 @@ import app.softnetwork.elastic.sql.schema.{ PartitionDate, RemoveProcessor, RenameProcessor, + Schema, ScriptProcessor, Table => DdlTable } @@ -77,7 +78,8 @@ package object query { score: Option[Double] = None, deleteByQuery: Boolean = false, updateByQuery: Boolean = false, - onConflict: Option[OnConflict] = None + onConflict: Option[OnConflict] = None, + schema: Option[Schema] = None ) extends DqlStatement { override def sql: String = s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}${asString(onConflict)}" @@ -147,7 +149,11 @@ package object query { lazy val sorts: Map[String, SortOrder] = orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap - def update(): SingleSearch = { + def update(schema: Option[Schema] = None): SingleSearch = { + schema match { + case Some(s) => return this.copy(schema = Some(s)).update() + case None => // continue + } (for { from <- Option(this.copy(from = from.update(this))) select <- Option( @@ -529,7 +535,7 @@ package object query { case None => Map.empty } - lazy val ddlTable: DdlTable = DdlTable( + lazy val schema: Schema = DdlTable( name = table, columns = columns.toList, primaryKey = primaryKey, @@ -539,7 +545,7 @@ package object query { aliases = aliases ).update() - lazy val defaultPipeline: IngestPipeline = ddlTable.defaultPipeline + lazy val defaultPipeline: IngestPipeline = schema.defaultPipeline } @@ -556,10 +562,10 @@ package object query { s"ALTER TABLE $table$ifExistsClause $statementsSql" } - lazy val ddlProcessors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) + lazy val processors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) lazy val pipeline: IngestPipeline = - IngestPipeline(s"alter-$table-${Instant.now}", IngestPipelineType.Custom, ddlProcessors) + IngestPipeline(s"alter-$table-${Instant.now}", IngestPipelineType.Custom, processors) } sealed trait AlterTableStatement extends Token { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala new file mode 100644 index 00000000..fd01b775 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala @@ -0,0 +1,38 @@ +package app.softnetwork.elastic.sql.schema + +object MappingsRules { + + // Options can be modified without reindexing + private val safeOptions = Set( + "ignore_above", + "null_value", + "store", + "boost", + "coerce", + "copy_to", + "meta" + ) + + // Options requiring a reindex + private val unsafeOptions = Set( + "analyzer", + "search_analyzer", + "normalizer", + "index", + "doc_values", + "format", + "fields", + "similarity", + "eager_global_ordinals" + ) + + def isSafe(key: String): Boolean = { + + if (safeOptions.contains(key)) return true + + if (unsafeOptions.contains(key)) return false + + // Default: cautious → UNSAFE + false + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala new file mode 100644 index 00000000..727b6cc0 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala @@ -0,0 +1,43 @@ +package app.softnetwork.elastic.sql.schema + +object SettingsRules { + + private val dynamicPrefixes = Seq( + "refresh_interval", + "number_of_replicas", + "max_result_window", + "max_inner_result_window", + "max_rescore_window", + "max_docvalue_fields_search", + "max_script_fields", + "max_ngram_diff", + "max_shingle_diff", + "blocks.read_only", + "blocks.read_only_allow_delete", + "blocks.write", + "routing.allocation" + ) + + private val staticPrefixes = Seq( + "number_of_shards", + "codec", + "routing_partition_size", + "analysis", + "similarity", + "sort", + "mapping.total_fields.limit", + "mapping.depth.limit", + "mapping.nested_fields.limit" + ) + + def isDynamic(key: String): Boolean = { + // If the setting is explicitly static → false + if (staticPrefixes.exists(key.contains)) return false + + // If the setting is explicitly dynamic → true + if (dynamicPrefixes.exists(key.contains)) return true + + // Default: cautious → UNSAFE + false + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala index ce18d118..bf5722b4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala @@ -17,101 +17,141 @@ package app.softnetwork.elastic.sql.schema import app.softnetwork.elastic.sql.Value -import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} import app.softnetwork.elastic.sql.query._ +sealed trait DiffSafety +case object Safe extends DiffSafety +case object UnsafeReindex extends DiffSafety +case object Impossible extends DiffSafety + sealed trait AlterTableStatementDiff { def stmt: AlterTableStatement + def safety: DiffSafety } sealed trait ColumnDiff extends AlterTableStatementDiff case class ColumnAdded(column: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AddColumn(column) + override def safety: DiffSafety = Safe } case class ColumnRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumn(name) + override def safety: DiffSafety = UnsafeReindex } // case class ColumnRenamed(oldName: String, newName: String) extends ColumnDiff case class ColumnTypeChanged(name: String, from: SQLType, to: SQLType) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnType(name, to) + + override def safety: DiffSafety = + if (SQLTypeUtils.canConvert(from, to)) + UnsafeReindex + else + Impossible } case class ColumnDefaultSet(name: String, value: Value[_]) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnDefault(name, value) + override def safety: DiffSafety = Safe } case class ColumnDefaultRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnDefault(name) + override def safety: DiffSafety = Safe } case class ColumnScriptSet(name: String, script: ScriptProcessor) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnScript(name, script) + override def safety: DiffSafety = Safe } case class ColumnScriptRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnScript(name) + override def safety: DiffSafety = Safe } case class ColumnCommentSet(name: String, comment: String) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnComment(name, comment) + override def safety: DiffSafety = Safe } case class ColumnCommentRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnComment(name) + override def safety: DiffSafety = Safe } case class ColumnNotNullSet(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnNotNull(name) + override def safety: DiffSafety = Safe } case class ColumnNotNullRemoved(name: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnNotNull(name) + override def safety: DiffSafety = Safe } case class ColumnOptionSet(name: String, key: String, value: Value[_]) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnOption(name, key, value) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex } case class ColumnOptionRemoved(name: String, key: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnOption(name, key) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex } case class FieldAdded(column: String, field: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnField(column, field) + override def safety: DiffSafety = Safe } case class FieldRemoved(column: String, fieldName: String) extends ColumnDiff { override def stmt: AlterTableStatement = DropColumnField(column, fieldName) + override def safety: DiffSafety = Safe } case class FieldAltered(column: String, field: Column) extends ColumnDiff { override def stmt: AlterTableStatement = AlterColumnField(column, field) + override def safety: DiffSafety = Safe } sealed trait MappingDiff extends AlterTableStatementDiff case class MappingSet(key: String, value: Value[_]) extends MappingDiff { override def stmt: AlterTableStatement = AlterTableMapping(key, value) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex } case class MappingRemoved(key: String) extends MappingDiff { override def stmt: AlterTableStatement = DropTableMapping(key) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex } sealed trait SettingDiff extends AlterTableStatementDiff case class SettingSet(key: String, value: Value[_]) extends SettingDiff { override def stmt: AlterTableStatement = AlterTableSetting(key, value) + override def safety: DiffSafety = + if (SettingsRules.isDynamic(key)) Safe + else UnsafeReindex } case class SettingRemoved(key: String) extends SettingDiff { override def stmt: AlterTableStatement = DropTableSetting(key) + override def safety: DiffSafety = + if (SettingsRules.isDynamic(key)) Safe + else UnsafeReindex } sealed trait AliasDiff extends AlterTableStatementDiff case class AliasSet(key: String, value: Value[_]) extends AliasDiff { override def stmt: AlterTableStatement = AlterTableAlias(key, value) + override def safety: DiffSafety = Safe } case class AliasRemoved(key: String) extends AliasDiff { override def stmt: AlterTableStatement = DropTableAlias(key) + override def safety: DiffSafety = Safe } sealed trait AlterPipelineStatementDiff { def stmt: AlterPipelineStatement + def safety: DiffSafety = Safe + def processor: IngestProcessor + def pipelineType: IngestPipelineType = processor.pipelineType } sealed trait PipelineDiff extends AlterPipelineStatementDiff @@ -141,6 +181,8 @@ case class ProcessorChanged( diff: ProcessorDiff ) extends PipelineDiff { override def stmt: AlterPipelineStatement = AlterPipelineProcessor(to) + override def pipelineType: IngestPipelineType = from.pipelineType + override def processor: IngestProcessor = to } case class TableDiff( @@ -171,18 +213,61 @@ case class TableDiff( } } - def alterPipeline(name: String, ifExists: Boolean): Option[AlterPipeline] = { - val pipelineStatements = pipeline.map(_.stmt) + def defaultPipeline: List[PipelineDiff] = + pipeline.filter(_.pipelineType == IngestPipelineType.Default) + + def finalPipeline: List[PipelineDiff] = + pipeline.filter(_.pipelineType == IngestPipelineType.Final) + + def createPipeline(name: String, pipelineType: IngestPipelineType): Option[CreatePipeline] = { + val pipelineProcessors = pipeline.filter(_.pipelineType == pipelineType).map(_.processor) + if (pipelineProcessors.isEmpty) { + None + } else { + Some( + CreatePipeline( + name, + pipelineType, + ifNotExists = true, + orReplace = false, + pipelineProcessors + ) + ) + } + } + + def alterPipeline(name: String, pipelineType: IngestPipelineType): Option[AlterPipeline] = { + val pipelineStatements = pipeline.filter(_.pipelineType == pipelineType).map(_.stmt) if (pipelineStatements.isEmpty) { None } else { Some( AlterPipeline( name, - ifExists, + ifExists = true, pipelineStatements ) ) } } + + def safety: DiffSafety = { + val safeties = + columns.map(_.safety) ++ + mappings.map(_.safety) ++ + settings.map(_.safety) ++ + pipeline.map(_.safety) ++ + aliases.map(_.safety) + + if (safeties.contains(Impossible)) Impossible + else if (safeties.contains(UnsafeReindex)) UnsafeReindex + else Safe + } + + def impossible: Boolean = safety == Impossible + + def requiresReindex: Boolean = safety == UnsafeReindex + + def safe: Boolean = safety == Safe + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index c1e63f65..3155ec7f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -31,6 +31,8 @@ import scala.language.implicitConversions package object schema { val mapper: ObjectMapper = JacksonConfig.objectMapper + type Schema = Table + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() sealed trait IngestProcessorType { @@ -1010,7 +1012,7 @@ package object schema { "_meta" -> ObjectValue(mappings.get("_meta") match { case Some(ObjectValue(value)) => - value ++ _meta + (value - "primary_key" - "partition_by") ++ _meta case _ => _meta }) ) @@ -1304,6 +1306,35 @@ package object schema { } + def mergeWithSearch(search: SingleSearch): Table = { + val fields = search.update(Some(this)).select.fields + val cols = + fields.foldLeft(List.empty[Column]) { (cols, field) => + val colName = field.fieldAlias.map(_.alias).getOrElse(field.sourceField) + val col = this.find(colName) match { + case Some(c) if field.functions.isEmpty => c + case None => + Column( + name = colName, + dataType = field.out + ) + } + cols :+ col + } + val primaryKey = + this.primaryKey.filter(pk => cols.exists(_.name == pk)) + val partitionBy = + this.partitionBy match { + case Some(partition) => + if (cols.exists(_.name == partition.column)) + Some(partition) + else + None + case _ => None + } + this.copy(columns = cols, primaryKey = primaryKey, partitionBy = partitionBy).update() + } + override def validate(): Either[String, Unit] = { var errors = Seq[String]() // check that primary key columns exist @@ -1403,6 +1434,24 @@ package object schema { TableAlias(name, aliasName, value) }.toSeq + lazy val indexTemplate: ObjectNode = { + val node = mapper.createObjectNode() + node.put("index_patterns", s"$name-*") + node.put("priority", 1) + val template = mapper.createObjectNode() + template.set("mappings", indexMappings) + template.set("settings", indexSettings) + if (aliases.nonEmpty) { + val aliasesNode = mapper.createObjectNode() + indexAliases.foreach { alias => + aliasesNode.set(alias.alias, alias.node) + } + template.set("aliases", aliasesNode) + } + node.set("template", template) + node + } + lazy val defaultPipelineNode: ObjectNode = defaultPipeline.node lazy val finalPipelineNode: ObjectNode = finalPipeline.node diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index 0dca230e..d381e9f9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -321,4 +321,65 @@ object SQLTypeUtils { s"($expr != null ? $ret : null)" } + private val numericRank: Map[Class[_], Int] = Map( + classOf[SQLTinyInt] -> 1, + classOf[SQLSmallInt] -> 2, + classOf[SQLInt] -> 3, + classOf[SQLBigInt] -> 4, + classOf[SQLReal] -> 5, + classOf[SQLDouble] -> 6 + ) + + private def numericRankOf(t: SQLType): Option[Int] = + numericRank.collectFirst { case (cls, rank) if cls.isInstance(t) => rank } + + def canConvert(from: SQLType, to: SQLType): Boolean = { + + // 1. Identity + if (from.typeId == to.typeId) return true + + // 2. ANY / NULL + if (to.isInstanceOf[SQLAny]) return true + if (from.isInstanceOf[SQLNull]) return true + + // 3. Numerics + (numericRankOf(from), numericRankOf(to)) match { + case (Some(r1), Some(r2)) => + return r1 <= r2 // expansion only + case _ => + } + + // 4. Literals (VARCHAR, CHAR, TEXT, KEYWORD) + if (from.isInstanceOf[SQLLiteral] && to.isInstanceOf[SQLLiteral]) + return true + + // 5. Booleans + if (from.isInstanceOf[SQLBool] && to.isInstanceOf[SQLBool]) + return true + + // 6. Temporals + if (from.isInstanceOf[SQLTemporal] && to.isInstanceOf[SQLTemporal]) + return true + + // 7. Arrays + (from, to) match { + case (f: SQLArray, t: SQLArray) => + canConvert(f.elementType, t.elementType) + case _ => + } + + // 8. Structs + (from, to) match { + case (f: SQLStruct, t: SQLStruct) => + structConvertible(f, t) + case _ => + } + + false + } + + private def structConvertible(from: SQLStruct, to: SQLStruct): Boolean = { + // TODO To be refined according to our definition of SQLStruct + true + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 2f399b51..38670d95 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -988,19 +988,19 @@ class ParserSpec extends AnyFlatSpec with Matchers { "_ingest.timestamp" ) ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) - val sql = ct.ddlTable.sql + val sql = ct.schema.sql println(sql) - println(ct.ddlTable.defaultPipeline.ddl) - val json = ct.ddlTable.defaultPipeline.json + println(ct.schema.defaultPipeline.ddl) + val json = ct.schema.defaultPipeline.json println(json) json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" - val indexMappings = ct.ddlTable.indexMappings + val indexMappings = ct.schema.indexMappings println(indexMappings) indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" - val indexSettings = ct.ddlTable.indexSettings + val indexSettings = ct.schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" - val pipeline = ct.ddlTable.defaultPipelineNode + val pipeline = ct.schema.defaultPipelineNode println(pipeline) pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex @@ -1014,13 +1014,13 @@ class ParserSpec extends AnyFlatSpec with Matchers { settings = settings, defaultPipeline = Some(pipeline) ) - val ddlTable = esIndex.asTable + val ddlTable = esIndex.schema println(s"""esIndex ddl -> ${ddlTable.sql}""") println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") println(s"""esIndex pipeline -> ${ddlTable.defaultPipelineNode.toString}""") println(s"""esIndex ddl pipeline -> ${ddlTable.defaultPipeline.ddl}""") - val ddlTableDiff = ddlTable.diff(ct.ddlTable) + val ddlTableDiff = ddlTable.diff(ct.schema) ddlTableDiff.columns.isEmpty shouldBe true ddlTableDiff.mappings.isEmpty shouldBe true ddlTableDiff.settings.isEmpty shouldBe true diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index b9633cba..04cc8bd7 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -168,7 +168,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M index match { case Some(idx) => - val table: Table = idx.asTable + val table: Table = idx.schema log.info(table.sql) table.columns.size shouldBe 3 table.columns.map(_.name) should contain allOf ("uuid", "name", "birthDate") @@ -200,7 +200,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M pClient.getAliases("person") match { case ElasticSuccess(aliases) => - aliases should contain("person_alias") + aliases.map(_.alias) should contain("person_alias") case ElasticFailure(elasticError) => fail(elasticError.fullMessage) } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala index 667d5e75..e52a00f1 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala @@ -20,7 +20,7 @@ import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Source import app.softnetwork.elastic.client.bulk.BulkOptions -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.client.result.{DmlResult, ElasticFailure, ElasticSuccess} import app.softnetwork.elastic.scalatest.ElasticTestKit import app.softnetwork.persistence.generateUUID import org.scalatest.concurrent.ScalaFutures @@ -189,7 +189,7 @@ trait InsertByQuerySpec |VALUES ('C010', 'Bob', 'bob@example.com', 'FR')""".stripMargin val result = client.insertByQuery("customers", sql).futureValue - result shouldBe ElasticSuccess(1L) + result shouldBe ElasticSuccess(DmlResult(inserted = 1L)) } it should "upsert a product with ON CONFLICT DO UPDATE" in { @@ -199,7 +199,7 @@ trait InsertByQuerySpec |ON CONFLICT DO UPDATE""".stripMargin.replaceAll("\\s+", " ") val result = client.insertByQuery("products", sql).futureValue - result shouldBe ElasticSuccess(1L) + result shouldBe ElasticSuccess(DmlResult(inserted = 1L)) } it should "insert orders from a SELECT with alias mapping" in { @@ -213,7 +213,7 @@ trait InsertByQuerySpec |FROM staging_orders""".stripMargin.replaceAll("\\s+", " ") val result = client.insertByQuery("orders", sql).futureValue - result shouldBe ElasticSuccess(3L) + result shouldBe ElasticSuccess(DmlResult(inserted = 3L)) } it should "upsert orders with composite PK using ON CONFLICT DO UPDATE" in { @@ -228,7 +228,7 @@ trait InsertByQuerySpec |ON CONFLICT (order_id, customer_id) DO UPDATE""".stripMargin val result = client.insertByQuery("orders", sql).futureValue - result shouldBe ElasticSuccess(2L) + result shouldBe ElasticSuccess(DmlResult(inserted = 2L)) } it should "fail when conflictTarget does not match PK" in { diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index 96d07a6c..023bb907 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -33,7 +33,7 @@ import scala.language.implicitConversions /** Created by smanciot on 12/04/2020. */ -trait MockElasticClientApi extends ElasticClientApi { +trait MockElasticClientApi extends NopeClientApi { def elasticVersion: String @@ -124,8 +124,7 @@ trait MockElasticClientApi extends ElasticClientApi { // ==================== AliasApi ==================== override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = ElasticResult.success(true) From f68234a7803d70e4d74d77f876fec9a8627fc242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 31 Dec 2025 11:09:40 +0100 Subject: [PATCH 43/95] add headers --- .../elastic/sql/schema/MappingsRules.scala | 16 ++++++++++++++++ .../elastic/sql/schema/SettingsRules.scala | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala index fd01b775..c6f00f33 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.sql.schema object MappingsRules { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala index 727b6cc0..e990a344 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package app.softnetwork.elastic.sql.schema object SettingsRules { From b3bbc324a2cc3930781d37f4e1d7a34b9f3631c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 31 Dec 2025 16:34:36 +0100 Subject: [PATCH 44/95] add support for SHOW and DESCRIBE TABLE, add SQL Gateway specifications --- .../elastic/client/SqlGateway.scala | 32 +- .../elastic/client/result/package.scala | 3 + .../elastic/sql/parser/Parser.scala | 16 +- .../elastic/sql/query/package.scala | 9 + .../elastic/sql/schema/package.scala | 12 + .../client/SqlGatewayIntegrationSpec.scala | 1176 +++++++++++++++++ 6 files changed, 1246 insertions(+), 2 deletions(-) create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala index 6ebeb4f1..4f0bdee4 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala @@ -25,8 +25,10 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess, QueryResult, + QueryRows, QueryStream, - QueryStructured + QueryStructured, + QueryTable } import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ @@ -34,6 +36,7 @@ import app.softnetwork.elastic.sql.query.{ CreateTable, DdlStatement, Delete, + DescribeTable, DmlStatement, DqlStatement, DropTable, @@ -41,6 +44,7 @@ import app.softnetwork.elastic.sql.query.{ MultiSearch, PipelineStatement, SelectStatement, + ShowTable, SingleSearch, Statement, TableStatement, @@ -226,6 +230,32 @@ class TableExecutor( implicit val ec: ExecutionContext = system.dispatcher // handle TABLE statement statement match { + // handle SHOW TABLE statement + case show: ShowTable => + api.loadSchema(show.table) match { + case ElasticSuccess(schema) => + logger.info(s"✅ Retrieved schema for index ${show.table}.") + Future.successful(ElasticResult.success(QueryTable(schema))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + // handle DESCRIBE TABLE statement + case describe: DescribeTable => + api.loadSchema(describe.table) match { + case ElasticSuccess(schema) => + logger.info(s"✅ Retrieved schema for index ${describe.table}.") + Future.successful(ElasticResult.success(QueryRows(schema.columns.flatMap(_.asMap)))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } // handle CREATE TABLE statement case create: CreateTable => val ifNotExists = create.ifNotExists diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index d1231101..905f070e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.stream.scaladsl.Source import app.softnetwork.elastic.client.scroll.ScrollMetrics +import app.softnetwork.elastic.sql.schema.Table import scala.util.control.NonFatal @@ -400,4 +401,6 @@ package object result { // DDL (CREATE / ALTER / DROP / TRUNCATE) // -------------------- case class DdlResult(success: Boolean) extends QueryResult + + case class QueryTable(table: Table) extends QueryResult } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index b4eb3a2b..a0062bc7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -314,6 +314,16 @@ object Parser } } + def showTable: PackratParser[ShowTable] = + ("SHOW" ~ "TABLE") ~ ident ^^ { case _ ~ table => + ShowTable(table) + } + + def describeTable: PackratParser[DescribeTable] = + ("DESCRIBE" ~ "TABLE") ~ ident ^^ { case _ ~ table => + DescribeTable(table) + } + def dropTable: PackratParser[DropTable] = ("DROP" ~ "TABLE") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => DropTable(name, ifExists = ie) @@ -517,6 +527,8 @@ object Parser alterPipeline | dropTable | truncateTable | + showTable | + describeTable | dropPipeline def onConflict: PackratParser[OnConflict] = @@ -764,7 +776,9 @@ trait Parser "replace", "on", "conflict", - "do" + "do", + "show", + "describe" ) private val identifierRegexStr = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index c2022bf5..6eaa8550 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -775,4 +775,13 @@ package object query { case class TruncateTable(table: String) extends TableStatement { override def sql: String = s"TRUNCATE TABLE $table" } + + case class ShowTable(table: String) extends TableStatement { + override def sql: String = s"SHOW TABLE $table" + } + + case class DescribeTable(table: String) extends TableStatement { + override def sql: String = s"DESCRIBE TABLE $table" + } + } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 3155ec7f..ebb27dfc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -641,6 +641,18 @@ package object schema { s"$tabs$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" } + def asMap: Seq[Map[String, Any]] = Seq( + Map( + "name" -> path, + "type" -> dataType.typeId, + "script" -> script.map(_.script), + "default" -> defaultValue.map(_.value).getOrElse(""), + "notNull" -> notNull, + "comment" -> comment.getOrElse(""), + "options" -> ObjectValue(options).ddl + ) + ) ++ multiFields.flatMap(_.asMap) + def processors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => DefaultValueProcessor( diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala new file mode 100644 index 00000000..4a5a0f14 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala @@ -0,0 +1,1176 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ + DdlResult, + DmlResult, + ElasticResult, + QueryResult, + QueryRows, + QueryStream, + QueryStructured, + QueryTable +} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.schema.Table +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContextExecutor} +import java.util.concurrent.TimeUnit + +// --------------------------------------------------------------------------- +// Base test trait — to be mixed with ElasticDockerTestKit +// --------------------------------------------------------------------------- + +trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(30, Seconds)) + + // Provided by concrete test class + def client: SqlGateway + + override def beforeAll(): Unit = { + self.beforeAll() + } + + override def afterAll(): Unit = { + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + self.afterAll() + } + + // ------------------------------------------------------------------------- + // Helper: assert SELECT result type + // ------------------------------------------------------------------------- + + def assertSelectResult(res: ElasticResult[QueryResult]): Unit = { + res.isSuccess shouldBe true + res.toOption.get match { + case QueryStream(_) => succeed + case QueryStructured(_) => succeed + case QueryRows(_) => succeed + case other => fail(s"Unexpected QueryResult type for SELECT: $other") + } + } + + // ------------------------------------------------------------------------- + // Helper: assert DDL result type + // ------------------------------------------------------------------------- + + def assertDdl(res: ElasticResult[QueryResult]): Unit = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[DdlResult] + } + + // ------------------------------------------------------------------------- + // Helper: assert DML result type + // ------------------------------------------------------------------------- + + def assertDml(res: ElasticResult[QueryResult]): Unit = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[DmlResult] + } + + // ------------------------------------------------------------------------- + // Helper: assert SHOW TABLE result type + // ------------------------------------------------------------------------- + + def assertShowTable(res: ElasticResult[QueryResult]): Table = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryTable] + res.toOption.get.asInstanceOf[QueryTable].table + } + + // ------------------------------------------------------------------------- + // SHOW / DESCRIBE TABLE tests + // ------------------------------------------------------------------------- + + behavior of "SHOW / DESCRIBE TABLE" + + it should "return the table schema using SHOW TABLE" in { + val create = + """CREATE TABLE IF NOT EXISTS show_users ( + | id INT NOT NULL, + | name VARCHAR, + | age INT DEFAULT 0 + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val show = client.run("SHOW TABLE show_users").futureValue + val table = assertShowTable(show) + + val ddl = table.ddl + ddl should include("CREATE TABLE show_users") + ddl should include("id INT NOT NULL") + ddl should include("name VARCHAR") + ddl should include("age INT DEFAULT 0") + } + + it should "describe a table using DESCRIBE TABLE" in { + val create = + """CREATE TABLE IF NOT EXISTS desc_users ( + | id INT NOT NULL, + | name VARCHAR DEFAULT 'anonymous', + | age INT, + | profile STRUCT FIELDS( + | city VARCHAR, + | followers INT + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val res = client.run("DESCRIBE TABLE desc_users").futureValue + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryRows] + + val rows = res.toOption.get.asInstanceOf[QueryRows].rows + + rows.exists(_("name") == "id") shouldBe true + rows.exists(_("name") == "name") shouldBe true + rows.exists(_("name") == "profile.city") shouldBe true + } + + // =========================================================================== + // 2. DDL — CREATE TABLE, ALTER TABLE, DROP TABLE, TRUNCATE TABLE + // =========================================================================== + + behavior of "DDL statements" + + // --------------------------------------------------------------------------- + // CREATE TABLE — full complex schema + // --------------------------------------------------------------------------- + + it should "create a table with complex columns, defaults, scripts, struct, PK, partitioning and options" in { + val sql = + """CREATE TABLE IF NOT EXISTS users ( + | id INT NOT NULL COMMENT 'user identifier', + | name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), + | birthdate DATE, + | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), + | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) COMMENT 'user profile', + | PRIMARY KEY (id) + |) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false));""".stripMargin + + assertDdl(client.run(sql).futureValue) + + // Vérification via SHOW TABLE + val table = assertShowTable(client.run("SHOW TABLE users").futureValue) + val ddl = table.ddl + + ddl should include("CREATE TABLE users") + ddl should include("id INT NOT NULL") + ddl should include("name VARCHAR") + ddl should include("DEFAULT 'anonymous'") + ddl should include("SCRIPT AS (DATEDIFF") + ddl should include("STRUCT FIELDS") + ddl should include("PRIMARY KEY (id)") + ddl should include("PARTITION BY birthdate (MONTH)") + } + + // --------------------------------------------------------------------------- + // CREATE TABLE IF NOT EXISTS + // --------------------------------------------------------------------------- + + it should "create a simple table and allow re-creation with IF NOT EXISTS" in { + val sql = + """CREATE TABLE IF NOT EXISTS accounts ( + | id INT NOT NULL, + | owner VARCHAR, + | balance DECIMAL(10,2) + |) WITH ( + | number_of_shards = 1, + | number_of_replicas = 0 + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + assertDdl(client.run(sql).futureValue) // second call should succeed + } + + // --------------------------------------------------------------------------- + // CREATE OR REPLACE TABLE AS SELECT + // --------------------------------------------------------------------------- + + it should "create or replace a table from a SELECT query" in { + val createSource = + """CREATE TABLE IF NOT EXISTS accounts_src ( + | id INT NOT NULL, + | name VARCHAR, + | active BOOLEAN + |);""".stripMargin + + assertDdl(client.run(createSource).futureValue) + + val insertSource = + """INSERT INTO accounts_src (id, name, active) VALUES + | (1, 'Alice', true), + | (2, 'Bob', false), + | (3, 'Chloe', true);""".stripMargin + + assertDml(client.run(insertSource).futureValue) + + val createOrReplace = + """CREATE OR REPLACE TABLE users_cr AS + |SELECT id, name FROM accounts_src WHERE active = true;""".stripMargin + + assertDdl(client.run(createOrReplace).futureValue) + + // Vérification via SHOW TABLE + val table = assertShowTable(client.run("SHOW TABLE users_cr").futureValue) + table.ddl should include("CREATE TABLE users_cr") + table.ddl should include("id INT") + table.ddl should include("name VARCHAR") + } + + // --------------------------------------------------------------------------- + // DROP TABLE + // --------------------------------------------------------------------------- + + it should "drop a table if it exists" in { + val create = + """CREATE TABLE IF NOT EXISTS tmp_drop ( + | id INT NOT NULL, + | value VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val drop = + """DROP TABLE IF EXISTS tmp_drop;""".stripMargin + + assertDdl(client.run(drop).futureValue) + } + + // --------------------------------------------------------------------------- + // TRUNCATE TABLE + // --------------------------------------------------------------------------- + + it should "truncate a table" in { + val create = + """CREATE TABLE IF NOT EXISTS tmp_truncate ( + | id INT NOT NULL, + | value VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO tmp_truncate (id, value) VALUES + | (1, 'a'), + | (2, 'b'), + | (3, 'c');""".stripMargin + + assertDml(client.run(insert).futureValue) + + val truncate = "TRUNCATE TABLE tmp_truncate;" + assertDdl(client.run(truncate).futureValue) + + // Vérification : SELECT doit renvoyer 0 lignes + val select = client.run("SELECT * FROM tmp_truncate").futureValue + assertSelectResult(select) + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — ADD COLUMN + // --------------------------------------------------------------------------- + + it should "add a column IF NOT EXISTS with DEFAULT" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter1 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter1 + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter1").futureValue) + table.ddl should include("age INT DEFAULT 0") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — RENAME COLUMN + // --------------------------------------------------------------------------- + + it should "rename a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter2 ( + | id INT NOT NULL, + | name VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter2 + | RENAME COLUMN name TO full_name;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter2").futureValue) + table.ddl should include("full_name VARCHAR") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — COMMENT + // --------------------------------------------------------------------------- + + it should "set and drop a column COMMENT" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter3 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setComment = + """ALTER TABLE users_alter3 + | ALTER COLUMN IF EXISTS status SET COMMENT 'a description';""".stripMargin + + val dropComment = + """ALTER TABLE users_alter3 + | ALTER COLUMN IF EXISTS status DROP COMMENT;""".stripMargin + + assertDdl(client.run(setComment).futureValue) + assertDdl(client.run(dropComment).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter3").futureValue) + table.ddl should not include "COMMENT 'a description'" + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — DEFAULT + // --------------------------------------------------------------------------- + + it should "set and drop a DEFAULT value on a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter4 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setDefault = + """ALTER TABLE users_alter4 + | ALTER COLUMN status SET DEFAULT 'active';""".stripMargin + + val dropDefault = + """ALTER TABLE users_alter4 + | ALTER COLUMN status DROP DEFAULT;""".stripMargin + + assertDdl(client.run(setDefault).futureValue) + assertDdl(client.run(dropDefault).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter4").futureValue) + table.ddl should not include "DEFAULT 'active'" + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — STRUCT SET FIELDS + // --------------------------------------------------------------------------- + + it should "alter a STRUCT column with SET FIELDS" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter5 ( + | id INT NOT NULL, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter5 + | ALTER COLUMN profile SET FIELDS ( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)), + | reputation DOUBLE DEFAULT 0.0 + | );""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter5").futureValue) + table.ddl should include("reputation DOUBLE DEFAULT 0.0") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — NOT NULL + // --------------------------------------------------------------------------- + + it should "set and drop NOT NULL on a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter6 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setNotNull = + """ALTER TABLE users_alter6 + | ALTER COLUMN status SET NOT NULL;""".stripMargin + + val dropNotNull = + """ALTER TABLE users_alter6 + | ALTER COLUMN status DROP NOT NULL;""".stripMargin + + assertDdl(client.run(setNotNull).futureValue) + assertDdl(client.run(dropNotNull).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter6").futureValue) + table.ddl should not include "NOT NULL" + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — TYPE CHANGE (UNSAFE → migration) + // --------------------------------------------------------------------------- + + it should "change a column data type (UNSAFE → migration)" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter7 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter7 + | ALTER COLUMN status SET DATA TYPE BIGINT;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter7").futureValue) + table.ddl should include("status BIGINT") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE — multi-alteration + // --------------------------------------------------------------------------- + + it should "apply multiple alterations in a single ALTER TABLE statement" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter8 ( + | id INT NOT NULL, + | name VARCHAR, + | status VARCHAR, + | profile STRUCT FIELDS( + | description VARCHAR, + | visibility BOOLEAN + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter8 ( + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + | RENAME COLUMN name TO full_name, + | ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + | ALTER COLUMN IF EXISTS profile SET FIELDS ( + | description VARCHAR DEFAULT 'N/A', + | visibility BOOLEAN DEFAULT true + | ) + |);""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter8").futureValue) + + table.ddl should include("age INT DEFAULT 0") + table.ddl should include("full_name VARCHAR") + table.ddl should include("status VARCHAR DEFAULT 'active'") + table.ddl should include("description VARCHAR DEFAULT 'N/A'") + table.ddl should include("visibility BOOLEAN DEFAULT true") + } + + // =========================================================================== + // 3. DML — INSERT / UPDATE / DELETE + // =========================================================================== + + behavior of "DML statements" + + // --------------------------------------------------------------------------- + // INSERT + // --------------------------------------------------------------------------- + + it should "insert rows into a table and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_users ( + | id INT NOT NULL, + | name VARCHAR, + | age INT + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_users (id, name, age) VALUES + | (1, 'Alice', 30), + | (2, 'Bob', 40), + | (3, 'Chloe', 25);""".stripMargin + + val res = client.run(insert).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.inserted shouldBe 3 + + // Vérification via SELECT + val select = client.run("SELECT * FROM dml_users ORDER BY id ASC").futureValue + assertSelectResult(select) + } + + // --------------------------------------------------------------------------- + // UPDATE + // --------------------------------------------------------------------------- + + it should "update rows using UPDATE ... WHERE and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_accounts ( + | id INT NOT NULL, + | owner VARCHAR, + | balance DECIMAL(10,2) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_accounts (id, owner, balance) VALUES + | (1, 'Alice', 100.00), + | (2, 'Bob', 50.00), + | (3, 'Chloe', 75.00);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val update = + """UPDATE dml_accounts + |SET balance = balance + 25 + |WHERE owner = 'Alice';""".stripMargin + + val res = client.run(update).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.updated should be >= 1L + + // Vérification via SELECT + val select = + """SELECT owner, balance + |FROM dml_accounts + |WHERE owner = 'Alice';""".stripMargin + + val q = client.run(select).futureValue + assertSelectResult(q) + } + + // --------------------------------------------------------------------------- + // DELETE + // --------------------------------------------------------------------------- + + it should "delete rows using DELETE ... WHERE and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_logs ( + | id INT NOT NULL, + | level VARCHAR, + | message VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_logs (id, level, message) VALUES + | (1, 'INFO', 'started'), + | (2, 'ERROR', 'failed'), + | (3, 'INFO', 'running');""".stripMargin + + assertDml(client.run(insert).futureValue) + + val delete = + """DELETE FROM dml_logs + |WHERE level = 'ERROR';""".stripMargin + + val res = client.run(delete).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.deleted shouldBe 1L + + // Vérification via SELECT + val select = + """SELECT id, level + |FROM dml_logs + |ORDER BY id ASC;""".stripMargin + + val q = client.run(select).futureValue + assertSelectResult(q) + } + + // --------------------------------------------------------------------------- + // INSERT + UPDATE + DELETE chain + // --------------------------------------------------------------------------- + + it should "support a full DML lifecycle: INSERT → UPDATE → DELETE" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_chain ( + | id INT NOT NULL, + | value INT + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_chain (id, value) VALUES + | (1, 10), + | (2, 20), + | (3, 30);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val update = + """UPDATE dml_chain + |SET value = value * 2 + |WHERE id IN (1, 3);""".stripMargin + + assertDml(client.run(update).futureValue) + + val delete = + """DELETE FROM dml_chain + |WHERE value > 40;""".stripMargin + + assertDml(client.run(delete).futureValue) + + // Vérification finale + val select = client.run("SELECT * FROM dml_chain ORDER BY id ASC").futureValue + assertSelectResult(select) + } + + // =========================================================================== + // 4. DQL — SELECT, JOIN, UNNEST, GROUP BY, HAVING, ORDER BY, LIMIT/OFFSET, + // functions, window functions, nested fields, UNION ALL + // =========================================================================== + + behavior of "DQL statements" + + // --------------------------------------------------------------------------- + // Setup for DQL tests: base table with nested fields + // --------------------------------------------------------------------------- + + it should "prepare DQL test data" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_users ( + | id INT NOT NULL, + | name VARCHAR, + | age INT, + | birthdate DATE, + | profile STRUCT FIELDS( + | city VARCHAR, + | followers INT + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_users (id, name, age, birthdate, profile) VALUES + | (1, 'Alice', 30, '1994-01-01', { "city": "Paris", "followers": 100 }), + | (2, 'Bob', 40, '1984-05-10', { "city": "Lyon", "followers": 50 }), + | (3, 'Chloe', 25, '1999-07-20', { "city": "Paris", "followers": 200 }), + | (4, 'David', 50, '1974-03-15', { "city": "Marseille", "followers": 10 }); + |""".stripMargin + + assertDml(client.run(insert).futureValue) + } + + // --------------------------------------------------------------------------- + // SELECT simple + alias + nested fields + // --------------------------------------------------------------------------- + + it should "execute a simple SELECT with aliases and nested fields" in { + val sql = + """SELECT id, + | name AS full_name, + | profile.city AS city, + | profile.followers AS followers + |FROM dql_users + |ORDER BY id ASC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // UNION ALL + // --------------------------------------------------------------------------- + + it should "execute a UNION ALL query" in { + val sql = + """SELECT id, name FROM dql_users WHERE age > 30 + |UNION ALL + |SELECT id, name FROM dql_users WHERE age <= 30;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // JOIN + UNNEST + // --------------------------------------------------------------------------- + + it should "execute a JOIN with UNNEST on array of structs" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_orders ( + | id INT NOT NULL, + | customer_id INT, + | items ARRAY + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_orders (id, customer_id, items) VALUES + | (1, 1, [ { "product": "A", "quantity": 2, "price": 10.0 }, + | { "product": "B", "quantity": 1, "price": 20.0 } ]), + | (2, 2, [ { "product": "C", "quantity": 3, "price": 5.0 } ]);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val sql = + """SELECT o.id, i.product, i.quantity + |FROM dql_orders o + |JOIN UNNEST(o.items) AS i + |ORDER BY o.id ASC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // WHERE — complex conditions + // --------------------------------------------------------------------------- + + it should "support WHERE with complex logical and comparison operators" in { + val sql = + """SELECT id, name, age + |FROM dql_users + |WHERE (age > 20 AND profile.followers >= 100) + | OR (profile.city = 'Lyon' AND age < 50) + |ORDER BY age DESC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // ORDER BY + LIMIT + OFFSET + // --------------------------------------------------------------------------- + + it should "support ORDER BY with multiple fields, LIMIT and OFFSET" in { + val sql = + """SELECT id, name, age + |FROM dql_users + |ORDER BY age DESC, name ASC + |LIMIT 2 OFFSET 1;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // GROUP BY + HAVING + // --------------------------------------------------------------------------- + + it should "support GROUP BY with aggregations and HAVING" in { + val sql = + """SELECT profile.city AS city, + | COUNT(*) AS cnt, + | AVG(age) AS avg_age + |FROM dql_users + |GROUP BY profile.city + |HAVING COUNT(*) >= 1 + |ORDER BY cnt DESC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Arithmetic, IN, BETWEEN, IS NULL, LIKE, RLIKE + // --------------------------------------------------------------------------- + + it should "support arithmetic, IN, BETWEEN, IS NULL, LIKE, RLIKE" in { + val sql = + """SELECT id, + | age + 10 AS age_plus_10, + | name + |FROM dql_users + |WHERE age BETWEEN 20 AND 50 + | AND name IN ('Alice', 'Bob', 'Chloe') + | AND name IS NOT NULL + | AND (name LIKE 'A%' OR name RLIKE '.*o.*');""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // CAST, TRY_CAST, SAFE_CAST, operator :: + // --------------------------------------------------------------------------- + + it should "support CAST, TRY_CAST, SAFE_CAST and the :: operator" in { + val sql = + """SELECT id, + | age::BIGINT AS age_bigint, + | CAST(age AS DOUBLE) AS age_double, + | TRY_CAST('123' AS INT) AS try_cast_ok, + | SAFE_CAST('abc' AS INT) AS safe_cast_null + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Numeric + Trigonometric functions + // --------------------------------------------------------------------------- + + it should "support numeric and trigonometric functions" in { + val sql = + """SELECT id, + | ABS(age) AS abs_age, + | CEIL(age / 3.0) AS ceil_div, + | FLOOR(age / 3.0) AS floor_div, + | ROUND(age / 3.0, 2) AS round_div, + | SQRT(age) AS sqrt_age, + | POWER(age, 2) AS pow_age, + | LOG(age) AS log_age, + | LOG10(age) AS log10_age, + | EXP(age) AS exp_age, + | SIN(age) AS sin_age, + | COS(age) AS cos_age, + | TAN(age) AS tan_age + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // String functions + // --------------------------------------------------------------------------- + + it should "support string functions" in { + val sql = + """SELECT id, + | CONCAT(name, '_suffix') AS name_concat, + | SUBSTRING(name, 1, 2) AS name_sub, + | LOWER(name) AS name_lower, + | UPPER(name) AS name_upper, + | TRIM(name) AS name_trim, + | LENGTH(name) AS name_len, + | REPLACE(name, 'A', 'X') AS name_repl, + | LEFT(name, 1) AS name_left, + | RIGHT(name, 1) AS name_right, + | REVERSE(name) AS name_rev + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Date / Time functions + // --------------------------------------------------------------------------- + + it should "support date and time functions" in { + val sql = + """SELECT id, + | YEAR(birthdate) AS year_b, + | MONTH(birthdate) AS month_b, + | DAY(birthdate) AS day_b, + | DATE_ADD(birthdate, INTERVAL 1 DAY) AS plus_one_day, + | DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Geospatial functions + // --------------------------------------------------------------------------- + + it should "support geospatial functions POINT and ST_DISTANCE" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_geo ( + | id INT NOT NULL, + | lon DOUBLE, + | lat DOUBLE + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_geo (id, lon, lat) VALUES + | (1, 2.3522, 48.8566), + | (2, 4.8357, 45.7640);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val sql = + """SELECT id, + | POINT(lon, lat) AS point, + | ST_DISTANCE(POINT(lon, lat), POINT(2.3522, 48.8566)) AS dist_paris + |FROM dql_geo;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Window functions + // --------------------------------------------------------------------------- + + it should "support window functions with OVER clause" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_sales ( + | id INT NOT NULL, + | product VARCHAR, + | customer VARCHAR, + | amount DOUBLE, + | ts TIMESTAMP + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_sales (id, product, customer, amount, ts) VALUES + | (1, 'A', 'C1', 10.0, '2024-01-01T10:00:00Z'), + | (2, 'A', 'C2', 20.0, '2024-01-01T11:00:00Z'), + | (3, 'B', 'C1', 30.0, '2024-01-01T12:00:00Z'), + | (4, 'A', 'C3', 40.0, '2024-01-01T13:00:00Z');""".stripMargin + + assertDml(client.run(insert).futureValue) + + val sql = + """SELECT + | product, + | customer, + | amount, + | SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + | COUNT(*) OVER (PARTITION BY product) AS cnt_per_product, + | FIRST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS first_amount, + | LAST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS last_amount, + | ARRAY_AGG(amount) OVER (PARTITION BY product ORDER BY ts ASC LIMIT 10) AS amounts_array + |FROM dql_sales + |ORDER BY product, ts;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // =========================================================================== + // 5. PIPELINES — CREATE / ALTER / DROP + // =========================================================================== + + behavior of "PIPELINE statements" + + // --------------------------------------------------------------------------- + // CREATE OR REPLACE PIPELINE with full processor set + // --------------------------------------------------------------------------- + + it should "create or replace a pipeline with SET, SCRIPT, DATE_INDEX_NAME processors" in { + val sql = + """CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ), + | SCRIPT ( + | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + | lang = "painless", + | source = "def p1 = ctx.birthdate; def p2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (p1 == null) ? null : ChronoUnit.YEARS.between(p1, p2)", + | ignore_failure = true + | ), + | SET ( + | field = "ingested_at", + | if = "ctx.ingested_at == null", + | description = "DEFAULT _ingest.timestamp", + | ignore_failure = true, + | value = "_ingest.timestamp" + | ), + | SCRIPT ( + | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", + | lang = "painless", + | source = "def p1 = ctx.profile?.join_date; def p2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (p1 == null) ? null : ChronoUnit.DAYS.between(p1, p2)", + | ignore_failure = true + | ), + | DATE_INDEX_NAME ( + | field = "birthdate", + | index_name_prefix = "users-", + | date_formats = ["yyyy-MM"], + | ignore_failure = true, + | date_rounding = "M", + | description = "PARTITION BY birthdate (MONTH)", + | separator = "-" + | ), + | SET ( + | field = "_id", + | description = "PRIMARY KEY (id)", + | ignore_failure = false, + | ignore_empty_value = false, + | value = "{{id}}" + | ) + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + } + + // --------------------------------------------------------------------------- + // ALTER PIPELINE — ADD PROCESSOR, DROP PROCESSOR + // --------------------------------------------------------------------------- + + it should "alter an existing pipeline by adding and dropping processors" in { + val sql = + """ALTER PIPELINE IF EXISTS user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | if = "ctx.status == null", + | description = "status DEFAULT 'active'", + | ignore_failure = true, + | value = "active" + | ), + | DROP PROCESSOR SET (_id) + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + } + + // --------------------------------------------------------------------------- + // DROP PIPELINE + // --------------------------------------------------------------------------- + + it should "drop a pipeline" in { + val sql = "DROP PIPELINE IF EXISTS user_pipeline;" + assertDdl(client.run(sql).futureValue) + } + + // =========================================================================== + // 6. TEMPLATES — CREATE / ALTER / DROP + // =========================================================================== + + behavior of "TEMPLATE statements" + + it should "create, alter and drop index templates via SQL if supported" in { + val create = + """CREATE TEMPLATE logs_template + |PATTERN 'logs-*' + |WITH ( + | number_of_shards = 1, + | number_of_replicas = 1 + |);""".stripMargin + + val createRes = client.run(create).futureValue + + // Certains backends Elasticsearch ne supportent pas les templates via SQL + if (createRes.isSuccess) { + assertDdl(createRes) + + val alter = + """ALTER TEMPLATE logs_template + |SET ( number_of_replicas = 2 );""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val drop = "DROP TEMPLATE logs_template;" + assertDdl(client.run(drop).futureValue) + + } else { + log.warn("Templates not supported by this backend, skipping template tests.") + } + } + + // =========================================================================== + // 7. ERRORS — parsing errors, unsupported SQL + // =========================================================================== + + behavior of "SQL error handling" + + // --------------------------------------------------------------------------- + // Parsing error + // --------------------------------------------------------------------------- + + it should "return a parsing error for invalid SQL" in { + val invalidSql = "CREAT TABL missing_keyword" + val res = client.run(invalidSql).futureValue + + res.isFailure shouldBe true + res.toEither.left.get.message should include("Error parsing schema DDL statement") + } + + // --------------------------------------------------------------------------- + // Unsupported SQL + // --------------------------------------------------------------------------- + + it should "return an error for unsupported SQL statements" in { + val unsupportedSql = "GRANT SELECT ON users TO user1" + val res = client.run(unsupportedSql).futureValue + + res.isFailure shouldBe true + res.toEither.left.get.message should include("Unsupported SQL statement") + } + +} From 327deb16eac648345a47cd494f24c0d659882116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 09:23:53 +0100 Subject: [PATCH 45/95] to fix doc values with inner hits --- .../elastic/sql/bridge/ElasticBridge.scala | 11 ++++------- .../app/softnetwork/elastic/sql/bridge/package.scala | 7 +------ .../elastic/sql/bridge/ElasticBridge.scala | 11 ++++------- .../app/softnetwork/elastic/sql/bridge/package.scala | 7 +------ 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 394aecaf..c013e82c 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -77,13 +77,10 @@ case class ElasticBridge(filter: ElasticFilter) { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.map { source => - (n.path.split('.').toSeq ++ Seq(source)).mkString(".") - }.toArray - ) + inner = inner.docValueFields( + n.sources.map { source => + (n.path.split('.').toSeq ++ Seq(source)).mkString(".") + } ) } inner diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index b6e767dc..d8de91e1 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -349,12 +349,7 @@ package object bridge { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.toArray - ) - ) + inner = inner.docValueFields(n.sources) } inner } diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 799a3509..ad5ad8ad 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -72,13 +72,10 @@ case class ElasticBridge(filter: ElasticFilter) { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.map { source => - (n.path.split('.').toSeq ++ Seq(source)).mkString(".") - }.toArray - ) + inner = inner.docValueFields( + n.sources.map { source => + (n.path.split('.').toSeq ++ Seq(source)).mkString(".") + } ) } inner diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 4d616455..99e37c2c 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -343,12 +343,7 @@ package object bridge { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.toArray - ) - ) + inner = inner.docValueFields(n.sources) } inner } From ddcd9b36a57f18b35b70dad7cd4ad48d4bc6f7fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 09:24:47 +0100 Subject: [PATCH 46/95] implements Updatable with arithmetic expressions --- .../operator/math/ArithmeticExpression.scala | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 24e199c9..6f0f439c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -27,7 +27,8 @@ case class ArithmeticExpression( right: PainlessScript, group: Boolean = false ) extends TransformFunction[SQLNumeric, SQLNumeric] - with BinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] { + with BinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] + with Updateable { override def fun: Option[ArithmeticOperator] = Some(operator) @@ -129,4 +130,23 @@ case class ArithmeticExpression( expr } + override def update(request: query.SingleSearch): ArithmeticExpression = { + (left, right) match { + case (l: Updateable, r: Updateable) => + this.copy( + left = l.update(request).asInstanceOf[PainlessScript], + right = r.update(request).asInstanceOf[PainlessScript] + ) + case (l: Updateable, _) => + this.copy( + left = l.update(request).asInstanceOf[PainlessScript] + ) + case (_, r: Updateable) => + this.copy( + right = r.update(request).asInstanceOf[PainlessScript] + ) + case _ => + this + } + } } From c14283b33ac1b6455edd82d9289f7964eaf85aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 09:25:56 +0100 Subject: [PATCH 47/95] remove unnecessary printing --- .../softnetwork/elastic/sql/bridge/ElasticAggregation.scala | 5 ----- .../softnetwork/elastic/sql/bridge/ElasticAggregation.scala | 5 ----- 2 files changed, 10 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 52405a90..f3cfbb3a 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -22,7 +22,6 @@ import app.softnetwork.elastic.sql.query.{ Asc, BucketIncludesExcludes, BucketNode, - BucketTree, Criteria, Desc, Field, @@ -350,10 +349,6 @@ object ElasticAggregation { nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long): Seq[Aggregation] = { - val trees = BucketTree(buckets.flatMap(_.headOption)) - println( - s"[DEBUG] buildBuckets called with buckets: \n$trees" - ) for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index c54c4451..806fa8ce 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -22,7 +22,6 @@ import app.softnetwork.elastic.sql.query.{ Asc, BucketIncludesExcludes, BucketNode, - BucketTree, Criteria, Desc, Field, @@ -350,10 +349,6 @@ object ElasticAggregation { nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long): Seq[Aggregation] = { - val trees = BucketTree(buckets.flatMap(_.headOption)) - println( - s"[DEBUG] buildBuckets called with buckets: \n$trees" - ) for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => From ccb5ee281dbedc9d9b402e383aadbdbc41e48640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 09:28:55 +0100 Subject: [PATCH 48/95] to fix multi queries (UNION ALL) --- .../app/softnetwork/elastic/client/ElasticConversion.scala | 5 ++++- .../scala/app/softnetwork/elastic/sql/operator/package.scala | 2 +- .../scala/app/softnetwork/elastic/sql/parser/Parser.scala | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 8ece3037..c15dddeb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -64,7 +64,10 @@ trait ElasticConversion { fieldAliases: Map[String, String], aggregations: Map[String, ClientAggregation] ): Try[Seq[Map[String, Any]]] = { - val json = mapper.readTree(results) + var json = mapper.readTree(results) + if (json.has("responses")) { + json = json.get("responses") + } // Check if it's a multi-search response (array of responses) if (json.isArray) { parseMultiSearchResponse(json, fieldAliases, aggregations) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala index b698a751..5569800c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -74,7 +74,7 @@ package object operator { case object AND extends Expr("AND") with PredicateOperator case object OR extends Expr("OR") with PredicateOperator - case object UNION extends Expr("UNION") with Operator with TokenRegex + case object UNION extends Expr("UNION ALL") with Operator with TokenRegex sealed trait ElasticOperator extends Operator with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index a0062bc7..c8fb76c1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -61,7 +61,7 @@ object Parser with LimitParser { def single: PackratParser[SingleSearch] = { - phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.? ~ onConflict.?) ^^ { + select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.? ~ onConflict.? ^^ { case s ~ f ~ w ~ g ~ h ~ o ~ l ~ oc => SingleSearch(s, f, w, g, h, o, l, onConflict = oc).update() } From cb52628fd130bd873bd85c07f5b8109cf67d2dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 10:19:03 +0100 Subject: [PATCH 49/95] to fix script fields within results --- .../elastic/client/ElasticConversion.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index c15dddeb..b4665f43 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -184,7 +184,12 @@ trait ElasticConversion { val source = extractSource(hit, fieldAliases) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) - globalMetrics ++ allTopHits ++ source ++ metadata ++ innerHits + val fieldsNode = Option(hit.path("fields")) + .filter(!_.isMissingNode) + val fields = fieldsNode + .map(jsonNodeToMap(_, fieldAliases)) + .getOrElse(Map.empty) + globalMetrics ++ allTopHits ++ source ++ metadata ++ innerHits ++ fields } case _ => @@ -202,7 +207,12 @@ trait ElasticConversion { val source = extractSource(hit, fieldAliases) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) - source ++ metadata ++ innerHits + val fieldsNode = Option(hit.path("fields")) + .filter(!_.isMissingNode) + val fields = fieldsNode + .map(jsonNodeToMap(_, fieldAliases)) + .getOrElse(Map.empty) + source ++ metadata ++ innerHits ++ fields } } From 84f5887b9d4e075d9ec8fd1321d319a0f1982762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 10:57:40 +0100 Subject: [PATCH 50/95] to fix painless scripts for substring, left and right string functions --- .../elastic/sql/function/string/package.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 4629692c..894a74c1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -163,11 +163,11 @@ package object string { callArgs match { // SUBSTRING(expr, start, length) case List(arg0, _, _) => - s"$arg0.substring(${start - 1}, Math.min(${start - 1 + length.get}, $arg0.length()))" + s"((String)$arg0).substring(${start - 1}, (int)Math.min(${start - 1 + length.get}, ((String)$arg0).length()))" // SUBSTRING(expr, start) case List(arg0, arg1) => - s"$arg0.substring(Math.min(${start - 1}, $arg0.length() - 1))" + s"((String)$arg0).substring((int)Math.min(${start - 1}, ((String)$arg0).length() - 1))" case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") } @@ -244,7 +244,7 @@ package object string { ): String = { callArgs match { case List(arg0, arg1) => - s"$arg0.substring(0, Math.min($arg1, $arg0.length()))" + s"((String)$arg0).substring(0, (int)Math.min($arg1, ((String)$arg0).length()))" case _ => throw new IllegalArgumentException("LEFT requires 2 arguments") } } @@ -273,7 +273,8 @@ package object string { callArgs match { case List(arg0, arg1) => if (length == 0) "" - else s"""$arg0.substring($arg0.length() - Math.min($arg1, $arg0.length()))""" + else + s"""((String)$arg0).substring(((String)$arg0).length() - (int)Math.min($arg1, ((String)$arg0).length()))""" case _ => throw new IllegalArgumentException("RIGHT requires 2 arguments") } } From 971be4124cb8010b42e80c8089a450a4df3849b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 17:01:10 +0100 Subject: [PATCH 51/95] to fix painless scripts with system functions --- .../elastic/sql/function/time/package.scala | 4 +++- .../elastic/sql/parser/Parser.scala | 2 +- .../sql/parser/function/time/package.scala | 22 +++++++++---------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index acc58ea7..589023f6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -129,8 +129,10 @@ package object time { override def baseType: SQLType = SQLTypes.Time } - sealed trait SystemFunction extends Function { + sealed trait SystemFunction extends FunctionWithIdentifier { override def system: Boolean = true + + override def identifier: Identifier = Identifier(this) } sealed trait CurrentFunction extends SystemFunction with PainlessScript with DateMathScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index c8fb76c1..0e88a224 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -817,7 +817,7 @@ trait Parser fi.identifier.withFunctions(f ++ fi.identifier.functions) case _ => Identifier(f) } - case Some(id) => id.withFunctions(id.functions ++ f) + case Some(id) => id.withFunctions(f ++ id.functions) } }) >> cast diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index aead8076..cd8525d9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -35,30 +35,30 @@ package object time { def parens: PackratParser[List[Delimiter]] = start ~ end ^^ { case s ~ e => s :: e :: Nil } - def current_date: PackratParser[CurrentFunction] = + def current_date: PackratParser[Identifier] = CurrentDate.regex ~ parens.? ^^ { case _ ~ p => - CurrentDate(p.isDefined) + CurrentDate(p.isDefined).identifier } - def current_time: PackratParser[CurrentFunction] = + def current_time: PackratParser[Identifier] = CurrentTime.regex ~ parens.? ^^ { case _ ~ p => - CurrentTime(p.isDefined) + CurrentTime(p.isDefined).identifier } - def current_timestamp: PackratParser[CurrentFunction] = + def current_timestamp: PackratParser[Identifier] = CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => - CurrentTimestamp(p.isDefined) + CurrentTimestamp(p.isDefined).identifier } - def now: PackratParser[CurrentFunction] = Now.regex ~ parens.? ^^ { case _ ~ p => - Now(p.isDefined) + def now: PackratParser[Identifier] = Now.regex ~ parens.? ^^ { case _ ~ p => + Now(p.isDefined).identifier } - def today: PackratParser[CurrentFunction] = Today.regex ~ parens.? ^^ { case _ ~ p => - Today(p.isDefined) + def today: PackratParser[Identifier] = Today.regex ~ parens.? ^^ { case _ ~ p => + Today(p.isDefined).identifier } - private[this] def current_function: PackratParser[CurrentFunction] = + private[this] def current_function: PackratParser[Identifier] = current_date | current_time | current_timestamp | now | today def currentFunctionWithIdentifier: PackratParser[Identifier] = From 682e1cd16ec9be55675b4e5ac2de0bcac83b1ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 19:34:34 +0100 Subject: [PATCH 52/95] to fix painless scripts with temporal function to apply first --- .../app/softnetwork/elastic/sql/package.scala | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 78d3169d..fbe7cf3b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -988,7 +988,31 @@ package object sql { else Option(s"(ctx.$path == null ? $nullValue : ctx.$path${painlessMethods.mkString("")})") + def originalType: SQLType = + if (name.trim.nonEmpty) SQLTypes.Any + else this.baseType + override def painless(context: Option[PainlessContext]): String = { + val orderedFunctions = FunctionUtils.transformFunctions(this).reverse + var currType = this.originalType + currType match { + case SQLTypes.Any => + orderedFunctions.headOption match { + case Some(f: TransformFunction[_, _]) => + f.in match { + case SQLTypes.Temporal => // the first function to apply required a Temporal as input type + context match { + case Some(_) => + // compatible ES6+ + this.addPainlessMethod(".toInstant().atZone(ZoneId.of('Z'))") + case _ => // do nothing + } + case _ => // do nothing + } + case _ => // do nothing + } + case _ => // do nothing + } val base = context match { case Some(ctx) => @@ -999,7 +1023,6 @@ package object sql { else paramName } - val orderedFunctions = FunctionUtils.transformFunctions(this).reverse var expr = base orderedFunctions.zipWithIndex.foreach { case (f, idx) => f match { @@ -1007,6 +1030,7 @@ package object sql { case f: PainlessScript => expr = s"$expr${f.painless(context)}" case f => expr = f.toSQL(expr) // fallback } + currType = f.out } expr } From cc00710a3bad10aad3508643e53fbf36d3528574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 5 Jan 2026 22:27:17 +0100 Subject: [PATCH 53/95] to fix painless script for diff function --- .../elastic/sql/function/package.scala | 16 +++++++++----- .../elastic/sql/function/time/package.scala | 21 +++++++++++++++++-- .../app/softnetwork/elastic/sql/package.scala | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 4f6982fc..dfc50212 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -285,13 +285,19 @@ package object function { ctx.addParam(LiteralParam(ret)) } case identifier: Identifier => - identifier.baseType match { - case SQLTypes.Any => // in painless context, Any is ZonedDateTime - out match { + identifier.originalType match { + case SQLTypes.Any if !ctx.isProcessor => + in match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + identifier.addPainlessMethod(".toInstant().atZone(ZoneId.of('Z'))") case SQLTypes.Date => - identifier.addPainlessMethod(".toLocalDate()") + identifier.addPainlessMethod( + ".toInstant().atZone(ZoneId.of('Z')).toLocalDate()" + ) case SQLTypes.Time => - identifier.addPainlessMethod(".toLocalTime()") + identifier.addPainlessMethod( + ".toInstant().atZone(ZoneId.of('Z')).toLocalTime()" + ) case _ => } case _ => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 589023f6..4a96eaee 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -418,8 +418,25 @@ package object time { override def toSQL(base: String): String = s"$sql(${start.sql}, ${end.sql}, ${unit.sql})" - override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = - s"${unit.painless(context)}${DateDiff.painless(context)}(${callArgs.mkString(", ")})" + override def in: SQLType = SQLTypes.Date + + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + val ret = + s"Long.valueOf(${unit.painless(context)}${DateDiff.painless(context)}(${callArgs.mkString(", ")}))" + context match { + case Some(ctx) + if ctx.isProcessor => // to fix bug in painless script processor context with elasticsearch v6 + ctx.addParam(LiteralParam(ret)) match { + case Some(p) => return p + case _ => + } + case _ => + } + ret + } } case object DateAdd extends Expr("DATE_ADD") with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index fbe7cf3b..7f31b75e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1005,6 +1005,7 @@ package object sql { case Some(_) => // compatible ES6+ this.addPainlessMethod(".toInstant().atZone(ZoneId.of('Z'))") + currType = SQLTypes.Timestamp case _ => // do nothing } case _ => // do nothing From 61cd6f56bce18797531498294a28e3b7dc51138c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 06:53:12 +0100 Subject: [PATCH 54/95] to fix windowing with scroll api --- .../main/scala/app/softnetwork/elastic/client/ScrollApi.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 26089bc6..b752a7f7 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -150,7 +150,7 @@ trait ScrollApi extends ElasticClientHelpers { // Single search case single: SingleSearch => - if (single.windowFunctions.nonEmpty) + if (single.windowFunctions.exists(_.isWindowing)) return scrollWithWindowEnrichment(single, config) val elasticQuery = From cf036716147cae31ba40003e95886660142c7def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 07:47:31 +0100 Subject: [PATCH 55/95] to fix painless scripts with math functions --- .../softnetwork/elastic/sql/function/math/package.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 79752d8a..97fb54f5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -82,6 +82,13 @@ package object math { override def identifier: Identifier = Identifier(this) + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + val ret = super.toPainlessCall(callArgs, context) + s"Double.valueOf($ret)" + } } case class MathematicalFunctionWithOp( @@ -120,7 +127,7 @@ package object math { override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = callArgs match { - case List(a, p) => s"${mathOp.painless(context)}(($a * $p) / $p)" + case List(a, p) => s"Long.valueOf(${mathOp.painless(context)}(($a * $p) / $p))" case _ => throw new IllegalArgumentException("Round function requires exactly one argument") } From 08e790090e4d82d0bd2ba53d3233d8d25cc75eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 08:30:33 +0100 Subject: [PATCH 56/95] to fix isWindowing for window functions --- .../softnetwork/elastic/sql/function/aggregate/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 89e2caa3..16bf57b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -113,7 +113,7 @@ package object aggregate { def window: Window def limit: Option[Limit] - override def isWindowing: Boolean = buckets.nonEmpty + override def isWindowing: Boolean = buckets.nonEmpty || orderBy.isDefined lazy val buckets: Seq[Bucket] = partitionBy.map(identifier => Bucket(identifier, None)) From 53caa7fbbd6a85e14b7fe2ba9d05aec27c28dbb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 10:47:26 +0100 Subject: [PATCH 57/95] to fix elastic conversion with nested / object field aliases --- .../elastic/client/ElasticConversion.scala | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index b4665f43..df7dd169 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -180,8 +180,20 @@ trait ElasticConversion { extractAllTopHits(aggs, fieldAliases, aggregations), aggregations ) - hits.map { hit => - val source = extractSource(hit, fieldAliases) + parseSimpleHits(hits, fieldAliases).map { row => + globalMetrics ++ allTopHits ++ row + } + /*hits.map { hit => + var source = extractSource(hit, fieldAliases) + fieldAliases.foreach(entry => { + val key = entry._1 + if (!source.contains(key)) { + findKeyValue(key, source) match { + case Some(value) => source += (entry._2 -> value) + case None => + } + } + }) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) val fieldsNode = Option(hit.path("fields")) @@ -190,13 +202,22 @@ trait ElasticConversion { .map(jsonNodeToMap(_, fieldAliases)) .getOrElse(Map.empty) globalMetrics ++ allTopHits ++ source ++ metadata ++ innerHits ++ fields - } + }*/ case _ => Seq.empty } } + def findKeyValue(path: String, map: Map[String, Any]): Option[Any] = { + val keys = path.split("\\.") + keys.foldLeft(Option(map): Option[Any]) { + case (Some(m: Map[_, _]), key) => + m.asInstanceOf[Map[String, Any]].get(key) + case _ => None + } + } + /** Parse simple hits (without aggregations) */ def parseSimpleHits( @@ -204,7 +225,15 @@ trait ElasticConversion { fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { hits.map { hit => - val source = extractSource(hit, fieldAliases) + var source = extractSource(hit, fieldAliases) + fieldAliases.foreach(entry => { + if (!source.contains(entry._2)) { + findKeyValue(entry._1, source) match { + case Some(value) => source += (entry._2 -> value) + case None => + } + } + }) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) val fieldsNode = Option(hit.path("fields")) From abf9b14f9056a918f12443dbedbb2ee44954fcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 10:48:11 +0100 Subject: [PATCH 58/95] to fix distance painless script --- .../app/softnetwork/elastic/sql/function/geo/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 2e80bc1c..c914deb7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -171,7 +171,7 @@ package object geo { } if (identifiers.nonEmpty) - s"($assignments ($nullCheck) ? null : $ret)" + s"$assignments ($nullCheck) ? null : $ret" else ret } From 937566e55eeb618e14574116315a0ecc6847da8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 14:39:28 +0100 Subject: [PATCH 59/95] to fix function with nested element(s) --- .../elastic/sql/bridge/ElasticAggregation.scala | 2 +- .../elastic/sql/bridge/ElasticAggregation.scala | 2 +- .../softnetwork/elastic/sql/function/package.scala | 8 +++++++- .../sql/operator/math/ArithmeticExpression.scala | 13 +++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index f3cfbb3a..72132c8d 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -275,7 +275,7 @@ object ElasticAggregation { throw new IllegalArgumentException(s"Unsupported aggregation type: $aggType") } - val nestedElement = identifier.nestedElement + val nestedElement = sqlAgg.nestedElement val nestedElements: Seq[NestedElement] = nestedElement.map(n => NestedElements.buildNestedTrees(Seq(n))).getOrElse(Nil) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 806fa8ce..792d3e0d 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -275,7 +275,7 @@ object ElasticAggregation { throw new IllegalArgumentException(s"Unsupported aggregation type: $aggType") } - val nestedElement = identifier.nestedElement + val nestedElement = sqlAgg.nestedElement val nestedElements: Seq[NestedElement] = nestedElement.map(n => NestedElements.buildNestedTrees(Seq(n))).getOrElse(Nil) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index dfc50212..9aba3e6a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -20,7 +20,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator -import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.query.{NestedElement, SingleSearch} package object function { @@ -33,11 +33,15 @@ package object function { } def expr: Token = _expr override def nullable: Boolean = expr.nullable + def functionNestedElement: Option[NestedElement] = None } trait FunctionWithIdentifier extends Function { def identifier: Identifier + override def functionNestedElement: Option[NestedElement] = + identifier.nestedElement.orElse(identifier.functionNestedElement) + override def shouldBeScripted: Boolean = identifier.shouldBeScripted } @@ -199,6 +203,8 @@ package object function { override def shouldBeScripted: Boolean = functions.exists(_.shouldBeScripted) + override def functionNestedElement: Option[NestedElement] = + functions.flatMap(_.functionNestedElement).headOption } trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 6f0f439c..c953dcf0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -20,6 +20,7 @@ import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} import app.softnetwork.elastic.sql.parser.Validator +import app.softnetwork.elastic.sql.query.NestedElement case class ArithmeticExpression( left: PainlessScript, @@ -149,4 +150,16 @@ case class ArithmeticExpression( this } } + + override def functionNestedElement: Option[NestedElement] = + (left, right) match { + case (l: Identifier, r: Identifier) => + l.nestedElement.orElse(r.nestedElement) + case (l: Identifier, _) => + l.nestedElement + case (_, r: Identifier) => + r.nestedElement + case _ => + None + } } From 356922a811d982619e20b52d7e1cece13b7eb8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 14:40:17 +0100 Subject: [PATCH 60/95] to fix client specifications with nested documents --- .../app/softnetwork/elastic/client/ElasticClientSpec.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 04cc8bd7..a2efa13c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -1251,6 +1251,9 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M | }, | "birthDate": { | "type": "date" + | }, + | "parentId": { + | "type": "keyword" | } | } | }, From ac41aba9de9f046e094883b16eb0aa18bef56850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 14:42:32 +0100 Subject: [PATCH 61/95] to fix selected fields with nested documents --- .../scala/app/softnetwork/elastic/sql/query/Select.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index 14d03d57..bfe8adf4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -93,7 +93,14 @@ case class Field( override def validate(): Either[String, Unit] = identifier.validate() - lazy val nested: Boolean = identifier.nested + def nestedElement: Option[NestedElement] = + identifier.nestedElement + .orElse( + identifier.functionNestedElement + ) + .orElse(this.functionNestedElement) + + lazy val nested: Boolean = nestedElement.isDefined lazy val path: String = identifier.path From 7f725fbe5a65406161a1d42c42eb93e3819c1c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 14:42:59 +0100 Subject: [PATCH 62/95] to fix windows enrichment conditions --- .../softnetwork/elastic/client/ScrollApi.scala | 16 ++++++++-------- .../softnetwork/elastic/client/SearchApi.scala | 10 ++++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index b752a7f7..74b3d44f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -19,12 +19,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.{Sink, Source} -import app.softnetwork.elastic.client.result.{ - ElasticError, - ElasticFailure, - ElasticResult, - ElasticSuccess -} +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll.{ ScrollConfig, ScrollMetrics, @@ -150,7 +145,11 @@ trait ScrollApi extends ElasticClientHelpers { // Single search case single: SingleSearch => - if (single.windowFunctions.exists(_.isWindowing)) + if ( + single.windowFunctions.exists(_.isWindowing) && (!single.select.fields.forall( + _.isAggregation + ) || single.scriptFields.nonEmpty) + ) return scrollWithWindowEnrichment(single, config) val elasticQuery = @@ -430,7 +429,8 @@ trait ScrollApi extends ElasticClientHelpers { scrollWithMetrics( ElasticQuery( baseQuery, - collection.immutable.Seq(baseQuery.sources: _*) + collection.immutable.Seq(baseQuery.sources: _*), + sql = Some(baseQuery.sql) ), baseQuery.fieldAliases, baseQuery.sqlAggregations, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 14aa804f..7977c791 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -93,7 +93,13 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { collection.immutable.Seq(single.sources: _*), sql = Some(query) ) - if (single.windowFunctions.exists(_.isWindowing) && single.groupBy.isEmpty) + if ( + single.windowFunctions.exists( + _.isWindowing + ) && single.groupBy.isEmpty && (!single.select.fields.forall( + _.isAggregation + ) || single.scriptFields.nonEmpty) + ) searchWithWindowEnrichment(single) else singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) @@ -1216,7 +1222,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { // - Same WHERE clause (to match base query filtering) request .copy( - select = request.select.copy(fields = request.windowFields), + select = request.select.copy(fields = request.windowFields.map(_.update(request))), groupBy = None, //request.groupBy.map(_.copy(buckets = request.windowBuckets)), orderBy = None, // Not needed for aggregations limit = None // Need all buckets From 18bccf0a39f950c253ec4b269a72bc7ae58b8244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 15:32:06 +0100 Subject: [PATCH 63/95] finalize SQL Gateway --- .../elastic/client/ElasticsearchVersion.scala | 7 + .../elastic/client/IndicesApi.scala | 257 ++++---- .../elastic/client/MappingConverter.scala | 73 +++ .../elastic/client/PipelineApi.scala | 32 +- .../elastic/client/SqlGateway.scala | 166 +++-- .../elastic/client/TemplateConverter.scala | 19 +- .../elastic/client/result/package.scala | 8 + .../elastic/client/jest/JestIndicesApi.scala | 1 + .../elastic/client/JestSqlGatewaySpec.scala | 10 + .../client/rest/RestHighLevelClientApi.scala | 1 + .../rest/RestHighLevelClientHelpers.scala | 14 + .../RestHighLevelClientSqlGatewaySpec.scala | 10 + .../client/rest/RestHighLevelClientApi.scala | 1 + .../rest/RestHighLevelClientHelpers.scala | 14 + .../RestHighLevelClientSqlGatewaySpec.scala | 10 + .../elastic/client/java/JavaClientApi.scala | 1 + .../client/JavaClientSqlGatewaySpec.scala | 8 + .../elastic/client/java/JavaClientApi.scala | 1 + .../client/JavaClientSqlGatewaySpec.scala | 8 + .../softnetwork/elastic/schema/package.scala | 76 ++- .../elastic/sql/function/package.scala | 22 +- .../app/softnetwork/elastic/sql/package.scala | 3 + .../elastic/sql/parser/Parser.scala | 44 +- .../elastic/sql/parser/type/package.scala | 6 +- .../softnetwork/elastic/sql/query/Where.scala | 2 +- .../elastic/sql/query/package.scala | 72 ++- .../elastic/sql/schema/package.scala | 577 +++++++++--------- .../elastic/sql/serialization/package.scala | 11 +- .../elastic/sql/type/SQLType.scala | 2 + .../elastic/sql/type/SQLTypeUtils.scala | 16 +- .../elastic/sql/type/SQLTypes.scala | 23 +- .../elastic/sql/parser/ParserSpec.scala | 59 +- .../client/SqlGatewayIntegrationSpec.scala | 475 ++++++++++---- 33 files changed, 1377 insertions(+), 652 deletions(-) create mode 100644 core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 8bc35f59..24e9ad72 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -105,6 +105,13 @@ object ElasticsearchVersion { major == 6 } + /** Check if version is ES 7 + */ + def isEs7(version: String): Boolean = { + val (major, _, _) = parse(version) + major == 7 + } + /** Check if Data Streams are supported (ES >= 7.9) */ def supportsDataStreams(version: String): Boolean = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 13522b18..71f4f1ff 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -26,10 +26,12 @@ import app.softnetwork.elastic.sql.query.{Delete, From, Insert, SingleSearch, Ta import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, Schema, TableAlias} import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + /** Index management API. * * This implementation provides: @@ -39,7 +41,7 @@ import scala.concurrent.{ExecutionContext, Future} * - Automatic retry for transient errors */ trait IndicesApi extends ElasticClientHelpers { - _: RefreshApi with PipelineApi with BulkApi with ScrollApi with VersionApi => + _: RefreshApi with PipelineApi with BulkApi with ScrollApi with VersionApi with TemplateApi => // ======================================================================== // PUBLIC METHODS @@ -172,41 +174,7 @@ trait IndicesApi extends ElasticClientHelpers { } val updatedMappings = - if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion)) { - mappings match { - case Some(m) => - val root = mapper.readTree(m).asInstanceOf[ObjectNode] - - if (root.has("properties") || root.has("_meta")) { - logger.info(s"Wrapping mappings with '_doc' type for ES version $elasticVersion") - - val doc = mapper.createObjectNode() - - // Move properties - if (root.has("properties")) { - doc.set("properties", root.get("properties")) - root.remove("properties") - } - - // Move _meta - if (root.has("_meta")) { - doc.set("_meta", root.get("_meta")) - root.remove("_meta") - } - - // Wrap into _doc - root.set("_doc", doc) - - Some(root.toString) - } else { - Some(m) - } - - case None => None - } - } else { - mappings - } + mappings.map(mapping => MappingConverter.convert(mapping, elasticVersion)) executeCreateIndex(index, settings, updatedMappings, aliases) match { case success @ ElasticSuccess(true) => @@ -232,13 +200,65 @@ trait IndicesApi extends ElasticClientHelpers { case ElasticSuccess(Some(idx)) => ElasticSuccess(idx.schema) case ElasticSuccess(None) => - logger.info(s"Index '$index' not found for schema loading") - ElasticFailure( - ElasticError.notFound( - index = index, - operation = "loadSchema" - ) - ) + logger.warn(s"Index '$index' not found for schema loading") + getTemplate(index) match { + case ElasticSuccess(Some(template)) => + logger.info(s"✅ Template '$index' found for schema loading") + var templateNode = mapper.readTree(template).asInstanceOf[ObjectNode] + if (templateNode.has("index_template")) { + // Index template + templateNode = templateNode.get("index_template").asInstanceOf[ObjectNode] + } + if (templateNode.has("template")) { + // Composable template + templateNode = templateNode.get("template").asInstanceOf[ObjectNode] + } + val root = mapper.createObjectNode() + if (templateNode.has("mappings")) { + root.set("mappings", templateNode.get("mappings")) + } + if (templateNode.has("settings")) { + root.set("settings", templateNode.get("settings")) + } + if (templateNode.has("aliases")) { + root.set("aliases", templateNode.get("aliases")) + } + loadIndexAsSchema(index, root.toString) match { + case ElasticSuccess(Some(idx)) => + ElasticSuccess(idx.schema) + case ElasticSuccess(None) => + logger.error( + s"❌ Failed to load schema from template for index '$index'" + ) + ElasticFailure( + ElasticError( + message = s"Failed to load schema from template for index '$index'", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("loadSchema") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to load schema from template for index '$index': ${error.message}" + ) + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } + case ElasticSuccess(None) => + logger.warn(s"Template '$index' not found for schema loading") + ElasticFailure( + ElasticError.notFound( + index = index, + operation = "loadSchema" + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to load template for schema of index '$index': ${error.message}" + ) + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } case ElasticFailure(error) => logger.error(s"❌ Failed to load schema for index '$index': ${error.message}") ElasticFailure(error.copy(operation = Some("loadSchema"))) @@ -270,75 +290,11 @@ trait IndicesApi extends ElasticClientHelpers { executeGetIndex(index) match { case ElasticSuccess(Some(json)) => logger.info(s"✅ Index '$index' retrieved successfully") - var tempIndex = Index(index, json) - tempIndex.defaultIngestPipelineName match { - case Some(pipeline) => - logger.info( - s"Index '$index' has default ingest pipeline '$pipeline'" - ) - getPipeline(pipeline) match { - case ElasticSuccess(Some(json)) => - logger.info( - s"✅ Ingest pipeline '$pipeline' for index '$index' exists" - ) - tempIndex = tempIndex.copy(defaultPipeline = Some(json)) - case ElasticSuccess(None) => - logger.warn( - s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" - ) - return ElasticFailure( - ElasticError( - message = - s"Default ingest pipeline '$pipeline' for index '$index' does not exist", - cause = None, - statusCode = Some(404), - index = Some(index), - operation = Some("getIndex") - ) - ) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" - ) - return ElasticFailure(error.copy(operation = Some("getIndex"))) - } - case None => // No default ingest pipeline - } - tempIndex.finalIngestPipelineName match { - case Some(pipeline) => - logger.info( - s"Index '$index' has final ingest pipeline '$pipeline'" - ) - getPipeline(pipeline) match { - case ElasticSuccess(Some(json)) => - logger.info( - s"✅ Ingest pipeline '$pipeline' for index '$index' exists" - ) - tempIndex = tempIndex.copy(finalPipeline = Some(json)) - case ElasticSuccess(None) => - logger.warn( - s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" - ) - return ElasticFailure( - ElasticError( - message = - s"Final ingest pipeline '$pipeline' for index '$index' does not exist", - cause = None, - statusCode = Some(404), - index = Some(index), - operation = Some("getIndex") - ) - ) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" - ) - return ElasticFailure(error.copy(operation = Some("getIndex"))) - } - case None => // No default ingest pipeline - } - ElasticSuccess(Some(tempIndex)) + loadIndexAsSchema(index, json) case ElasticSuccess(None) => + logger.warn(s"✅ Index '$index' not found") + ElasticSuccess(None) + case ElasticFailure(error) if error.statusCode.contains(404) => logger.info(s"✅ Index '$index' not found") ElasticSuccess(None) case failure @ ElasticFailure(error) => @@ -347,7 +303,77 @@ trait IndicesApi extends ElasticClientHelpers { } } + private def loadIndexAsSchema(index: String, json: String): ElasticResult[Option[Index]] = { + var tempIndex = Index(index, json) + tempIndex.defaultIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has default ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"✅ Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(defaultPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = s"Default ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + tempIndex.finalIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has final ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"✅ Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(finalPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = s"Final ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + ElasticSuccess(Some(tempIndex)) + } + /** Delete an index with the provided name. + * * @param index * - the name of the index to delete * @return @@ -606,6 +632,9 @@ trait IndicesApi extends ElasticClientHelpers { val existenceStr = if (exists) "exists" else "does not exist" logger.debug(s"✅ Index '$index' $existenceStr") success + case f: ElasticFailure if f.isNotFound => + logger.debug(s"✅ Index '$index' does not exist") + ElasticSuccess(false) case failure @ ElasticFailure(error) => logger.error(s"❌ Failed to check existence of index '$index': ${error.message}") failure @@ -1052,7 +1081,9 @@ trait IndicesApi extends ElasticClientHelpers { case Right(_) => parsed.toJson match { case Some(jsonNode) => - Right(Source.single(jsonNode.toString)) + val arrayNode = jsonNode.asInstanceOf[ArrayNode] + val docs: Seq[JsonNode] = arrayNode.elements().asScala.toSeq + Right(Source.fromIterator(() => docs.map(_.toString).toIterator)) case None => Left( ElasticError( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala new file mode 100644 index 00000000..de224a34 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.{Logger, LoggerFactory} + +object MappingConverter { + + private val logger: Logger = LoggerFactory.getLogger(getClass) + + private lazy val mapper: ObjectMapper = JacksonConfig.objectMapper + + def convert(mapping: String, elasticVersion: String): String = { + val root = mapper.readTree(mapping).asInstanceOf[ObjectNode] + mapper.writeValueAsString(convert(root, elasticVersion)) + } + + def convert(mapping: ObjectNode, elasticVersion: String): ObjectNode = { + if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mapping.has("_doc")) { + wrapDocType(mapping) + } else if ( + !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mapping.has("_doc") + ) { + unwrapDocType(mapping) + } else mapping + } + + private def wrapDocType(mappingsNode: ObjectNode): ObjectNode = { + if (!mappingsNode.has("_doc")) { + logger.info(s"Wrapping mappings from '_doc' type") + val root = mapper.createObjectNode() + // Wrap into _doc + root.set("_doc", mappingsNode) + root + } else { + mappingsNode + } + } + + private def unwrapDocType(mappingsNode: ObjectNode): ObjectNode = { + val root = mapper.createObjectNode() + + if (mappingsNode.has("_doc")) { + logger.info(s"Unwrapping mappings from '_doc' type") + val doc = mappingsNode.get("_doc").asInstanceOf[ObjectNode] + doc + .properties() + .forEach(entry => { + root.set(entry.getKey, entry.getValue) + }) + root + } else { + mappingsNode + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index bad60055..385f75a3 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -94,7 +94,21 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => processors = ddl.ddlPipeline.processors.map { processor => GenericProcessor( processorType = processor.processorType, - properties = processor.properties.filterNot(_._1 == "description") + properties = processor.properties + .filterNot(_._1 == "description") + .filterNot(_._1 == "separator") + .filterNot(_._1 == "ignore_empty_value") + ) + } + ) + createPipeline(ddl.name, pipeline.json) + } else if (ElasticsearchVersion.isEs7(elasticVersion)) { + val pipeline = ddl.ddlPipeline.copy( + processors = ddl.ddlPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "separator") ) } ) @@ -123,7 +137,21 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => processors = updatingPipeline.processors.map { processor => GenericProcessor( processorType = processor.processorType, - properties = processor.properties.filterNot(_._1 == "description") + properties = processor.properties + .filterNot(_._1 == "description") + .filterNot(_._1 == "separator") + .filterNot(_._1 == "ignore_empty_value") + ) + } + ) + updatePipeline(ddl.name, pipeline.json) + } else if (ElasticsearchVersion.isEs7(elasticVersion)) { + val pipeline = updatingPipeline.copy( + processors = updatingPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "separator") ) } ) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala index 4f0bdee4..201a257b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala @@ -33,6 +33,7 @@ import app.softnetwork.elastic.client.result.{ import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ AlterTable, + CreatePipeline, CreateTable, DdlStatement, Delete, @@ -52,7 +53,6 @@ import app.softnetwork.elastic.sql.query.{ Update } import app.softnetwork.elastic.sql.schema.{ - ColumnOptionSet, Impossible, IngestPipeline, IngestPipelineType, @@ -109,10 +109,22 @@ class DqlExecutor(api: ScrollApi with SearchApi, logger: Logger) extends Executo // SingleSearch → SCROLL // ============================ case single: SingleSearch => - logger.info(s"▶ Executing scroll search on index ${single.from.tables.mkString(",")}") - Future.successful( - ElasticSuccess(QueryStream(api.scroll(single))) - ) + single.limit match { + case Some(l) if l.offset.map(_.offset).getOrElse(0) > 0 => + logger.info(s"▶ Executing classic search on index ${single.from.tables.mkString(",")}") + api.searchAsync(single) map { + case ElasticSuccess(results) => + logger.info(s"✅ Search returned ${results.results.size} hits.") + ElasticSuccess(QueryStructured(results)) + case ElasticFailure(err) => + ElasticFailure(err.copy(operation = Some("dql"))) + } + case _ => + logger.info(s"▶ Executing scroll search on index ${single.from.tables.mkString(",")}") + Future.successful( + ElasticSuccess(QueryStream(api.scroll(single))) + ) + } // ============================ // MultiSearch → searchAsync @@ -445,7 +457,7 @@ class TableExecutor( logger.info( s"🔄 Merging existing index $indexName DDL with new DDL." ) - var updatedTable: Table = schema.merge(alter.statements) + val updatedTable: Table = schema.merge(alter.statements) // load default pipeline diff if needed val defaultPipelineDiff: Option[List[PipelineDiff]] = @@ -539,7 +551,20 @@ class TableExecutor( // ------------------------------------------------------------ case UnsafeReindex => logger.warn(s"⚠️ ALTER TABLE requires REINDEX for $indexName.") - migrateToNewSchema(indexName, schema, updatedTable, diff) match { + migrateToNewSchema( + indexName, + schema, + updatedTable.copy( + settings = schema.settings // remove settings that cannot be copied + - "uuid" + - "creation_date" + - "provided_name" + - "version" + - "default_pipeline" + - "final_pipeline" + ), + diff + ) match { case Right(result) => Future.successful(ElasticSuccess(DdlResult(result))) case Left(error) => Future.successful(ElasticFailure(error.copy(operation = Some("schema")))) @@ -613,7 +638,18 @@ class TableExecutor( api.loadSchema(from) match { case ElasticSuccess(fromSchema) => // we update the schema based on the DQL select clause - fromSchema.mergeWithSearch(single) + fromSchema + .mergeWithSearch(single) + .copy( + name = indexName, // set index name + settings = fromSchema.settings // remove settings that cannot be copied + - "uuid" + - "creation_date" + - "provided_name" + - "version" + - "default_pipeline" + - "final_pipeline" + ) case ElasticFailure(elasticError) => val error = ElasticError( message = @@ -641,7 +677,16 @@ class TableExecutor( // create index pipeline(s) if needed table.defaultPipeline match { case pipeline if pipeline.processors.nonEmpty => - createIndexPipeline(indexName, pipeline) match { + createIndexPipeline( + indexName, + CreatePipeline( + pipeline.name, + pipeline.pipelineType, + ifNotExists = true, + orReplace = false, + pipeline.processors + ) + ) match { case ElasticSuccess(_) => table = table.setDefaultPipelineName(pipelineName = pipeline.name) case ElasticFailure(elasticError) => @@ -656,7 +701,16 @@ class TableExecutor( table.finalPipeline match { case pipeline if pipeline.processors.nonEmpty => - createIndexPipeline(indexName, pipeline) match { + createIndexPipeline( + indexName, + CreatePipeline( + pipeline.name, + pipeline.pipelineType, + ifNotExists = true, + orReplace = false, + pipeline.processors + ) + ) match { case ElasticSuccess(_) => table = table.setFinalPipelineName(pipelineName = pipeline.name) case ElasticFailure(elasticError) => @@ -680,7 +734,7 @@ class TableExecutor( s"🚚 Populating index $indexName based on DQL." ) val query = - s"""INSERT INTO ${create.table} AS ${single.sql} ON DUPLICATE KEY NOTHING""" + s"""INSERT INTO ${table.name} AS ${single.sql} ON CONFLICT DO NOTHING""" val result = api.insertByQuery( index = indexName, query = query @@ -714,20 +768,20 @@ class TableExecutor( */ private def createIndexPipeline( indexName: String, - pipeline: IngestPipeline + statement: CreatePipeline ): ElasticResult[Boolean] = { logger.info( - s"🔧 Creating ${pipeline.pipelineType.name} ingesting pipeline ${pipeline.name} for index $indexName." + s"🔧 Creating ${statement.pipelineType.name} ingesting pipeline ${statement.name} for index $indexName." ) - api.createPipeline(pipeline.name, pipeline.json) match { + api.pipeline(statement) match { case success @ ElasticSuccess(true) => - logger.info(s"✅ Pipeline ${pipeline.name} created successfully.") + logger.info(s"✅ Pipeline ${statement.name} created successfully.") success case ElasticSuccess(_) => // pipeline creation failed val error = ElasticError( - message = s"Failed to create pipeline ${pipeline.name}.", + message = s"Failed to create pipeline ${statement.name}.", statusCode = Some(500), operation = Some("schema") ) @@ -745,21 +799,22 @@ class TableExecutor( val indexName = table.name if (partitioned) { // create index template - api.createTemplate(s"template_$indexName", table.indexTemplate) match { + api.createTemplate(indexName, table.indexTemplate) match { case success @ ElasticSuccess(true) => - logger.info(s"✅ Template template_$indexName created successfully.") + logger.info(s"✅ Template $indexName created successfully.") success case ElasticSuccess(_) => // template creation failed val error = ElasticError( - message = s"Failed to create template template_$indexName.", + message = s"Failed to create template $indexName.", statusCode = Some(500), operation = Some("schema") ) logger.error(s"❌ ${error.message}") ElasticFailure(error) - case failure @ ElasticFailure(_) => + case failure @ ElasticFailure(error) => + logger.error(s"❌ ${error.message}") failure } } else { @@ -844,7 +899,7 @@ class TableExecutor( ): Either[ElasticError, Boolean] = { val mappingUpdate = - if (diff.mappings.nonEmpty || diff.columns.exists(_.isInstanceOf[ColumnOptionSet])) + if (diff.mappings.nonEmpty || diff.columns.nonEmpty) api.setMapping(indexName, updated.indexMappings) else ElasticSuccess(true) @@ -1052,28 +1107,61 @@ trait SqlGateway extends ElasticClientHelpers { // ======================================================================== def run(sql: String)(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { - ElasticResult.attempt(Parser(sql)) match { - case ElasticSuccess(parsedStatement) => - parsedStatement match { - case Right(statement) => - run(statement) - case Left(l) => + val normalizedQuery = + sql + .split("\n") + .map(_.split("--")(0).trim) + .filterNot(w => w.isEmpty || w.startsWith("--")) + .mkString(" ") + normalizedQuery.split(";\\s*$").toList match { + case Nil => + val error = + ElasticError( + message = s"Empty SQL query.", + statusCode = Some(400), + operation = Some("sql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + case statement :: Nil => + ElasticResult.attempt(Parser(statement)) match { + case ElasticSuccess(parsedStatement) => + parsedStatement match { + case Right(statement) => + run(statement) + case Left(l) => + // parsing error + val error = + ElasticError( + message = s"Error parsing schema DDL statement: ${l.msg}", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + + case ElasticFailure(elasticError) => // parsing error - val error = - ElasticError( - message = s"Error parsing schema DDL statement: ${l.msg}", - statusCode = Some(400), - operation = Some("schema") - ) - logger.error(s"❌ ${error.message}") - Future.successful(ElasticFailure(error)) + Future.successful(ElasticFailure(elasticError.copy(operation = Some("schema")))) } - case ElasticFailure(elasticError) => - // parsing error - Future.successful(ElasticFailure(elasticError.copy(operation = Some("schema")))) + case statements => + implicit val ec: ExecutionContext = system.dispatcher + // run each statement sequentially and return the result of the last one + val last = statements + .foldLeft( + Future.successful[ElasticResult[QueryResult]](ElasticSuccess(QueryResult.empty)) + ) { (acc, statement) => + acc.flatMap { + case ElasticSuccess(_) => + run(statement) + case failure @ ElasticFailure(_) => + return Future.successful(failure) + } + } + last } - } def run( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala index 35e63649..6a5ba13a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala @@ -231,23 +231,8 @@ object TemplateConverter { } if (templateNode.has("mappings")) { - val mappingsNode = templateNode.get("mappings") - - // Check if we need to wrap in _doc for ES 6.x - if ( - ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mappingsNode.has("_doc") - ) { - val wrappedMappings = mapper.createObjectNode() - wrappedMappings.set("_doc", mappingsNode) - legacy.set("mappings", wrappedMappings) - logger.debug(s"Wrapped mappings in '_doc' for ES $elasticVersion") - } else if ( - !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mappingsNode.has("_doc") - ) { - legacy.set("mappings", mappingsNode.get("_doc")) - } else { - legacy.set("mappings", mappingsNode) - } + val mappingsNode = templateNode.get("mappings").asInstanceOf[ObjectNode] + legacy.set("mappings", MappingConverter.convert(mappingsNode, elasticVersion)) } if (templateNode.has("aliases")) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 905f070e..479ecb6e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -155,6 +155,8 @@ package object result { override def get: Nothing = throw new NoSuchElementException( s"ElasticFailure.get: ${elasticError.message}" ) + + def isNotFound: Boolean = elasticError.statusCode.contains(404) } /** Represents an Elasticsearch error. @@ -376,6 +378,12 @@ package object result { sealed trait QueryResult + case object EmptyResult extends QueryResult + + object QueryResult { + def empty: QueryResult = EmptyResult + } + // -------------------- // DQL (SELECT) // -------------------- diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index dcfbe003..65433051 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -38,6 +38,7 @@ trait JestIndicesApi extends IndicesApi with JestClientHelpers { with JestScrollApi with JestBulkApi with JestVersionApi + with JestTemplateApi with JestClientCompanion => /** Create an index with the given settings. diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala new file mode 100644 index 00000000..e63c19c1 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class JestSqlGatewaySpec extends SqlGatewayIntegrationSpec with EmbeddedElasticTestKit { + override def client: SqlGateway = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 8f3f4273..66dc032f 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -148,6 +148,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH with RestHighLevelClientScrollApi with RestHighLevelClientBulkApi with RestHighLevelClientVersionApi + with RestHighLevelClientTemplateApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala index afa5f20e..522cbdac 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala @@ -106,6 +106,20 @@ trait RestHighLevelClientHelpers extends ElasticClientHelpers { _: RestHighLevel operation = Some(operation) ) ) + case Failure(ex: org.elasticsearch.client.ResponseException) => + val statusCode = Some(ex.getResponse.getStatusLine.getStatusCode) + logger.error( + s"Response exception during operation '$operation'$indexStr: ${ex.getMessage}", + ex + ) + ElasticResult.failure( + ElasticError( + message = s"HTTP error during $operation: ${ex.getMessage}", + cause = Some(ex), + statusCode = statusCode, + operation = Some(operation) + ) + ) case Failure(ex) => logger.error(s"Exception during operation '$operation'$indexStr: ${ex.getMessage}", ex) ElasticResult.failure( diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala new file mode 100644 index 00000000..5e57e684 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class RestHighLevelClientSqlGatewaySpec + extends SqlGatewayIntegrationSpec + with EmbeddedElasticTestKit { + override lazy val client: SqlGateway = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index ef4666d7..c21aaf84 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -157,6 +157,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH with RestHighLevelClientScrollApi with RestHighLevelClientBulkApi with RestHighLevelClientVersionApi + with RestHighLevelClientTemplateApi with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala index afa5f20e..522cbdac 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala @@ -106,6 +106,20 @@ trait RestHighLevelClientHelpers extends ElasticClientHelpers { _: RestHighLevel operation = Some(operation) ) ) + case Failure(ex: org.elasticsearch.client.ResponseException) => + val statusCode = Some(ex.getResponse.getStatusLine.getStatusCode) + logger.error( + s"Response exception during operation '$operation'$indexStr: ${ex.getMessage}", + ex + ) + ElasticResult.failure( + ElasticError( + message = s"HTTP error during $operation: ${ex.getMessage}", + cause = Some(ex), + statusCode = statusCode, + operation = Some(operation) + ) + ) case Failure(ex) => logger.error(s"Exception during operation '$operation'$indexStr: ${ex.getMessage}", ex) ElasticResult.failure( diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala new file mode 100644 index 00000000..20305fa9 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientSqlGatewaySpec + extends SqlGatewayIntegrationSpec + with ElasticDockerTestKit { + override lazy val client: SqlGateway = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 3f0b2e55..9005dee0 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -126,6 +126,7 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { with JavaClientScrollApi with JavaClientBulkApi with JavaClientVersionApi + with JavaClientTemplateApi with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala new file mode 100644 index 00000000..b7ae07c7 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { + override lazy val client: SqlGateway = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 75d6fbfb..563120b2 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -120,6 +120,7 @@ trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { with JavaClientPipelineApi with JavaClientScrollApi with JavaClientBulkApi + with JavaClientTemplateApi with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala new file mode 100644 index 00000000..b7ae07c7 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { + override lazy val client: SqlGateway = new JavaClientSpi().client(elasticConfig) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index b2ca0768..417e1f0a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -63,11 +63,39 @@ package object schema { } object IndexField { - def apply(name: String, node: JsonNode): IndexField = { - val tpe = Option(node.get("type")).map(_.asText()).getOrElse("object") + def apply(name: String, node: JsonNode, _meta: Option[ObjectValue]): IndexField = { + val tpe = _meta + .flatMap { + case m: ObjectValue => + m.value.get("data_type") match { + case Some(v: StringValue) => Some(v.value) + case _ => None + } + case _ => None + } + .getOrElse(Option(node.get("type")).map(_.asText()).getOrElse("object")) val nullValue = - Option(node.get("null_value")).flatMap(Value(_)).map(Value(_)) + Option(node.get("null_value")) + .flatMap(Value(_)) + .map(Value(_)) + .orElse(_meta.flatMap { + case m: ObjectValue => + m.value.get("default_value") match { + case Some(v: Value[_]) => Some(v) + case _ => None + } + case _ => None + }) + + val fields_meta = _meta.flatMap { + case m: ObjectValue => + m.value.get("multi_fields") match { + case Some(f: ObjectValue) => Some(f) + case _ => None + } + case _ => None + } val fields = Option(node.get("fields")) // multi-fields @@ -75,15 +103,22 @@ package object schema { .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue - apply(name, value) + val _meta = fields_meta.flatMap { + case m: ObjectValue => + m.value.get(name) match { + case Some(colMeta: ObjectValue) => Some(colMeta) + case _ => None + } + case _ => None + } + apply(name, value, _meta) }.toList) .getOrElse(Nil) val options = extractObject(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) - val meta = options.get("meta") - val comment = meta.flatMap { + val comment = _meta.flatMap { case m: ObjectValue => m.value.get("comment") match { case Some(c: StringValue) => Some(c.value) @@ -91,15 +126,18 @@ package object schema { } case _ => None } - val notNull = meta.flatMap { + + val notNull = _meta.flatMap { case m: ObjectValue => m.value.get("not_null") match { case Some(c: BooleanValue) => Some(c.value) + case Some(c: StringValue) => Some(c.value.toBoolean) case _ => None } case _ => None } - val script = meta.flatMap { + + val script = _meta.flatMap { case m: ObjectValue => m.value.get("script") match { case Some(st: ObjectValue) => @@ -154,16 +192,32 @@ package object schema { val docNode = root.path("_doc") return apply(docNode) } + val options = extractObject(root, ignoredKeys = Set("properties")) + val meta = options.get("_meta") + val columns = meta.flatMap { + case m: ObjectValue => + m.value.get("columns") match { + case Some(cols: ObjectValue) => Some(cols) + case _ => None + } + case _ => None + } val fields = Option(root.get("properties")) .map(_.properties().asScala.map { entry => val name = entry.getKey val value = entry.getValue - IndexField(name, value) + val _meta = columns.flatMap { + case c: ObjectValue => + c.value.get(name) match { + case Some(colMeta: ObjectValue) => Some(colMeta) + case _ => None + } + case _ => None + } + IndexField(name, value, _meta) }.toList) .getOrElse(Nil) - val options = extractObject(root, ignoredKeys = Set("properties")) - val meta = options.get("_meta") val primaryKey: List[String] = meta .map { case m: ObjectValue => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 9aba3e6a..9f99cf29 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -45,8 +45,10 @@ package object function { override def shouldBeScripted: Boolean = identifier.shouldBeScripted } - trait FunctionWithValue[+T] extends Function with TokenValue { + trait FunctionWithValue[+T] extends FunctionWithIdentifier with TokenValue { def value: T + + override def identifier: Identifier = Identifier(this) } object FunctionUtils { @@ -274,7 +276,7 @@ package object function { val ret = SQLTypeUtils .coerce( a, - argTypes(i), + in, context ) if (ret.startsWith(".")) { @@ -306,9 +308,21 @@ package object function { ) case _ => } - case _ => + Option(paramName) + case SQLTypes.Any if ctx.isProcessor => + in match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + val param = SQLTypeUtils + .coerce( + a, + in, + context + ) + ctx.addParam(LiteralParam(param)) + case _ => Option(paramName) + } + case _ => Option(paramName) } - Option(paramName) case _ => Option(paramName) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 7f31b75e..1f7a333e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -763,6 +763,9 @@ package object sql { case class ObjectValues(override val values: Seq[ObjectValue]) extends Values[Map[String, Value[_]], ObjectValue](values) { override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Struct) + import app.softnetwork.elastic.sql.serialization._ + + def toJson: JsonNode = this } def toRegex(value: String): String = { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 0e88a224..ebe6c583 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -76,21 +76,55 @@ object Parser def ident: Parser[String] = """[a-zA-Z_][a-zA-Z0-9_]*""".r + private val lparen: Parser[String] = "(" + private val rparen: Parser[String] = ")" + private val comma: Parser[String] = "," + private val lbracket: Parser[String] = "[" + private val rbracket: Parser[String] = "]" + private val startStruct: Parser[String] = "{" + private val endStruct: Parser[String] = "}" + def objectValue: PackratParser[ObjectValue] = - start ~ repsep(option, separator) ~ end ^^ { case _ ~ opts ~ _ => + lparen ~> rep1sep(option, comma) <~ rparen ^^ { opts => ObjectValue(opts.toMap) } + def objectValues: PackratParser[ObjectValues] = + lbracket ~> rep1sep(objectValue, comma) <~ rbracket ^^ { ovs => + ObjectValues(ovs) + } + def option: PackratParser[(String, Value[_])] = - ident ~ "=" ~ (value | objectValue) ^^ { case key ~ _ ~ value => + ident ~ "=" ~ (objectValues | objectValue | value) ^^ { case key ~ _ ~ value => (key, value) } def options: PackratParser[Map[String, Value[_]]] = - "OPTIONS" ~ start ~ repsep(option, separator) ~ end ^^ { case _ ~ _ ~ opts ~ _ => + "OPTIONS" ~ lparen ~ repsep(option, comma) ~ rparen ^^ { case _ ~ _ ~ opts ~ _ => opts.toMap } + def array_of_struct: PackratParser[ObjectValues] = + lbracket ~> repsep(struct, comma) <~ rbracket ^^ { ovs => + ObjectValues(ovs) + } + + def struct_entry: PackratParser[(String, Value[_])] = + ident ~ "=" ~ (array_of_struct | struct | value) ^^ { case key ~ _ ~ v => + key -> v + } + + def struct: PackratParser[ObjectValue] = + startStruct ~> repsep(struct_entry, comma) <~ endStruct ^^ { entries => + ObjectValue(entries.toMap) + } + + def row: PackratParser[List[Value[_]]] = + lparen ~> repsep(array_of_struct | struct | value, comma) <~ rparen + + def rows: PackratParser[List[List[Value[_]]]] = + repsep(row, comma) + def processorType: PackratParser[IngestProcessorType] = ident ^^ { name => name.toLowerCase match { @@ -542,8 +576,8 @@ object Parser /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ def insert: PackratParser[Insert] = - ("INSERT" ~ "INTO") ~ ident ~ opt(start ~> repsep(ident, separator) <~ end) ~ - (("VALUES" ~ start ~> repsep(value, separator) <~ end) ^^ { vs => Right(vs) } + ("INSERT" ~ "INTO") ~ ident ~ opt(lparen ~> repsep(ident, comma) <~ rparen) ~ + (("VALUES" ~> rows) ^^ { vs => Right(vs) } | "AS".? ~> dqlStatement ^^ { q => Left(q) }) ~ opt(onConflict) ^^ { case _ ~ table ~ colsOpt ~ vals ~ conflict => conflict match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index b25f96dd..d4b5380c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -130,6 +130,10 @@ package object `type` { def keyword_type: PackratParser[SQLTypes.Keyword.type] = "(?i)keyword".r ^^ (_ => SQLTypes.Keyword) - def extension_type: PackratParser[SQLType] = sql_type | text_type | keyword_type + def geo_point_type: PackratParser[SQLTypes.GeoPoint.type] = + "(?i)(geo_point|geopoint)".r ^^ (_ => SQLTypes.GeoPoint) + + def extension_type: PackratParser[SQLType] = + sql_type | text_type | keyword_type | geo_point_type } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index 2dcd82f9..58cffbfd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -400,7 +400,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { case Some(ctx) => ctx.addParam(identifier) match { case Some(_) => - identifier.baseType match { + identifier.originalType match { case SQLTypes.Any => // in painless context, Any is ZonedDateTime maybeValue.map(_.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Date => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index 6eaa8550..ee64b909 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -293,27 +293,50 @@ package object query { case class Insert( table: String, cols: Seq[String], - values: Either[DqlStatement, Seq[Value[_]]], + values: Either[DqlStatement, Seq[Seq[Value[_]]]], onConflict: Option[OnConflict] = None ) extends DmlStatement { lazy val conflictTarget: Option[Seq[String]] = onConflict.flatMap(_.target) lazy val doUpdate: Boolean = onConflict.exists(_.doUpdate) + private def valueToSql(value: Value[_]): String = value match { + case v if v.isInstanceOf[ObjectValues] => + v.asInstanceOf[ObjectValues] + .values + .map(valueToSql) + .mkString("[", ", ", "]") + case v if v.isInstanceOf[ObjectValue] => + v.asInstanceOf[ObjectValue] + .value + .map { case (k, v) => + v match { + case IdValue | IngestTimestampValue => s"""$k = "${v.ddl}"""" + case _ => s"""$k = ${v.ddl}""" + } + } + .mkString("{", ", ", "}") + case v => s"${v.ddl}" + } + + private def rowToSql(row: Seq[Value[_]]): String = { + val rowSql = row + .map(valueToSql) + .mkString(", ") + s"($rowSql)" + } + override def sql: String = { values match { case Left(query) if cols.isEmpty => s"INSERT INTO $table ${query.sql}${asString(onConflict)}" case Left(query) => s"INSERT INTO $table (${cols.mkString(",")}) ${query.sql}${asString(onConflict)}" - case Right(vs) => - val valuesSql = vs - .map { - case v if v.isInstanceOf[StringValue] => s"'${v.value}'" - case v => s"${v.value}" - } + case Right(rows) => + val valuesSql = rows + .map(rowToSql) .mkString(", ") - s"INSERT INTO $table ${cols.mkString(",")} VALUES ($valuesSql)" + s"INSERT INTO $table (${cols.mkString(",")}) VALUES $valuesSql${asString(onConflict)}" } } @@ -321,8 +344,15 @@ package object query { for { _ <- values match { case Left(query) => query.validate() - case Right(vs) if cols.size != vs.size => - Left(s"Number of columns (${cols.size}) does not match number of values (${vs.size})") + case Right(rows) => + val invalidRows = rows.filter(_.size != cols.size) + if (invalidRows.nonEmpty) + Left( + s"Some rows have invalid number of values: ${invalidRows + .map(r => s"(${r.map(_.value).mkString(",")})") + .mkString("; ")}" + ) + else Right(()) case _ => Right(()) } @@ -354,15 +384,19 @@ package object query { def toJson: Option[JsonNode] = { values match { - case Right(vs) if cols.size == vs.size => - val map: Map[String, Value[_]] = - cols - .zip(vs) - .map { case (k, v) => - k -> v - } - .toMap - val json: JsonNode = ObjectValue(map) + case Right(rows) => + val maps: Seq[ObjectValue] = + for (row <- rows) yield { + val map: Map[String, Value[_]] = + cols + .zip(row) + .map { case (k, v) => + k -> v + } + .toMap + ObjectValue(map) + } + val json: JsonNode = ObjectValues(maps) Some(json) case _ => None } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index ebb27dfc..3f1969c6 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -187,7 +187,10 @@ package object schema { } case "script" => - val desc = props.get("description").asText() + val desc = { + if (props.has("description")) props.get("description").asText() + else "" + } val lang = props.get("lang").asText() require(lang == "painless", s"Only painless supported, got $lang") val source = props.get("source").asText() @@ -375,7 +378,6 @@ package object schema { dateRounding: String, dateFormats: List[String], prefix: String, - separator: String = "-", ignoreFailure: Boolean = true ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.DateIndexName @@ -386,7 +388,6 @@ package object schema { "date_rounding" -> dateRounding, "date_formats" -> dateFormats, "index_name_prefix" -> prefix, - "separator" -> separator, "ignore_failure" -> ignoreFailure ) @@ -578,8 +579,42 @@ package object schema { } } + def _meta: Map[String, Value[_]] = { + Map( + "data_type" -> StringValue(dataType.typeId), + "not_null" -> StringValue(s"$notNull") + ) ++ defaultValue.map(d => "default_value" -> d) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ script + .map { sc => + ObjectValue( + Map( + "sql" -> StringValue(sc.script), + "column" -> StringValue(path), + "painless" -> StringValue(sc.source) + ) + ) + } + .map("script" -> _) ++ Map( + "multi_fields" -> ObjectValue( + multiFields.map(field => field.name -> ObjectValue(field._meta)).toMap + ) + ) + } + + def updateStruct(): Column = { + struct + .map { st => + val updated = st.copy(multiFields = st.multiFields.filterNot(_.name == this.name) :+ this) + updated.updateStruct() + } + .getOrElse(this) + .update() + } + def update(struct: Option[Column] = None): Column = { val updated = this.copy(struct = struct) + // update script accordingly with new path val updated_script = script.map { sc => sc.copy( @@ -587,41 +622,15 @@ package object schema { source = sc.source.replace(s"ctx.$name", s"ctx.${updated.path}") ) } - val sql_script: Option[ObjectValue] = - updated_script match { - case Some(s) => - Some( - ObjectValue( - Map( - "sql" -> StringValue(s.script), - "column" -> StringValue(updated.path), - "painless" -> StringValue(s.source) - ) - ) - ) - case _ => None - } updated.copy( multiFields = multiFields.map { field => field.update(Some(updated)) }, script = updated_script, - options = options ++ Map( - "meta" -> ObjectValue( - options.get("meta") match { - case Some(ObjectValue(value)) => - value ++ Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => - "comment" -> StringValue(ct) - ) ++ sql_script.map(st => "script" -> st) - case _ => - Map("not_null" -> BooleanValue(notNull)) ++ comment.map(ct => - "comment" -> StringValue(ct) - ) ++ sql_script.map(st => "script" -> st) - } - ) - ) + options = options ) } + def sql: String = { val opts = if (options.nonEmpty) { s" OPTIONS ${ObjectValue(options).ddl}" @@ -666,8 +675,14 @@ package object schema { val root = mapper.createObjectNode() val esType = SQLTypeUtils.elasticType(dataType) root.put("type", esType) - defaultValue.foreach { dv => - updateNode(root, Map("null_value" -> dv)) + dataType match { + case SQLTypes.Varchar | SQLTypes.Text => // do not set null_value for text types + case _ => + defaultValue.foreach { + case IngestTimestampValue => () // do not set null_value for ingest timestamp + case dv => // set null_value for other types + updateNode(root, Map("null_value" -> dv)) + } } if (multiFields.nonEmpty) { val name = @@ -1016,19 +1031,23 @@ package object schema { ) ) ) - .getOrElse(Map.empty) - - def update(): Table = this.copy( - columns = columns.map(_.update()), - mappings = mappings ++ Map( - "_meta" -> - ObjectValue(mappings.get("_meta") match { - case Some(ObjectValue(value)) => - (value - "primary_key" - "partition_by") ++ _meta - case _ => _meta - }) + .getOrElse(Map.empty) ++ + Map("columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) })) + + def update(): Table = { + val updated = + this.copy(columns = columns.map(_.update())) // update columns first with struct info + updated.copy( + mappings = updated.mappings ++ Map( + "_meta" -> + ObjectValue(updated.mappings.get("_meta") match { + case Some(ObjectValue(value)) => + (value - "primary_key" - "partition_by" - "columns") ++ updated._meta + case _ => updated._meta + }) + ) ) - ) + } def sql: String = { val opts = @@ -1071,250 +1090,258 @@ package object schema { .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) def merge(statements: Seq[AlterTableStatement]): Table = { - statements.foldLeft(this) { (table, statement) => - statement match { - // table columns - case AddColumn(column, ifNotExists) => - if (ifNotExists && table.cols.contains(column.name)) table - else if (!table.cols.contains(column.name)) - table.copy(columns = table.columns :+ column) - else throw ColumnNotFound(column.name, table.name) - case DropColumn(columnName, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy(columns = table.columns.filterNot(_.name == columnName)) - else throw ColumnNotFound(columnName, table.name) - case RenameColumn(oldName, newName) => - if (cols.contains(oldName)) - table.copy( - columns = table.columns.map { col => - if (col.name == oldName) col.copy(name = newName) else col - } - ) - else throw ColumnNotFound(oldName, table.name) - // column type - case AlterColumnType(columnName, newType, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(dataType = newType) - else col - } - ) - else throw ColumnNotFound(columnName, table.name) - // column script - case AlterColumnScript(columnName, newScript, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(script = Some(newScript.copy(dataType = col.dataType))) - else col - } - ) - else throw ColumnNotFound(columnName, table.name) - case DropColumnScript(columnName, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(script = None) - else col - } - ) - else throw ColumnNotFound(columnName, table.name) - // column default value - case AlterColumnDefault(columnName, newDefault, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) - else col + statements + .foldLeft(this) { (table, statement) => + statement match { + // table columns + case AddColumn(column, ifNotExists) => + if (ifNotExists && table.cols.contains(column.name)) table + else if (!table.cols.contains(column.name)) + table.copy(columns = table.columns :+ column) + else throw ColumnNotFound(column.name, table.name) + case DropColumn(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy(columns = table.columns.filterNot(_.name == columnName)) + else throw ColumnNotFound(columnName, table.name) + case RenameColumn(oldName, newName) => + if (cols.contains(oldName)) + table.copy( + columns = table.columns.map { col => + if (col.name == oldName) col.copy(name = newName) else col + } + ) + else throw ColumnNotFound(oldName, table.name) + // column type + case AlterColumnType(columnName, newType, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(dataType = newType) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column script + case AlterColumnScript(columnName, newScript, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(script = Some(newScript.copy(dataType = col.dataType))) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnScript(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(script = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column default value + case AlterColumnDefault(columnName, newDefault, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnDefault(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column not null + case AlterColumnNotNull(columnName, ifExists) => + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = true) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnNotNull(columnName, ifExists) => + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = false) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column options + case AlterColumnOptions(columnName, newOptions, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options ++ newOptions) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case AlterColumnOption( + columnName, + optionKey, + optionValue, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + options = ObjectValue(col.options).set(optionKey, optionValue).value + ) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnOption( + columnName, + optionKey, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + options = ObjectValue(col.options).remove(optionKey).value + ) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column comments + case AlterColumnComment(columnName, newComment, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = Some(newComment)) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnComment(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // multi-fields + case AlterColumnFields(columnName, newFields, ifExists) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + val updated = c + .copy( + multiFields = newFields.toList.map(_.update(Some(c))) + ) + .updateStruct() + table.copy( + columns = table.columns.map { col => + if (col.name == updated.name) updated else col + } + ) + case _ => throw ColumnNotFound(columnName, table.name) } - ) - else throw ColumnNotFound(columnName, table.name) - case DropColumnDefault(columnName, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(defaultValue = None) - else col + } + case AlterColumnField( + columnName, + field, + ifExists + ) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + val updatedFields = c.multiFields.filterNot(_.name == field.name) :+ field + c.copy(multiFields = updatedFields) + table + case _ => throw ColumnNotFound(columnName, table.name) } - ) - else throw ColumnNotFound(columnName, table.name) - // column not null - case AlterColumnNotNull(columnName, ifExists) => - if (!table.cols.contains(columnName) && ifExists) table - else if (table.cols.contains(columnName)) - table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(notNull = true) - else col + } + case DropColumnField( + columnName, + fieldName, + ifExists + ) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + c.copy( + multiFields = c.multiFields.filterNot(_.name == fieldName) + ) + table + case _ => throw ColumnNotFound(columnName, table.name) } - ) - else throw ColumnNotFound(columnName, table.name) - case DropColumnNotNull(columnName, ifExists) => - if (!table.cols.contains(columnName) && ifExists) table - else if (table.cols.contains(columnName)) + } + // mappings / settings + case AlterTableMapping(optionKey, optionValue) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) col.copy(notNull = false) - else col - } + mappings = ObjectValue(table.mappings).set(optionKey, optionValue).value ) - else throw ColumnNotFound(columnName, table.name) - // column options - case AlterColumnOptions(columnName, newOptions, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) + case DropTableMapping(optionKey) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(options = col.options ++ newOptions) - else col - } + mappings = ObjectValue(table.mappings).remove(optionKey).value ) - else throw ColumnNotFound(columnName, table.name) - case AlterColumnOption( - columnName, - optionKey, - optionValue, - ifExists - ) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) + case AlterTableSetting(optionKey, optionValue) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy( - options = ObjectValue(col.options).set(optionKey, optionValue).value - ) - else col - } + settings = ObjectValue(table.settings).set(optionKey, optionValue).value ) - else throw ColumnNotFound(columnName, table.name) - case DropColumnOption( - columnName, - optionKey, - ifExists - ) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) + case DropTableSetting(optionKey) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy( - options = ObjectValue(col.options).remove(optionKey).value - ) - else col - } + settings = ObjectValue(table.settings).remove(optionKey).value ) - else throw ColumnNotFound(columnName, table.name) - // column comments - case AlterColumnComment(columnName, newComment, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) + case AlterTableAlias(aliasName, aliasValue) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(comment = Some(newComment)) - else col - } + aliases = table.aliases + (aliasName -> aliasValue) ) - else throw ColumnNotFound(columnName, table.name) - case DropColumnComment(columnName, ifExists) => - if (ifExists && !table.cols.contains(columnName)) table - else if (table.cols.contains(columnName)) + case DropTableAlias(aliasName) => table.copy( - columns = table.columns.map { col => - if (col.name == columnName) - col.copy(comment = None) - else col - } + aliases = table.aliases - aliasName ) - else throw ColumnNotFound(columnName, table.name) - // multi-fields - case AlterColumnFields(columnName, newFields, ifExists) => - val col = find(columnName) - val exists = col.isDefined - if (ifExists && !exists) table - else { - col match { - case Some(c) => - c.copy( - multiFields = newFields.toList - ) - table - case _ => throw ColumnNotFound(columnName, table.name) - } - } - case AlterColumnField( - columnName, - field, - ifExists - ) => - val col = find(columnName) - val exists = col.isDefined - if (ifExists && !exists) table - else { - col match { - case Some(c) => - val updatedFields = c.multiFields.filterNot(_.name == field.name) :+ field - c.copy(multiFields = updatedFields) - table - case _ => throw ColumnNotFound(columnName, table.name) - } - } - case DropColumnField( - columnName, - fieldName, - ifExists - ) => - val col = find(columnName) - val exists = col.isDefined - if (ifExists && !exists) table - else { - col match { - case Some(c) => - c.copy( - multiFields = c.multiFields.filterNot(_.name == fieldName) - ) - table - case _ => throw ColumnNotFound(columnName, table.name) - } - } - // mappings / settings - case AlterTableMapping(optionKey, optionValue) => - table.copy( - mappings = ObjectValue(table.mappings).set(optionKey, optionValue).value - ) - case DropTableMapping(optionKey) => - table.copy( - mappings = ObjectValue(table.mappings).remove(optionKey).value - ) - case AlterTableSetting(optionKey, optionValue) => - table.copy( - settings = ObjectValue(table.settings).set(optionKey, optionValue).value - ) - case DropTableSetting(optionKey) => - table.copy( - settings = ObjectValue(table.settings).remove(optionKey).value - ) - case AlterTableAlias(aliasName, aliasValue) => - table.copy( - aliases = table.aliases + (aliasName -> aliasValue) - ) - case DropTableAlias(aliasName) => - table.copy( - aliases = table.aliases - aliasName - ) - case _ => table + case _ => table + } } - } + .update() } @@ -1448,7 +1475,9 @@ package object schema { lazy val indexTemplate: ObjectNode = { val node = mapper.createObjectNode() - node.put("index_patterns", s"$name-*") + val patterns = mapper.createArrayNode() + patterns.add(s"$name-*") + node.set("index_patterns", patterns) node.put("priority", 1) val template = mapper.createObjectNode() template.set("mappings", indexMappings) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala index f1873c63..24a1df33 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} import com.fasterxml.jackson.databind.{ DeserializationFeature, JsonNode, @@ -81,6 +81,15 @@ package object serialization { node } + implicit def objectValuesToObjectNode(values: ObjectValues): ArrayNode = { + import JacksonConfig.{objectMapper => mapper} + val node = mapper.createArrayNode() + values.values.foreach { value => + node.add(value) + } + node + } + private[sql] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { import JacksonConfig.{objectMapper => mapper} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index e5b273c1..89681c5a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -56,3 +56,5 @@ sealed trait EsqlType extends SQLType trait EsqlText extends EsqlType with SQLVarchar trait EsqlKeyword extends EsqlType with SQLVarchar + +trait EsqlGeoPoint extends EsqlType diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index d381e9f9..d127b457 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -70,6 +70,7 @@ object SQLTypeUtils { case DateTime => "date" case Timestamp => "date" case Temporal => "date" + case GeoPoint => "geo_point" case Array(Struct) => "nested" case Struct => "object" case _ => "object" @@ -166,11 +167,11 @@ object SQLTypeUtils { def coerce(in: PainlessScript, to: SQLType, context: Option[PainlessContext]): String = { context match { - case Some(_) => + case Some(ctx) => in match { case identifier: Identifier => - identifier.baseType match { - case SQLTypes.Any => // in painless context, Any is ZonedDateTime + identifier.originalType match { + case SQLTypes.Any if !ctx.isProcessor => // in painless context, Any is ZonedDateTime to match { case SQLTypes.Date => identifier.addPainlessMethod(".toLocalDate()") @@ -178,6 +179,15 @@ object SQLTypeUtils { identifier.addPainlessMethod(".toLocalTime()") case _ => // do nothing } + case SQLTypes.Any if ctx.isProcessor => + to match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + val expr = identifier.painless(context) + val from = SQLTypes.BigInt + val ret = coerce(expr, from, to, identifier.nullable, context) + return ret + case _ => // do nothing + } case _ => // do nothing } case _ => // do nothing diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 1c7c021e..da1fc9c4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -54,6 +54,8 @@ object SQLTypes { case object Struct extends SQLStruct { val typeId = "STRUCT" } + case object GeoPoint extends EsqlGeoPoint { val typeId = "GEO_POINT" } + def apply(typeName: String): SQLType = typeName.toLowerCase match { case "null" => Null case "boolean" => Boolean @@ -64,31 +66,18 @@ object SQLTypes { case "keyword" => Keyword case "text" => Text case "varchar" => Varchar - case "datetime" | "timestamp" => DateTime + case "timestamp" => Timestamp + case "datetime" => DateTime case "date" => Date case "time" => Time case "double" => Double case "float" | "real" => Real case "object" | "struct" => Struct case "nested" | "array" => Array(Struct) + case "geo_point" => GeoPoint case _ => Any } - def apply(field: IndexField): SQLType = field.`type` match { - case "null" => Null - case "boolean" => Boolean - case "integer" => Int - case "long" => BigInt - case "short" => SmallInt - case "byte" => TinyInt - case "keyword" => Keyword - case "text" => Text - case "date" => DateTime - case "double" => Double - case "float" => Real - case "object" => Struct - case "nested" => Array(Struct) - case _ => Any - } + def apply(field: IndexField): SQLType = apply(field.`type`) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 38670d95..f8d0a151 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -980,29 +980,30 @@ class ParserSpec extends AnyFlatSpec with Matchers { .map(p => p.source) .getOrElse("") should include( """def param1 = ctx.birthdate; - |def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); - |ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)""".stripMargin - .replaceAll("\n", " ") + |def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); + |def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); + |ctx.age = (param1 == null) ? null : param3""".stripMargin.replaceAll("\n", " ") ) cols.find(_.name == "ingested_at").get.defaultValue.map(_.value) shouldBe Some( "_ingest.timestamp" ) ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) - val sql = ct.schema.sql + val schema = ct.schema + val sql = schema.sql println(sql) - println(ct.schema.defaultPipeline.ddl) - val json = ct.schema.defaultPipeline.json + println(schema.defaultPipeline.ddl) + val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" - val indexMappings = ct.schema.indexMappings + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer","meta":{"not_null":true,"comment":"user identifier"}},"name":{"type":"text","null_value":"anonymous","fields":{"raw":{"type":"keyword","meta":{"not_null":false,"comment":"sortable"}}},"analyzer":"french","search_analyzer":"french","meta":{"not_null":false}},"birthdate":{"type":"date","meta":{"not_null":false}},"age":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)"}}},"ingested_at":{"type":"date","null_value":"_ingest.timestamp","meta":{"not_null":false}},"profile":{"type":"object","properties":{"bio":{"type":"text","meta":{"not_null":false}},"followers":{"type":"integer","meta":{"not_null":false}},"join_date":{"type":"date","meta":{"not_null":false}},"seniority":{"type":"integer","meta":{"not_null":false,"script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)"}}}},"meta":{"not_null":false,"comment":"user profile"}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"}}}""" - val indexSettings = ct.schema.indexSettings + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date","null_value":"_ingest.timestamp"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin + val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" - val pipeline = ct.schema.defaultPipelineNode + val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : ChronoUnit.YEARS.between(param1, param2)","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : ChronoUnit.DAYS.between(param1, param2)","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) @@ -1030,7 +1031,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { } it should "parse CREATE OR REPLACE TABLE as select" in { - val sql = "CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts" + val sql = "CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts WHERE active = true" val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get @@ -1294,7 +1295,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { | SCRIPT ( | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", | lang = "painless", - | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null = ChronoUnit.YEARS.between(param1, param2)", + | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))", | ignore_failure = true | ), | SET ( @@ -1307,7 +1308,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { | SCRIPT ( | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", | lang = "painless", - | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)", + | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))", | ignore_failure = true | ), | DATE_INDEX_NAME ( @@ -1363,7 +1364,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) ) => source should include( - "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null = ChronoUnit.YEARS.between(param1, param2)" + "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))" ) case other => fail(s"Expected DdlScriptProcessor for age, got $other") } @@ -1391,7 +1392,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) ) => source should include( - "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null = ChronoUnit.DAYS.between(param1, param2)" + "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" ) case other => fail(s"Expected DdlScriptProcessor for profile.seniority, got $other") } @@ -1404,7 +1405,6 @@ class ParserSpec extends AnyFlatSpec with Matchers { "M", List("yyyy-MM"), "users-", - "-", true ) ) => @@ -1490,26 +1490,27 @@ class ParserSpec extends AnyFlatSpec with Matchers { stmt match { case Insert("users", cols, Right(values), Some(OnConflict(None, false))) => cols should contain inOrder ("id", "name") - values.map(_.value) should contain inOrder (1, "Alice") + values.head.map(_.value) should contain inOrder (1, "Alice") case _ => fail("Expected Insert with values") } } it should "parse INSERT INTO ... VALUES with DO UPDATE" in { - val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO UPDATE" + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice'), (2, 'BOB') ON CONFLICT DO UPDATE" val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get stmt match { case Insert("users", cols, Right(values), Some(OnConflict(None, true))) => cols should contain inOrder ("id", "name") - values.map(_.value) should contain inOrder (1, "Alice") + values.head.map(_.value) should contain inOrder (1, "Alice") + values.last.map(_.value) should contain inOrder (2, "BOB") case _ => fail("Expected Insert with values") } } it should "parse INSERT INTO ... SELECT" in { - val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT DO NOTHING" + val sql = "INSERT INTO users AS SELECT id, name FROM old_users ON CONFLICT DO NOTHING" val result = Parser(sql) result.isRight shouldBe true val stmt = result.toOption.get @@ -1558,4 +1559,18 @@ class ParserSpec extends AnyFlatSpec with Matchers { case _ => fail("Expected Delete") } } + + it should "parse UNION ALL" in { + val sql = + "SELECT id, name FROM dql_users WHERE age > 30 UNION ALL SELECT id, name FROM dql_users WHERE age <= 30" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case MultiSearch(Seq(left: DqlStatement, right: DqlStatement)) => + left.sql should include("SELECT id, name FROM dql_users WHERE age > 30") + right.sql should include("SELECT id, name FROM dql_users WHERE age <= 30") + case _ => fail("Expected Union") + } + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala index 4a5a0f14..1361256b 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem +import akka.stream.scaladsl.Sink import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, @@ -27,7 +28,9 @@ import app.softnetwork.elastic.client.result.{ QueryStructured, QueryTable } +import app.softnetwork.elastic.client.scroll.ScrollMetrics import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.Table import app.softnetwork.persistence.generateUUID import org.scalatest.concurrent.ScalaFutures @@ -69,13 +72,57 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala // Helper: assert SELECT result type // ------------------------------------------------------------------------- - def assertSelectResult(res: ElasticResult[QueryResult]): Unit = { + private def normalizeRow(row: Map[String, Any]): Map[String, Any] = { + val updated = row - "_id" - "_index" - "_score" - "_version" - "_sort" + updated.map(entry => + entry._2 match { + case m: Map[_, _] => + entry._1 -> normalizeRow(m.asInstanceOf[Map[String, Any]]) + case seq: Seq[_] if seq.nonEmpty && seq.head.isInstanceOf[Map[_, _]] => + entry._1 -> seq + .asInstanceOf[Seq[Map[String, Any]]] + .map(m => normalizeRow(m)) + case other => entry._1 -> other + } + ) + } + + def assertSelectResult( + res: ElasticResult[QueryResult], + rows: Seq[Map[String, Any]] = Seq.empty + ): Unit = { res.isSuccess shouldBe true res.toOption.get match { - case QueryStream(_) => succeed - case QueryStructured(_) => succeed - case QueryRows(_) => succeed - case other => fail(s"Unexpected QueryResult type for SELECT: $other") + case QueryStream(stream) => + val sink = Sink.fold[Seq[Map[String, Any]], (Map[String, Any], ScrollMetrics)](Seq.empty) { + case (acc, (row, _)) => + acc :+ normalizeRow(row) + } + val results = stream.runWith(sink).futureValue + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case QueryStructured(response) => + val results = + response.results.map(normalizeRow) + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case q: QueryRows => + val results = q.rows.map(normalizeRow) + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case other => fail(s"Unexpected QueryResult type for SELECT: $other") } } @@ -92,9 +139,17 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala // Helper: assert DML result type // ------------------------------------------------------------------------- - def assertDml(res: ElasticResult[QueryResult]): Unit = { + def assertDml(res: ElasticResult[QueryResult], result: Option[DmlResult] = None): Unit = { res.isSuccess shouldBe true res.toOption.get shouldBe a[DmlResult] + result match { + case Some(expected) => + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.inserted shouldBe expected.inserted + dml.updated shouldBe expected.updated + dml.deleted shouldBe expected.deleted + case None => // do nothing + } } // ------------------------------------------------------------------------- @@ -127,7 +182,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val table = assertShowTable(show) val ddl = table.ddl - ddl should include("CREATE TABLE show_users") + ddl should include("CREATE OR REPLACE TABLE show_users") ddl should include("id INT NOT NULL") ddl should include("name VARCHAR") ddl should include("age INT DEFAULT 0") @@ -189,14 +244,21 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala // Vérification via SHOW TABLE val table = assertShowTable(client.run("SHOW TABLE users").futureValue) - val ddl = table.ddl - - ddl should include("CREATE TABLE users") - ddl should include("id INT NOT NULL") - ddl should include("name VARCHAR") - ddl should include("DEFAULT 'anonymous'") - ddl should include("SCRIPT AS (DATEDIFF") - ddl should include("STRUCT FIELDS") + val ddl = table.ddl.replaceAll("\\s+", " ") + + ddl should include("CREATE OR REPLACE TABLE users") + ddl should include("id INT NOT NULL COMMENT 'user identifier'") + ddl should include( + """name VARCHAR FIELDS ( raw KEYWORD COMMENT 'sortable' ) DEFAULT 'anonymous' OPTIONS (analyzer = "french", search_analyzer = "french")""" + ) + ddl should include("birthdate DATE") + ddl should include("age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))") + ddl should include("ingested_at TIMESTAMP DEFAULT _ingest.timestamp") + ddl should include("profile STRUCT FIELDS (") + ddl should include("bio VARCHAR") + ddl should include("followers INT") + ddl should include("join_date DATE") + ddl should include("seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))") ddl should include("PRIMARY KEY (id)") ddl should include("PARTITION BY birthdate (MONTH)") } @@ -210,10 +272,12 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala """CREATE TABLE IF NOT EXISTS accounts ( | id INT NOT NULL, | owner VARCHAR, - | balance DECIMAL(10,2) - |) WITH ( - | number_of_shards = 1, - | number_of_replicas = 0 + | balance DOUBLE + |) OPTIONS ( + | settings = ( + | number_of_shards = 1, + | number_of_replicas = 0 + | ) |);""".stripMargin assertDdl(client.run(sql).futureValue) @@ -229,28 +293,28 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala """CREATE TABLE IF NOT EXISTS accounts_src ( | id INT NOT NULL, | name VARCHAR, - | active BOOLEAN + | active BOOLEAN, + | PRIMARY KEY (id) |);""".stripMargin assertDdl(client.run(createSource).futureValue) val insertSource = """INSERT INTO accounts_src (id, name, active) VALUES - | (1, 'Alice', true), - | (2, 'Bob', false), - | (3, 'Chloe', true);""".stripMargin + | (1, 'Alice', true), + | (2, 'Bob', false), + | (3, 'Chloe', true);""".stripMargin assertDml(client.run(insertSource).futureValue) val createOrReplace = - """CREATE OR REPLACE TABLE users_cr AS - |SELECT id, name FROM accounts_src WHERE active = true;""".stripMargin + "CREATE OR REPLACE TABLE users_cr AS SELECT id, name FROM accounts_src WHERE active = true;" - assertDdl(client.run(createOrReplace).futureValue) + assertDml(client.run(createOrReplace).futureValue) // Vérification via SHOW TABLE val table = assertShowTable(client.run("SHOW TABLE users_cr").futureValue) - table.ddl should include("CREATE TABLE users_cr") + table.ddl should include("CREATE OR REPLACE TABLE users_cr") table.ddl should include("id INT") table.ddl should include("name VARCHAR") } @@ -437,6 +501,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val table = assertShowTable(client.run("SHOW TABLE users_alter5").futureValue) table.ddl should include("reputation DOUBLE DEFAULT 0.0") + table.defaultPipeline.processors.size shouldBe 2 } // --------------------------------------------------------------------------- @@ -446,7 +511,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala it should "set and drop NOT NULL on a column" in { val create = """CREATE TABLE IF NOT EXISTS users_alter6 ( - | id INT NOT NULL, + | id INT, | status VARCHAR |);""".stripMargin @@ -482,12 +547,12 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val alter = """ALTER TABLE users_alter7 - | ALTER COLUMN status SET DATA TYPE BIGINT;""".stripMargin + | ALTER COLUMN id SET DATA TYPE BIGINT;""".stripMargin assertDdl(client.run(alter).futureValue) val table = assertShowTable(client.run("SHOW TABLE users_alter7").futureValue) - table.ddl should include("status BIGINT") + table.ddl should include("id BIGINT NOT NULL") } // --------------------------------------------------------------------------- @@ -575,8 +640,8 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val create = """CREATE TABLE IF NOT EXISTS dml_accounts ( | id INT NOT NULL, - | owner VARCHAR, - | balance DECIMAL(10,2) + | owner KEYWORD, + | balance DOUBLE |);""".stripMargin assertDdl(client.run(create).futureValue) @@ -591,7 +656,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val update = """UPDATE dml_accounts - |SET balance = balance + 25 + |SET balance = 125 |WHERE owner = 'Alice';""".stripMargin val res = client.run(update).futureValue @@ -618,7 +683,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val create = """CREATE TABLE IF NOT EXISTS dml_logs ( | id INT NOT NULL, - | level VARCHAR, + | level KEYWORD, | message VARCHAR |);""".stripMargin @@ -675,7 +740,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val update = """UPDATE dml_chain - |SET value = value * 2 + |SET value = 50 |WHERE id IN (1, 3);""".stripMargin assertDml(client.run(update).futureValue) @@ -706,11 +771,13 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val create = """CREATE TABLE IF NOT EXISTS dql_users ( | id INT NOT NULL, - | name VARCHAR, + | name VARCHAR FIELDS( + | raw KEYWORD + | ) OPTIONS (fielddata = true), | age INT, | birthdate DATE, | profile STRUCT FIELDS( - | city VARCHAR, + | city VARCHAR OPTIONS (fielddata = true), | followers INT | ) |);""".stripMargin @@ -719,13 +786,13 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val insert = """INSERT INTO dql_users (id, name, age, birthdate, profile) VALUES - | (1, 'Alice', 30, '1994-01-01', { "city": "Paris", "followers": 100 }), - | (2, 'Bob', 40, '1984-05-10', { "city": "Lyon", "followers": 50 }), - | (3, 'Chloe', 25, '1999-07-20', { "city": "Paris", "followers": 200 }), - | (4, 'David', 50, '1974-03-15', { "city": "Marseille", "followers": 10 }); + | (1, 'Alice', 30, '1994-01-01', {city = "Paris", followers = 100}), + | (2, 'Bob', 40, '1984-05-10', {city = "Lyon", followers = 50}), + | (3, 'Chloe', 25, '1999-07-20', {city = "Paris", followers = 200}), + | (4, 'David', 50, '1974-03-15', {city = "Marseille", followers = 10}); |""".stripMargin - assertDml(client.run(insert).futureValue) + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 4))) } // --------------------------------------------------------------------------- @@ -742,7 +809,39 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |ORDER BY id ASC;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "full_name" -> "Alice", + "city" -> "Paris", + "followers" -> 100, + "profile" -> Map("city" -> "Paris", "followers" -> 100) + ), + Map( + "id" -> 2, + "full_name" -> "Bob", + "city" -> "Lyon", + "followers" -> 50, + "profile" -> Map("city" -> "Lyon", "followers" -> 50) + ), + Map( + "id" -> 3, + "full_name" -> "Chloe", + "city" -> "Paris", + "followers" -> 200, + "profile" -> Map("city" -> "Paris", "followers" -> 200) + ), + Map( + "id" -> 4, + "full_name" -> "David", + "city" -> "Marseille", + "followers" -> 10, + "profile" -> Map("city" -> "Marseille", "followers" -> 10) + ) + ) + ) } // --------------------------------------------------------------------------- @@ -756,7 +855,15 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |SELECT id, name FROM dql_users WHERE age <= 30;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map("id" -> 2, "name" -> "Bob"), + Map("id" -> 4, "name" -> "David"), + Map("id" -> 1, "name" -> "Alice"), + Map("id" -> 3, "name" -> "Chloe") + ) + ) } // --------------------------------------------------------------------------- @@ -768,31 +875,56 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala """CREATE TABLE IF NOT EXISTS dql_orders ( | id INT NOT NULL, | customer_id INT, - | items ARRAY FIELDS( + | product VARCHAR OPTIONS (fielddata = true), | quantity INT, | price DOUBLE - | )> + | ) OPTIONS (include_in_parent = false) |);""".stripMargin assertDdl(client.run(create).futureValue) + val table = assertShowTable(client.run("SHOW TABLE dql_orders").futureValue) + table.ddl should include("items ARRAY FIELDS") + val insert = """INSERT INTO dql_orders (id, customer_id, items) VALUES - | (1, 1, [ { "product": "A", "quantity": 2, "price": 10.0 }, - | { "product": "B", "quantity": 1, "price": 20.0 } ]), - | (2, 2, [ { "product": "C", "quantity": 3, "price": 5.0 } ]);""".stripMargin + | (1, 1, [ { product = "A", quantity = 2, price = 10.0 }, + | { product = "B", quantity = 1, price = 20.0 } ]), + | (2, 2, [ { product = "C", quantity = 3, price = 5.0 } ]);""".stripMargin - assertDml(client.run(insert).futureValue) + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 2))) val sql = - """SELECT o.id, i.product, i.quantity + """SELECT + | o.id, + | items.product, + | items.quantity, + | SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price |FROM dql_orders o - |JOIN UNNEST(o.items) AS i + |JOIN UNNEST(o.items) AS items + |WHERE items.quantity >= 1 |ORDER BY o.id ASC;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "items" -> Seq( + Map("product" -> "A", "quantity" -> 2, "price" -> 10.0), + Map("product" -> "B", "quantity" -> 1, "price" -> 20.0) + ), + "total_price" -> 40.0 + ), + Map( + "id" -> 2, + "items" -> Seq(Map("product" -> "C", "quantity" -> 3, "price" -> 5.0)), + "total_price" -> 15.0 + ) + ) + ) } // --------------------------------------------------------------------------- @@ -808,7 +940,13 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |ORDER BY age DESC;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map("id" -> 1, "name" -> "Alice", "age" -> 30), + Map("id" -> 3, "name" -> "Chloe", "age" -> 25) + ) + ) } // --------------------------------------------------------------------------- @@ -838,7 +976,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |FROM dql_users |GROUP BY profile.city |HAVING COUNT(*) >= 1 - |ORDER BY cnt DESC;""".stripMargin + |ORDER BY COUNT(*) DESC;""".stripMargin val res = client.run(sql).futureValue assertSelectResult(res) @@ -888,11 +1026,11 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val sql = """SELECT id, | ABS(age) AS abs_age, - | CEIL(age / 3.0) AS ceil_div, - | FLOOR(age / 3.0) AS floor_div, - | ROUND(age / 3.0, 2) AS round_div, + | CEIL(age) AS ceil_div, + | FLOOR(age) AS floor_div, + | ROUND(age, 2) AS round_div, | SQRT(age) AS sqrt_age, - | POWER(age, 2) AS pow_age, + | POW(age, 2) AS pow_age, | LOG(age) AS log_age, | LOG10(age) AS log10_age, | EXP(age) AS exp_age, @@ -912,20 +1050,77 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala it should "support string functions" in { val sql = """SELECT id, - | CONCAT(name, '_suffix') AS name_concat, - | SUBSTRING(name, 1, 2) AS name_sub, - | LOWER(name) AS name_lower, - | UPPER(name) AS name_upper, - | TRIM(name) AS name_trim, - | LENGTH(name) AS name_len, - | REPLACE(name, 'A', 'X') AS name_repl, - | LEFT(name, 1) AS name_left, - | RIGHT(name, 1) AS name_right, - | REVERSE(name) AS name_rev - |FROM dql_users;""".stripMargin + | CONCAT(name.raw, '_suffix') AS name_concat, + | SUBSTRING(name.raw, 1, 2) AS name_sub, + | LOWER(name.raw) AS name_lower, + | UPPER(name.raw) AS name_upper, + | TRIM(name.raw) AS name_trim, + | LENGTH(name.raw) AS name_len, + | REPLACE(name.raw, 'A', 'X') AS name_repl, + | LEFT(name.raw, 1) AS name_left, + | RIGHT(name.raw, 1) AS name_right, + | REVERSE(name.raw) AS name_rev + |FROM dql_users + |ORDER BY id ASC;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "name_concat" -> Seq("Alice_suffix"), + "name_sub" -> Seq("Al"), + "name_lower" -> Seq("alice"), + "name_upper" -> Seq("ALICE"), + "name_trim" -> Seq("Alice"), + "name_len" -> Seq(5), + "name_repl" -> Seq("Xlice"), + "name_left" -> Seq("A"), + "name_right" -> Seq("e"), + "name_rev" -> Seq("ecilA") + ), + Map( + "id" -> 2, + "name_concat" -> Seq("Bob_suffix"), + "name_sub" -> Seq("Bo"), + "name_lower" -> Seq("bob"), + "name_upper" -> Seq("BOB"), + "name_trim" -> Seq("Bob"), + "name_len" -> Seq(3), + "name_repl" -> Seq("Bob"), + "name_left" -> Seq("B"), + "name_right" -> Seq("b"), + "name_rev" -> Seq("boB") + ), + Map( + "id" -> 3, + "name_concat" -> Seq("Chloe_suffix"), + "name_sub" -> Seq("Ch"), + "name_lower" -> Seq("chloe"), + "name_upper" -> Seq("CHLOE"), + "name_trim" -> Seq("Chloe"), + "name_len" -> Seq(5), + "name_repl" -> Seq("Chloe"), + "name_left" -> Seq("C"), + "name_right" -> Seq("e"), + "name_rev" -> Seq("eolhC") + ), + Map( + "id" -> 4, + "name_concat" -> Seq("David_suffix"), + "name_sub" -> Seq("Da"), + "name_lower" -> Seq("david"), + "name_upper" -> Seq("DAVID"), + "name_trim" -> Seq("David"), + "name_len" -> Seq(5), + "name_repl" -> Seq("David"), + "name_left" -> Seq("D"), + "name_right" -> Seq("d"), + "name_rev" -> Seq("divaD") + ) + ) + ) } // --------------------------------------------------------------------------- @@ -935,10 +1130,32 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala it should "support date and time functions" in { val sql = """SELECT id, + | YEAR(CURRENT_DATE) AS current_year, + | MONTH(CURRENT_DATE) AS current_month, + | DAY(CURRENT_DATE) AS current_day, + | WEEKDAY(CURRENT_DATE) AS current_weekday, + | YEARDAY(CURRENT_DATE) AS current_yearday, + | HOUR(CURRENT_TIMESTAMP) AS current_hour, + | MINUTE(CURRENT_TIMESTAMP) AS current_minute, + | SECOND(CURRENT_TIMESTAMP) AS current_second, + | NANOSECOND(CURRENT_TIMESTAMP) AS current_nano, + | MICROSECOND(CURRENT_TIMESTAMP) AS current_micro, + | MILLISECOND(CURRENT_TIMESTAMP) AS current_milli, | YEAR(birthdate) AS year_b, | MONTH(birthdate) AS month_b, | DAY(birthdate) AS day_b, + | WEEKDAY(birthdate) AS weekday_b, + | YEARDAY(birthdate) AS yearday_b, + | HOUR(birthdate) AS hour_b, + | MINUTE(birthdate) AS minute_b, + | SECOND(birthdate) AS second_b, + | NANOSECOND(birthdate) AS nano_b, + | MICROSECOND(birthdate) AS micro_b, + | MILLISECOND(birthdate) AS milli_b, + | OFFSET_SECONDS(birthdate) AS epoch_day, + | DATE_TRUNC(birthdate, MONTH) AS trunc_month, | DATE_ADD(birthdate, INTERVAL 1 DAY) AS plus_one_day, + | DATE_SUB(birthdate, INTERVAL 1 DAY) AS minus_one_day, | DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years |FROM dql_users;""".stripMargin @@ -954,23 +1171,26 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val create = """CREATE TABLE IF NOT EXISTS dql_geo ( | id INT NOT NULL, - | lon DOUBLE, - | lat DOUBLE + | location GEO_POINT, + | PRIMARY KEY (id) |);""".stripMargin assertDdl(client.run(create).futureValue) + val table = assertShowTable(client.run("SHOW TABLE dql_geo").futureValue) + table.ddl should include("location GEO_POINT") + table.find("location").exists(_.dataType == SQLTypes.GeoPoint) shouldBe true + val insert = - """INSERT INTO dql_geo (id, lon, lat) VALUES - | (1, 2.3522, 48.8566), - | (2, 4.8357, 45.7640);""".stripMargin + """INSERT INTO dql_geo (id, location) VALUES + | (1, {lon = 2.3522, lat = 48.8566}), + | (2, {lon = 4.8357, lat = 45.7640});""".stripMargin - assertDml(client.run(insert).futureValue) + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 2))) val sql = """SELECT id, - | POINT(lon, lat) AS point, - | ST_DISTANCE(POINT(lon, lat), POINT(2.3522, 48.8566)) AS dist_paris + | ST_DISTANCE(location, POINT(2.3522, 48.8566)) AS dist_paris |FROM dql_geo;""".stripMargin val res = client.run(sql).futureValue @@ -985,7 +1205,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val create = """CREATE TABLE IF NOT EXISTS dql_sales ( | id INT NOT NULL, - | product VARCHAR, + | product KEYWORD, | customer VARCHAR, | amount DOUBLE, | ts TIMESTAMP @@ -1000,7 +1220,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala | (3, 'B', 'C1', 30.0, '2024-01-01T12:00:00Z'), | (4, 'A', 'C3', 40.0, '2024-01-01T13:00:00Z');""".stripMargin - assertDml(client.run(insert).futureValue) + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 4))) val sql = """SELECT @@ -1008,7 +1228,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala | customer, | amount, | SUM(amount) OVER (PARTITION BY product) AS sum_per_product, - | COUNT(*) OVER (PARTITION BY product) AS cnt_per_product, + | COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product, | FIRST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS first_amount, | LAST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS last_amount, | ARRAY_AGG(amount) OVER (PARTITION BY product ORDER BY ts ASC LIMIT 10) AS amounts_array @@ -1016,7 +1236,51 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |ORDER BY product, ts;""".stripMargin val res = client.run(sql).futureValue - assertSelectResult(res) + assertSelectResult( + res, + Seq( + Map( + "product" -> "A", + "customer" -> "C1", + "amount" -> 10.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "A", + "customer" -> "C2", + "amount" -> 20.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "A", + "customer" -> "C3", + "amount" -> 40.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "B", + "customer" -> "C1", + "amount" -> 30.0, + "sum_per_product" -> 30.0, + "cnt_per_product" -> 1, + "first_amount" -> 30.0, + "last_amount" -> 30.0, + "amounts_array" -> Seq(30.0) + ) + ) + ) } // =========================================================================== @@ -1042,7 +1306,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala | SCRIPT ( | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", | lang = "painless", - | source = "def p1 = ctx.birthdate; def p2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (p1 == null) ? null : ChronoUnit.YEARS.between(p1, p2)", + | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))", | ignore_failure = true | ), | SET ( @@ -1055,7 +1319,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala | SCRIPT ( | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", | lang = "painless", - | source = "def p1 = ctx.profile?.join_date; def p2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (p1 == null) ? null : ChronoUnit.DAYS.between(p1, p2)", + | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))", | ignore_failure = true | ), | DATE_INDEX_NAME ( @@ -1109,42 +1373,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala } // =========================================================================== - // 6. TEMPLATES — CREATE / ALTER / DROP - // =========================================================================== - - behavior of "TEMPLATE statements" - - it should "create, alter and drop index templates via SQL if supported" in { - val create = - """CREATE TEMPLATE logs_template - |PATTERN 'logs-*' - |WITH ( - | number_of_shards = 1, - | number_of_replicas = 1 - |);""".stripMargin - - val createRes = client.run(create).futureValue - - // Certains backends Elasticsearch ne supportent pas les templates via SQL - if (createRes.isSuccess) { - assertDdl(createRes) - - val alter = - """ALTER TEMPLATE logs_template - |SET ( number_of_replicas = 2 );""".stripMargin - - assertDdl(client.run(alter).futureValue) - - val drop = "DROP TEMPLATE logs_template;" - assertDdl(client.run(drop).futureValue) - - } else { - log.warn("Templates not supported by this backend, skipping template tests.") - } - } - - // =========================================================================== - // 7. ERRORS — parsing errors, unsupported SQL + // 6. ERRORS — parsing errors, unsupported SQL // =========================================================================== behavior of "SQL error handling" @@ -1170,7 +1399,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val res = client.run(unsupportedSql).futureValue res.isFailure shouldBe true - res.toEither.left.get.message should include("Unsupported SQL statement") + res.toEither.left.get.message should include("Error parsing schema DDL statement") } } From ff7be3ebe980b7920a7e46cfe1fdcf8a2c955ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 17:51:33 +0100 Subject: [PATCH 64/95] fix mapping api specifications --- .../elastic/client/MappingApiSpec.scala | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index 75f3558b..7bae66b9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -22,7 +22,7 @@ class MappingApiSpec val mockLogger: Logger = mock[Logger] // Valid test data - val validMapping: String = """{"properties":{"name":{"type":"text"}}}""" + val validMapping: String = """{"_doc":{"properties":{"name":{"type":"text"}}}}""" val validSettings: String = """{"my-index":{"settings":{"index":{"number_of_shards":"1"}}}}""" val updatedMapping: String = """{"properties":{"name":{"type":"text"},"age":{"type":"integer"}}}""" @@ -129,6 +129,7 @@ class MappingApiSpec result.isSuccess shouldBe true result.get shouldBe true + verify(mockLogger).info("✅ Elasticsearch version: 0.0.0") verify(mockLogger).debug(s"Setting mapping for index 'my-index': $validMapping") verify(mockLogger).info("✅ Mapping for index 'my-index' updated successfully") } @@ -553,9 +554,12 @@ class MappingApiSpec result.isSuccess shouldBe true result.get shouldBe true - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("✅ Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("✅ Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("✅ Index 'my-index' created successfully") - verify(mockLogger, atLeastOnce).info("✅ Mapping for index 'my-index' set successfully") + verify(mockLogger, atLeastOnce).info("✅ Index 'my-index' created with mapping successfully") } "do nothing when mapping is up to date" in { @@ -615,16 +619,15 @@ class MappingApiSpec "fail when setMapping fails during creation" in { // Given mappingApi.executeIndexExistsResult = ElasticSuccess(false) - mappingApi.executeCreateIndexResult = ElasticSuccess(true) - val error = ElasticError("Mapping invalid", statusCode = Some(400)) - mappingApi.executeSetMappingResult = ElasticFailure(error) // When - val result = mappingApi.updateMapping("my-index", validMapping) + val result = mappingApi.updateMapping("my-index", "invalid mapping") // Then result.isFailure shouldBe true - result.error.get.message should include("Mapping invalid") + result.error.get.message should include( + "Invalid mappings: Invalid JSON: Unrecognized token 'invalid'" + ) } "rollback when migration fails" in { @@ -874,9 +877,12 @@ class MappingApiSpec // Then result.isSuccess shouldBe true - verify(mockLogger).info("Creating new index 'products' with mapping") + verify(mockLogger).debug("Checking if index 'products' exists") + verify(mockLogger).debug("✅ Index 'products' does not exist") + verify(mockLogger).info(s"Creating index 'products' with settings: $validSettings") + verify(mockLogger).info("✅ Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("✅ Index 'products' created successfully") - verify(mockLogger).info("✅ Mapping for index 'products' set successfully") + verify(mockLogger).info("✅ Index 'products' created with mapping successfully") } "successfully update existing index with new mapping" in { @@ -1304,14 +1310,13 @@ class MappingApiSpec result.error.get.message shouldBe "Cannot retrieve mapping" } - "handle partial failure in createIndexWithMapping" in { + "handle partial failure in createIndex" in { // Given mappingApi.executeIndexExistsResult = ElasticSuccess(false) - mappingApi.executeCreateIndexResult = ElasticSuccess(true) // setMapping fails val error = ElasticError("Invalid mapping structure", statusCode = Some(400)) - mappingApi.executeSetMappingResult = ElasticFailure(error) + mappingApi.executeCreateIndexResult = ElasticFailure(error) // When val result = mappingApi.updateMapping("my-index", validMapping) @@ -1504,12 +1509,15 @@ class MappingApiSpec mappingApi.executeSetMappingResult = ElasticSuccess(true) // When - mappingApi.updateMapping("my-index", validMapping) + mappingApi.updateMapping("my-index", validMapping, validSettings) // Then - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("✅ Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("✅ Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("✅ Index 'my-index' created successfully") - verify(mockLogger).info("✅ Mapping for index 'my-index' set successfully") + verify(mockLogger).info("✅ Index 'my-index' created with mapping successfully") } /*"log backup failure" in { @@ -1798,9 +1806,12 @@ class MappingApiSpec // Then result.isSuccess shouldBe true - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("✅ Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("✅ Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("✅ Index 'my-index' created successfully") - verify(mockLogger).info("✅ Mapping for index 'my-index' set successfully") + verify(mockLogger).info("✅ Index 'my-index' created with mapping successfully") } "handle multi-tenant index mapping update" in { From 3db49063deb804b077fa825982aa6b1fbfe39c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 18:14:21 +0100 Subject: [PATCH 65/95] fix parser specifications --- .../app/softnetwork/elastic/sql/parser/ParserSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index f8d0a151..51f8977e 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -994,16 +994,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date","null_value":"_ingest.timestamp"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M","description":"PARTITION BY birthdate (MONTH)","separator":"-"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) From 52894b526c0d124682ae927948d7f21f9453383a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 18:26:28 +0100 Subject: [PATCH 66/95] fix sql query specifications --- .../elastic/sql/SQLQuerySpec.scala | 427 +++++++++--------- .../elastic/sql/SQLQuerySpec.scala | 235 +++++----- 2 files changed, 314 insertions(+), 348 deletions(-) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index e947fd4b..cd630515 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -426,20 +426,20 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested select" in { val select: ElasticSearchRequest = SelectStatement(""" - |SELECT - |profileId, - |profile_ccm.email as email, - |profile_ccm.city as city, - |profile_ccm.firstName as firstName, - |profile_ccm.lastName as lastName, - |profile_ccm.postalCode as postalCode, - |profile_ccm.birthYear as birthYear - |FROM index join unnest(index.profiles) as profile_ccm - |WHERE - |((profile_ccm.postalCode BETWEEN "10" AND "99999") - |AND - |(profile_ccm.birthYear <= 2000)) - |limit 100""".stripMargin) + |SELECT + |profileId, + |profile_ccm.email as email, + |profile_ccm.city as city, + |profile_ccm.firstName as firstName, + |profile_ccm.lastName as lastName, + |profile_ccm.postalCode as postalCode, + |profile_ccm.birthYear as birthYear + |FROM index join unnest(index.profiles) as profile_ccm + |WHERE + |((profile_ccm.postalCode BETWEEN "10" AND "99999") + |AND + |(profile_ccm.birthYear <= 2000)) + |limit 100""".stripMargin) val query = select.query println(query) query shouldBe @@ -474,17 +474,15 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "profile_ccm", | "from": 0, - | "_source": { - | "includes": [ - | "profiles.email", - | "profiles.postalCode", - | "profiles.firstName", - | "profiles.lastName", - | "profiles.birthYear", - | "profiles.city" - | ] - | }, - | "size": 100 + | "size": 100, + | "docvalue_fields": [ + | "profiles.email", + | "profiles.city", + | "profiles.firstName", + | "profiles.lastName", + | "profiles.postalCode", + | "profiles.birthYear" + | ] | } | } | } @@ -855,7 +853,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -1198,47 +1196,47 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { println(query) query shouldBe """{ - | "query": { - | "bool": { - | "filter": [ - | { - | "exists": { - | "field": "identifier2" - | } - | } - | ] - | } - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "identifier": { - | "terms": { - | "field": "identifier", - | "min_doc_count": 1, - | "order": { - | "ct": "desc" - | } - | }, - | "aggs": { - | "ct": { - | "value_count": { - | "field": "identifier2" - | } - | }, - | "lastSeen": { - | "max": { - | "field": "createdAt", - | "script": { - | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" - | } - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "identifier2" + | } + | } + | ] + | } + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "identifier": { + | "terms": { + | "field": "identifier", + | "min_doc_count": 1, + | "order": { + | "ct": "desc" + | } + | }, + | "aggs": { + | "ct": { + | "value_count": { + | "field": "identifier2" + | } + | }, + | "lastSeen": { + | "max": { + | "field": "createdAt", + | "script": { + | "lang": "painless", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + | } + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("defp", "def p") .replaceAll("defe", "def e") @@ -1282,49 +1280,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z'))); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1366,47 +1364,47 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { println(query) query shouldBe """{ - | "query": { - | "bool": { - | "filter": [ - | { - | "exists": { - | "field": "identifier2" - | } - | } - | ] - | } - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "identifier": { - | "terms": { - | "field": "identifier", - | "min_doc_count": 1, - | "order": { - | "ct": "desc" - | } - | }, - | "aggs": { - | "lastSeen": { - | "max": { - | "field": "createdAt", - | "script": { - | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" - | } - | } - | }, - | "ct": { - | "value_count": { - | "field": "identifier2" - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "identifier2" + | } + | } + | ] + | } + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "identifier": { + | "terms": { + | "field": "identifier", + | "min_doc_count": 1, + | "order": { + | "ct": "desc" + | } + | }, + | "aggs": { + | "lastSeen": { + | "max": { + | "field": "createdAt", + | "script": { + | "lang": "painless", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | } + | } + | }, + | "ct": { + | "value_count": { + | "field": "identifier2" + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("defp", "def p") .replaceAll("defe", "def e") @@ -1451,7 +1449,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1497,7 +1495,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" | } | } | }, @@ -1548,7 +1546,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param3 = (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param3, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1934,7 +1932,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", | "params": { | "__now__": 1767139200000 | } @@ -1989,7 +1987,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", | "params": { | "__now__": 1767139200000 | } @@ -2052,7 +2050,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", | "params": { | "__now__": 1767139200000 | } @@ -2263,91 +2261,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2480,7 +2478,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1)) > 100.0" | } | } | } @@ -2491,61 +2489,61 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Double.valueOf(Math.abs(param1)) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.ceil(param1))" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.floor(param1))" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1))" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.exp(param1))" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log(param1))" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log10(param1))" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.pow(param1, 3))" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "sign_identifier": { @@ -2557,43 +2555,43 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.cos(param1))" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.acos(param1))" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sin(param1))" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.asin(param1))" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.tan(param1))" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan(param1))" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan2(param1, 3.0))" | } | } | }, @@ -2679,7 +2677,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(3, ((String)param1).length()))" | } | }, | "tr": { @@ -2709,13 +2707,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(5, ((String)param1).length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(((String)param1).length() - (int)Math.min(3, ((String)param1).length()))" | } | }, | "rep": { @@ -2757,7 +2755,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("def_", "def _") .replaceAll("=_", " = _") .replaceAll(",_", ", _") - .replaceAll(",\\(", ", (") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") .replaceAll(":\\(", " : (") @@ -2994,91 +2991,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z'))); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3134,7 +3131,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3158,7 +3155,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon) < 2000000.0" | } | } | }, @@ -3177,7 +3174,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3187,7 +3184,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3364,26 +3361,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "comments.replies.reply_author", - | "comments.replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.replies.reply_author", + | "comments.replies.reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3459,13 +3452,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "replies.reply_author", - | "replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "replies.reply_author", + | "replies.reply_text" + | ] | } | } | } @@ -3484,13 +3475,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3581,26 +3570,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "reply_author", - | "reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "reply_author", + | "reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "author", - | "comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "author", + | "comments" + | ] | } | } | } @@ -3612,8 +3597,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") - .replaceAll("\\s+", "") - .replaceAll("\\s+", "") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 0526b085..4265603d 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -474,17 +474,15 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "profile_ccm", | "from": 0, - | "_source": { - | "includes": [ - | "profiles.email", - | "profiles.city", - | "profiles.firstName", - | "profiles.lastName", - | "profiles.postalCode", - | "profiles.birthYear" - | ] - | }, - | "size": 100 + | "size": 100, + | "docvalue_fields": [ + | "profiles.email", + | "profiles.city", + | "profiles.firstName", + | "profiles.lastName", + | "profiles.postalCode", + | "profiles.birthYear" + | ] | } | } | } @@ -855,7 +853,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -1282,49 +1280,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z'))); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1451,7 +1449,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1497,7 +1495,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" | } | } | }, @@ -1548,7 +1546,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param3 = (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param3, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1934,7 +1932,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", | "params": { | "__now__": 1767139200000 | } @@ -1989,7 +1987,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", | "params": { | "__now__": 1767139200000 | } @@ -2052,7 +2050,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", | "params": { | "__now__": 1767139200000 | } @@ -2263,91 +2261,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2480,7 +2478,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1)) > 100.0" | } | } | } @@ -2491,61 +2489,61 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Double.valueOf(Math.abs(param1)) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.ceil(param1))" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.floor(param1))" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1))" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.exp(param1))" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log(param1))" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log10(param1))" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.pow(param1, 3))" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "sign_identifier": { @@ -2557,43 +2555,43 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.cos(param1))" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.acos(param1))" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sin(param1))" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.asin(param1))" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.tan(param1))" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan(param1))" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan2(param1, 3.0))" | } | } | }, @@ -2679,7 +2677,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(3, ((String)param1).length()))" | } | }, | "tr": { @@ -2709,13 +2707,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(5, ((String)param1).length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(((String)param1).length() - (int)Math.min(3, ((String)param1).length()))" | } | }, | "rep": { @@ -2757,7 +2755,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("def_", "def _") .replaceAll("=_", " = _") .replaceAll(",_", ", _") - .replaceAll(",\\(", ", (") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") .replaceAll(":\\(", " : (") @@ -2994,91 +2991,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z'))); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3134,7 +3131,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3158,7 +3155,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon) < 2000000.0" | } | } | }, @@ -3177,7 +3174,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3187,7 +3184,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3364,26 +3361,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "comments.replies.reply_author", - | "comments.replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.replies.reply_author", + | "comments.replies.reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3459,13 +3452,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "replies.reply_author", - | "replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "replies.reply_author", + | "replies.reply_text" + | ] | } | } | } @@ -3484,13 +3475,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3581,26 +3570,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "reply_author", - | "reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "reply_author", + | "reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "author", - | "comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "author", + | "comments" + | ] | } | } | } @@ -3612,8 +3597,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") - .replaceAll("\\s+", "") - .replaceAll("\\s+", "") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") From 53cb3024bcb481b27876b8e47a04aa76cabef18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 19:04:14 +0100 Subject: [PATCH 67/95] fix jest specifications --- .../elastic/client/JestClientInsertByQuerySpec.scala | 4 ++-- .../elastic/client/JestClientTemplateApiSpec.scala | 4 ++-- .../app/softnetwork/elastic/client/JestSqlGatewaySpec.scala | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala index 06ab8d14..631ca626 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JestClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JestClientInsertByQuerySpec extends InsertByQuerySpec with EmbeddedElasticTestKit { +class JestClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { override def client: ElasticClientApi = new JestClientSpi().client(elasticConfig) override def elasticVersion: String = "6.7.2" diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala index 17681307..3882b343 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JestClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JestClientTemplateApiSpec extends TemplateApiSpec with EmbeddedElasticTestKit { +class JestClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { override def client: TemplateApi with VersionApi = new JestClientSpi().client(elasticConfig) override def elasticVersion: String = "6.7.2" diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala index e63c19c1..aa26c4be 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JestClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JestSqlGatewaySpec extends SqlGatewayIntegrationSpec with EmbeddedElasticTestKit { +class JestSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { override def client: SqlGateway = new JestClientSpi().client(elasticConfig) override def elasticVersion: String = "6.7.2" From 8dd20e52e99e89ac5c7472518ed0a994fcd7e4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 19:10:37 +0100 Subject: [PATCH 68/95] add SQL Gateway delegators --- .../client/ElasticClientDelegator.scala | 10 +++++++++ .../client/metrics/MetricsElasticClient.scala | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 0f597ba7..082ab08e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -1782,4 +1782,14 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { templateName: String ): ElasticResult[Boolean] = delegate.executeLegacyTemplateExists(templateName) + + // ==================== Gateway (delegate) ==================== + + override def run(sql: String)(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = + delegate.run(sql) + + override def run(statement: query.Statement)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] = + delegate.run(statement) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index b6e09136..7c4ebbe5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -1339,4 +1339,25 @@ class MetricsElasticClient( measureResult("templateExists") { delegate.templateExists(templateName) } + + // ==================== Gateway (delegate) ==================== + + override def run( + sql: String + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + measureAsync("run") { + delegate.run(sql) + } + } + + override def run( + statement: query.Statement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + measureAsync("run") { + delegate.run(statement) + } + } + } From 947e3401125d6e164344d376146cad3e1c83c33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 6 Jan 2026 20:06:50 +0100 Subject: [PATCH 69/95] update RHL client specifications using ElasticDockerTestKit --- .../softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala | 4 ++-- .../elastic/client/RestHighLevelClientCompanionSpec.scala | 4 ++-- .../elastic/client/RestHighLevelClientInsertByQuerySpec.scala | 4 ++-- .../elastic/client/RestHighLevelClientPipelineApiSpec.scala | 4 ++-- .../elastic/client/RestHighLevelClientSqlGatewaySpec.scala | 4 ++-- .../elastic/client/RestHighLevelClientTemplateApiSpec.scala | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala index e8db8fbc..9193e155 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala @@ -1,8 +1,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelBulkApiSpec extends BulkApiSpec with EmbeddedElasticTestKit { +class RestHighLevelBulkApiSpec extends BulkApiSpec with ElasticDockerTestKit { override lazy val client: BulkApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala index 89867667..71d5e8f9 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.rest.RestHighLevelClientCompanion -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit import app.softnetwork.persistence.generateUUID import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers @@ -16,7 +16,7 @@ import scala.util.Try class RestHighLevelClientCompanionSpec extends AnyWordSpec - with EmbeddedElasticTestKit + with ElasticDockerTestKit with Matchers with ScalaFutures { diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala index 2f81b747..5dbbd428 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala @@ -1,8 +1,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with EmbeddedElasticTestKit { +class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { override lazy val client: ElasticClientApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala index 2c38f9ce..b10763b7 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala @@ -1,8 +1,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with EmbeddedElasticTestKit { +class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { override lazy val client: PipelineApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala index 5e57e684..20305fa9 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala @@ -1,10 +1,10 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit class RestHighLevelClientSqlGatewaySpec extends SqlGatewayIntegrationSpec - with EmbeddedElasticTestKit { + with ElasticDockerTestKit { override lazy val client: SqlGateway = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala index c2a90a24..67b1622c 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala @@ -1,9 +1,9 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with EmbeddedElasticTestKit { +class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { override lazy val client: TemplateApi with VersionApi = new RestHighLevelClientSpi().client(elasticConfig) } From 43920d91ea983a0593f0c13276f2ce82d01f7153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 06:48:43 +0100 Subject: [PATCH 70/95] add support for SHOW and DESCRIBE PIPELINE --- .../elastic/client/ElasticClientApi.scala | 2 +- .../client/ElasticClientDelegator.scala | 5 +- .../{SqlGateway.scala => GatewayApi.scala} | 62 +++++++++++++++---- .../elastic/client/IndicesApi.scala | 8 +-- .../elastic/client/PipelineApi.scala | 29 +++++++++ .../client/metrics/MetricsElasticClient.scala | 7 ++- .../elastic/client/result/package.scala | 12 +++- ...waySpec.scala => JestGatewayApiSpec.scala} | 4 +- ...> RestHighLevelClientGatewayApiSpec.scala} | 6 +- ...> RestHighLevelClientGatewayApiSpec.scala} | 6 +- ...c.scala => JavaClientGatewayApiSpec.scala} | 4 +- ...c.scala => JavaClientGatewayApiSpec.scala} | 4 +- .../elastic/sql/parser/Parser.scala | 14 ++++- .../elastic/sql/query/package.scala | 8 +++ ....scala => GatewayApiIntegrationSpec.scala} | 21 ++++++- 15 files changed, 154 insertions(+), 38 deletions(-) rename core/src/main/scala/app/softnetwork/elastic/client/{SqlGateway.scala => GatewayApi.scala} (95%) rename es6/jest/src/test/scala/app/softnetwork/elastic/client/{JestSqlGatewaySpec.scala => JestGatewayApiSpec.scala} (65%) rename es6/rest/src/test/scala/app/softnetwork/elastic/client/{RestHighLevelClientSqlGatewaySpec.scala => RestHighLevelClientGatewayApiSpec.scala} (61%) rename es7/rest/src/test/scala/app/softnetwork/elastic/client/{RestHighLevelClientSqlGatewaySpec.scala => RestHighLevelClientGatewayApiSpec.scala} (61%) rename es8/java/src/test/scala/app/softnetwork/elastic/client/{JavaClientSqlGatewaySpec.scala => JavaClientGatewayApiSpec.scala} (62%) rename es9/java/src/test/scala/app/softnetwork/elastic/client/{JavaClientSqlGatewaySpec.scala => JavaClientGatewayApiSpec.scala} (62%) rename testkit/src/main/scala/app/softnetwork/elastic/client/{SqlGatewayIntegrationSpec.scala => GatewayApiIntegrationSpec.scala} (98%) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 13b9f2c1..964a4967 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -49,7 +49,7 @@ trait ElasticClientApi with SerializationApi with PipelineApi with TemplateApi - with SqlGateway + with GatewayApi with ClientCompanion { protected def logger: Logger diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 082ab08e..a45c174a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index -import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.{query, schema} import app.softnetwork.elastic.sql.query.{ DqlStatement, SQLAggregation, @@ -1645,6 +1645,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.getPipeline(pipelineName) } + override def loadPipeline(pipelineName: String): ElasticResult[schema.IngestPipeline] = + delegate.loadPipeline(pipelineName) + override private[client] def executeCreatePipeline( pipelineName: String, pipelineDefinition: String diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala similarity index 95% rename from core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala rename to core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 201a257b..43fcfbfa 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SqlGateway.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -24,6 +24,7 @@ import app.softnetwork.elastic.client.result.{ ElasticFailure, ElasticResult, ElasticSuccess, + QueryPipeline, QueryResult, QueryRows, QueryStream, @@ -37,6 +38,7 @@ import app.softnetwork.elastic.sql.query.{ CreateTable, DdlStatement, Delete, + DescribePipeline, DescribeTable, DmlStatement, DqlStatement, @@ -45,6 +47,7 @@ import app.softnetwork.elastic.sql.query.{ MultiSearch, PipelineStatement, SelectStatement, + ShowPipeline, ShowTable, SingleSearch, Statement, @@ -216,18 +219,51 @@ class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[Pip statement: PipelineStatement )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { implicit val ec: ExecutionContext = system.dispatcher - // handle PIPELINE statement - api.pipeline(statement) match { - case ElasticSuccess(result) => - logger.info(s"✅ Executed pipeline statement: $statement.") - Future.successful(ElasticResult.success(DdlResult(result))) - case ElasticFailure(elasticError) => - logger.error(s"❌ Error executing pipeline statement: $statement. ${elasticError.message}") - Future.successful( - ElasticFailure( - elasticError.copy(operation = Some("pipeline")) - ) - ) + statement match { + case show: ShowPipeline => + // handle SHOW PIPELINE statement + api.loadPipeline(show.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"✅ Retrieved pipeline ${show.name}.") + Future.successful(ElasticResult.success(QueryPipeline(pipeline))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case describe: DescribePipeline => + // handle DESCRIBE PIPELINE statement + api.loadPipeline(describe.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"✅ Retrieved pipeline ${describe.name}.") + Future.successful( + ElasticResult.success(QueryRows(pipeline.processors.map(_.properties))) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case _ => + // handle PIPELINE statement + api.pipeline(statement) match { + case ElasticSuccess(result) => + logger.info(s"✅ Executed pipeline statement: $statement.") + Future.successful(ElasticResult.success(DdlResult(result))) + case ElasticFailure(elasticError) => + logger.error( + s"❌ Error executing pipeline statement: $statement. ${elasticError.message}" + ) + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } } } } @@ -1066,7 +1102,7 @@ class DdlRouterExecutor( } } -trait SqlGateway extends ElasticClientHelpers { +trait GatewayApi extends ElasticClientHelpers { _: IndicesApi with PipelineApi with MappingApi diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 71f4f1ff..acafdffd 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -227,10 +227,7 @@ trait IndicesApi extends ElasticClientHelpers { case ElasticSuccess(Some(idx)) => ElasticSuccess(idx.schema) case ElasticSuccess(None) => - logger.error( - s"❌ Failed to load schema from template for index '$index'" - ) - ElasticFailure( + val error = ElasticError( message = s"Failed to load schema from template for index '$index'", cause = None, @@ -238,7 +235,8 @@ trait IndicesApi extends ElasticClientHelpers { index = Some(index), operation = Some("loadSchema") ) - ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) case ElasticFailure(error) => logger.error( s"❌ Failed to load schema from template for index '$index': ${error.message}" diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala index 385f75a3..67b253f5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -315,6 +315,35 @@ trait PipelineApi extends ElasticClientHelpers { _: VersionApi => } } + def loadPipeline(pipelineName: String): ElasticResult[IngestPipeline] = { + getPipeline(pipelineName) match { + case ElasticSuccess(Some(pipelineJson)) => + ElasticResult.attempt { + IngestPipeline(name = pipelineName, json = pipelineJson) + } match { + case success @ ElasticSuccess(_) => + success + case ElasticFailure(error) => + logger.error(s"❌ Failed to parse pipeline '$pipelineName': ${error.message}") + ElasticResult.failure( + error.copy(operation = Some("loadPipeline")) + ) + } + case ElasticSuccess(None) => + val error = + ElasticError.notFound( + resource = "Pipeline", + name = pipelineName, + operation = "loadPipeline" + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to load pipeline '$pipelineName': ${error.message}") + failure + } + } + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 7c4ebbe5..7b78425f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -31,7 +31,7 @@ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.schema.Index -import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.{query, schema} import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} import org.json4s.Formats @@ -1260,6 +1260,11 @@ class MetricsElasticClient( delegate.getPipeline(pipelineName) } + override def loadPipeline(pipelineName: String): ElasticResult[schema.IngestPipeline] = + measureResult("loadPipeline") { + delegate.loadPipeline(pipelineName) + } + // ==================== TemplateApi (delegate) ==================== /** Create or update an index template. diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 479ecb6e..61f41e3b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.stream.scaladsl.Source import app.softnetwork.elastic.client.scroll.ScrollMetrics -import app.softnetwork.elastic.sql.schema.Table +import app.softnetwork.elastic.sql.schema.{IngestPipeline, Table} import scala.util.control.NonFatal @@ -217,6 +217,14 @@ package object result { operation = Some(operation) ) } + + def notFound(resource: String, name: String, operation: String): ElasticError = { + ElasticError( + s"$resource '$name' not found during operation '$operation'", + statusCode = Some(404), + operation = Some(operation) + ) + } } /** Companion object with utility methods. @@ -411,4 +419,6 @@ package object result { case class DdlResult(success: Boolean) extends QueryResult case class QueryTable(table: Table) extends QueryResult + + case class QueryPipeline(pipeline: IngestPipeline) extends QueryResult } diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala similarity index 65% rename from es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala rename to es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala index aa26c4be..a79e53e9 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestSqlGatewaySpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala @@ -3,8 +3,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JestClientSpi import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JestSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { - override def client: SqlGateway = new JestClientSpi().client(elasticConfig) +class JestGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override def client: GatewayApi = new JestClientSpi().client(elasticConfig) override def elasticVersion: String = "6.7.2" } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala similarity index 61% rename from es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala rename to es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala index 20305fa9..bd302beb 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala @@ -3,8 +3,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelClientSqlGatewaySpec - extends SqlGatewayIntegrationSpec +class RestHighLevelClientGatewayApiSpec + extends GatewayApiIntegrationSpec with ElasticDockerTestKit { - override lazy val client: SqlGateway = new RestHighLevelClientSpi().client(elasticConfig) + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala similarity index 61% rename from es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala rename to es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala index 20305fa9..bd302beb 100644 --- a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSqlGatewaySpec.scala +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala @@ -3,8 +3,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelClientSqlGatewaySpec - extends SqlGatewayIntegrationSpec +class RestHighLevelClientGatewayApiSpec + extends GatewayApiIntegrationSpec with ElasticDockerTestKit { - override lazy val client: SqlGateway = new RestHighLevelClientSpi().client(elasticConfig) + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala similarity index 62% rename from es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala rename to es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala index b7ae07c7..d84c3345 100644 --- a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala @@ -3,6 +3,6 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JavaClientSpi import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JavaClientSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { - override lazy val client: SqlGateway = new JavaClientSpi().client(elasticConfig) +class JavaClientGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) } diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala similarity index 62% rename from es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala rename to es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala index b7ae07c7..d84c3345 100644 --- a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSqlGatewaySpec.scala +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala @@ -3,6 +3,6 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.JavaClientSpi import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class JavaClientSqlGatewaySpec extends SqlGatewayIntegrationSpec with ElasticDockerTestKit { - override lazy val client: SqlGateway = new JavaClientSpi().client(elasticConfig) +class JavaClientGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index ebe6c583..b95ea3af 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -163,6 +163,16 @@ object Parser DropPipeline(name, ifExists = ie) } + def showPipeline: PackratParser[ShowPipeline] = + ("SHOW" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + ShowPipeline(pipeline) + } + + def describePipeline: PackratParser[DescribePipeline] = + ("DESCRIBE" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + DescribePipeline(pipeline) + } + def addProcessor: PackratParser[AddPipelineProcessor] = ("ADD" ~ "PROCESSOR") ~ processor ^^ { case _ ~ proc => AddPipelineProcessor(proc) @@ -563,7 +573,9 @@ object Parser truncateTable | showTable | describeTable | - dropPipeline + dropPipeline | + showPipeline | + describePipeline def onConflict: PackratParser[OnConflict] = ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index ee64b909..c7f366c9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -498,6 +498,14 @@ package object query { } } + case class ShowPipeline(name: String) extends PipelineStatement { + override def sql: String = s"SHOW PIPELINE $name" + } + + case class DescribePipeline(name: String) extends PipelineStatement { + override def sql: String = s"DESCRIBE PIPELINE $name" + } + sealed trait TableStatement extends DdlStatement case class CreateTable( diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala similarity index 98% rename from testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala rename to testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index 1361256b..9d734c24 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/SqlGatewayIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, ElasticResult, + QueryPipeline, QueryResult, QueryRows, QueryStream, @@ -31,7 +32,7 @@ import app.softnetwork.elastic.client.result.{ import app.softnetwork.elastic.client.scroll.ScrollMetrics import app.softnetwork.elastic.scalatest.ElasticTestKit import app.softnetwork.elastic.sql.`type`.SQLTypes -import app.softnetwork.elastic.sql.schema.Table +import app.softnetwork.elastic.sql.schema.{IngestPipeline, Table} import app.softnetwork.persistence.generateUUID import org.scalatest.concurrent.ScalaFutures import org.scalatest.flatspec.AnyFlatSpecLike @@ -47,7 +48,7 @@ import java.util.concurrent.TimeUnit // Base test trait — to be mixed with ElasticDockerTestKit // --------------------------------------------------------------------------- -trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { +trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { self: ElasticTestKit => lazy val log: Logger = LoggerFactory getLogger getClass.getName @@ -57,7 +58,7 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(30, Seconds)) // Provided by concrete test class - def client: SqlGateway + def client: GatewayApi override def beforeAll(): Unit = { self.beforeAll() @@ -162,6 +163,16 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala res.toOption.get.asInstanceOf[QueryTable].table } + // ------------------------------------------------------------------------- + // Helper: assert SHOW PIPELINE result type + // ------------------------------------------------------------------------- + + def assertShowPipeline(res: ElasticResult[QueryResult]): IngestPipeline = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryPipeline] + res.toOption.get.asInstanceOf[QueryPipeline].pipeline + } + // ------------------------------------------------------------------------- // SHOW / DESCRIBE TABLE tests // ------------------------------------------------------------------------- @@ -1341,6 +1352,10 @@ trait SqlGatewayIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala |);""".stripMargin assertDdl(client.run(sql).futureValue) + + val pipeline = assertShowPipeline(client.run("SHOW PIPELINE user_pipeline").futureValue) + pipeline.name shouldBe "user_pipeline" + pipeline.processors.size shouldBe 6 } // --------------------------------------------------------------------------- From eae0a6c8390a3669af63eb99f2c563f2a3b4ef6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 08:39:17 +0100 Subject: [PATCH 71/95] add support for COPY INTO --- .../softnetwork/elastic/client/BulkApi.scala | 1 + .../elastic/client/GatewayApi.scala | 24 ++- .../elastic/client/IndicesApi.scala | 144 +++++++++++++++++- .../elastic/client/file/package.scala | 25 +-- .../elastic/client/file/FileSourceSpec.scala | 1 + .../elastic/sql/parser/Parser.scala | 20 ++- .../elastic/sql/query/package.scala | 49 ++++++ .../elastic/client/BulkApiSpec.scala | 2 +- .../client/GatewayApiIntegrationSpec.scala | 83 ++++++++++ 9 files changed, 318 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala index c2bdf29a..c396d50a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala @@ -23,6 +23,7 @@ import akka.stream.scaladsl.{Balance, Flow, GraphDSL, Merge, Sink, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.file._ import app.softnetwork.elastic.client.result.{ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.sql.query.{FileFormat, Unknown} import app.softnetwork.elastic.sql.schema.sqlConfig import org.apache.hadoop.conf.Configuration diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index 43fcfbfa..a3b3d1c5 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -34,6 +34,7 @@ import app.softnetwork.elastic.client.result.{ import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ AlterTable, + CopyInto, CreatePipeline, CreateTable, DdlStatement, @@ -190,14 +191,33 @@ class DmlExecutor(api: IndicesApi, logger: Logger) extends Executor[DmlStatement } case insert: Insert => api.insertByQuery(insert.table, insert.sql).map { - case ElasticSuccess(res) => + case success @ ElasticSuccess(res) => logger.info(s"✅ Inserted ${res.inserted} documents into ${insert.table}.") - ElasticResult.success(res) + success case ElasticFailure(elasticError) => ElasticFailure( elasticError.copy(operation = Some("schema")) ) } + case copy: CopyInto => + api + .copyInto( + copy.source, + copy.targetTable, + doUpdate = copy.onConflict.exists(_.doUpdate), + fileFormat = copy.fileFormat + ) + .map { + case success @ ElasticSuccess(res) => + logger.info( + s"✅ Copied ${res.inserted} documents into ${copy.targetTable} from ${copy.source}." + ) + success + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } case _ => // unsupported DML statement val error = diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index acafdffd..680f4ee9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -22,14 +22,23 @@ import app.softnetwork.elastic.client.bulk.BulkOptions import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.schema.Index import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{Delete, From, Insert, SingleSearch, Table, Update} +import app.softnetwork.elastic.sql.query.{ + Delete, + FileFormat, + From, + Insert, + SingleSearch, + Table, + Unknown, + Update +} import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, Schema, TableAlias} import app.softnetwork.elastic.sql.serialization._ import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import org.apache.hadoop.conf.Configuration import scala.concurrent.{ExecutionContext, Future} - import scala.jdk.CollectionConverters._ /** Index management API. @@ -1147,6 +1156,105 @@ trait IndicesApi extends ElasticClientHelpers { } } + /** Copy documents from files into an index. + * + * @param source + * - the source file path (can be local or remote) + * @param target + * - the target index name + * @param doUpdate + * - true to update existing documents, false to insert only + * @param fileFormat + * - optional file format (if not provided, will be inferred from file extension) + * @return + * the number of documents inserted/updated + */ + def copyInto( + source: String, + target: String, + doUpdate: Boolean, + fileFormat: Option[FileFormat] = None, + hadoopConf: Option[Configuration] = None + )(implicit + system: ActorSystem + ): Future[ElasticResult[DmlResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val result = for { + // 1. Validate target index + _ <- Future.fromTry( + validateIndexName(target) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("copyInto"), + statusCode = Some(400), + index = Some(target) + ) + ) + .toTry + ) + + // 2 Load target metadata + idx <- Future.fromTry(getIndex(target).toEither.flatMap { + case Some(i) => Right(i.schema) + case None => + Left(ElasticError.notFound(target, "copyInto")) + }.toTry) + + // 3. Derive bulk options + idKey = idx.primaryKey match { + case Nil => None + case pk => Some(pk.toSet) + } + suffixKey = idx.partitionBy.map(_.column) + suffixPattern = idx.partitionBy.flatMap(_.dateFormats.headOption) + + // 4. Bulk file copy + bulkResult <- bulkFromFile( + filePath = source, + format = fileFormat.getOrElse(Unknown), + indexKey = Some(target), + idKey = idKey, + suffixDateKey = suffixKey, + suffixDatePattern = suffixPattern, + update = Some(doUpdate), + hadoopConf = hadoopConf + )(BulkOptions(defaultIndex = target), system) + + } yield bulkResult + + result + .map(r => ElasticSuccess(DmlResult(inserted = r.successCount, rejected = r.failedCount))) + .recover { + case e: ElasticError => + ElasticFailure( + e.copy( + operation = Some("copyInto"), + index = Some(target) + ) + ) + case e => + ElasticFailure( + ElasticError( + message = e.getMessage, + operation = Some("copyInto"), + index = Some(target) + ) + ) + } + } + + /** Parse a query for update, determining if it's SQL or JSON. + * + * @param index + * - the name of the index to update + * @param query + * - the query (SQL or JSON) + * @return + * either the parsed SQL Update statement or the validated JSON query + */ private def parseQueryForUpdate( index: String, query: String @@ -1354,6 +1462,14 @@ trait IndicesApi extends ElasticClientHelpers { } } + /** Open an index if it is closed, returning a function to restore its state. + * + * @param index + * - the name of the index to open if needed + * @return + * a tuple containing a boolean indicating if the index was opened and a function to restore + * its state + */ private def openIfNeeded( index: String ): Either[ElasticError, (Boolean, () => ElasticResult[Boolean])] = { @@ -1374,6 +1490,19 @@ trait IndicesApi extends ElasticClientHelpers { } } + /** Resolve the final ingest pipeline to use for updateByQuery, merging user and SQL pipelines if + * needed. + * + * @param user + * - optional user-provided ingest pipeline + * @param sql + * - optional SQL-derived ingest pipeline + * @param elasticVersion + * - Elasticsearch version + * @return + * a tuple containing the final pipeline id to use (if any) and a boolean indicating if the + * pipeline is temporary and must be deleted after use + */ private def resolveFinalPipeline( user: Option[IngestPipeline], sql: Option[IngestPipeline], @@ -1461,7 +1590,16 @@ trait IndicesApi extends ElasticClientHelpers { } } - def parseInsertQuery( + /** Parse an SQL INSERT query. + * + * @param index + * - the name of the index to insert into + * @param query + * - the SQL INSERT query + * @return + * the parsed Insert statement + */ + private def parseInsertQuery( index: String, query: String ): ElasticResult[Insert] = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala index dd0c6a38..2758373e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.stream.scaladsl.Source +import app.softnetwork.elastic.sql.query.{Delta, FileFormat, Json, JsonArray, Parquet, Unknown} import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.fasterxml.jackson.core.{JsonFactory, JsonParser, JsonToken} import com.fasterxml.jackson.databind.node.ObjectNode @@ -44,30 +45,6 @@ package object file { private val logger: Logger = LoggerFactory.getLogger("FileSource") - sealed trait FileFormat { - def name: String - } - - case object Parquet extends FileFormat { - override def name: String = "Parquet" - } - - case object Json extends FileFormat { - override def name: String = "JSON" - } - - case object JsonArray extends FileFormat { - override def name: String = "JSON Array" - } - - case object Delta extends FileFormat { - override def name: String = "Delta Lake" - } - - case object Unknown extends FileFormat { - override def name: String = "Unknown" - } - /** Hadoop configuration with optimizations for local file system */ def hadoopConfiguration: Configuration = { val conf = new Configuration() diff --git a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala index 453441c9..c303d39d 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala @@ -2,6 +2,7 @@ package app.softnetwork.elastic.client.file import akka.actor.ActorSystem import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.sql.query.{Delta, Json, JsonArray, Parquet, Unknown} import app.softnetwork.elastic.sql.serialization.JacksonConfig import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.ScalaFutures diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index b95ea3af..8b1495fc 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -603,6 +603,21 @@ object Parser } } + def fileFormat: PackratParser[FileFormat] = + ("FILE_FORMAT" ~> ( + ("PARQUET" ^^^ Parquet) | + ("JSON" ^^^ Json) | + ("JSON_ARRAY" ^^^ JsonArray) | + ("DELTA_LAKE" ^^^ Delta) + )) ^^ { ff => ff } + + /** COPY INTO table FROM source */ + def copy: PackratParser[CopyInto] = + ("COPY" ~ "INTO") ~ ident ~ ("FROM" ~> literal) ~ opt(fileFormat) ~ opt(onConflict) ^^ { + case _ ~ table ~ source ~ format ~ conflict => + CopyInto(source.value, table, fileFormat = format, onConflict = conflict) + } + /** UPDATE table SET col1 = v1, col2 = v2 [WHERE ...] */ def update: PackratParser[Update] = ("UPDATE" ~> ident) ~ ("SET" ~> repsep(ident ~ "=" ~ value, separator)) ~ where.? ^^ { @@ -617,7 +632,7 @@ object Parser Delete(Table(table), w) } - def dmlStatement: PackratParser[DmlStatement] = insert | update | delete + def dmlStatement: PackratParser[DmlStatement] = insert | update | delete | copy def statement: PackratParser[Statement] = ddlStatement | dqlStatement | dmlStatement @@ -692,11 +707,14 @@ trait Parser "select", "insert", "update", + "copy", "delete", "create", "alter", "drop", "truncate", + "table", + "pipeline", "column", "from", "join", diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index c7f366c9..a176a9f9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -428,6 +428,55 @@ package object query { s"DELETE FROM ${table.name}${where.map(w => s" ${w.sql}").getOrElse("")}" } + sealed trait FileFormat extends Token { + def name: String + + override def sql: SQL = s" FILE_FORMAT = $name" + } + + case object Parquet extends FileFormat { + override def name: String = "PARQUET" + } + + case object Json extends FileFormat { + override def name: String = "JSON" + } + + case object JsonArray extends FileFormat { + override def name: String = "JSON_ARRAY" + } + + case object Delta extends FileFormat { + override def name: String = "DELTA_LAKE" + } + + case object Unknown extends FileFormat { + override def name: String = "UNKNOWN" + } + + object FileFormat { + def apply(format: String): FileFormat = { + format.toUpperCase match { + case "PARQUET" => Parquet + case "JSON" => Json + case "JSON_ARRAY" => JsonArray + case "DELTA_LAKE" => Delta + case _ => Unknown + } + } + } + + case class CopyInto( + source: String, + targetTable: String, + fileFormat: Option[FileFormat] = None, + onConflict: Option[OnConflict] = None + ) extends DmlStatement { + override def sql: String = { + s"COPY INTO $targetTable FROM $source${asString(fileFormat)}${asString(onConflict)}" + } + } + sealed trait DdlStatement extends Statement sealed trait PipelineStatement extends DdlStatement diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala index c594194c..6c3c423d 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala @@ -18,8 +18,8 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.bulk.BulkOptions -import app.softnetwork.elastic.client.file._ import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.query.JsonArray import app.softnetwork.persistence.generateUUID import org.scalatest.concurrent.ScalaFutures import org.scalatest.flatspec.AnyFlatSpecLike diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index 9d734c24..b6fda660 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, ElasticResult, + ElasticSuccess, QueryPipeline, QueryResult, QueryRows, @@ -40,6 +41,7 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Seconds, Span} import org.slf4j.{Logger, LoggerFactory} +import java.time.LocalDate import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContextExecutor} import java.util.concurrent.TimeUnit @@ -767,6 +769,87 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala assertSelectResult(select) } + // --------------------------------------------------------------------------- + // COPY INTO integration test + // --------------------------------------------------------------------------- + + it should "support COPY INTO for JSONL and JSON_ARRAY formats" in { + // 1. Create table with primary key + val create = + """CREATE TABLE IF NOT EXISTS copy_into_test ( + | uuid KEYWORD NOT NULL, + | name VARCHAR, + | birthDate DATE, + | childrenCount INT, + | PRIMARY KEY (uuid) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + // 2. Prepare sample documents + val persons = List( + """{"uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0}""", + """{"uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0}""", + """{"uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 2}""" + ) + + // 3. Create a temporary JSONL file + val jsonlFile = java.io.File.createTempFile("copy_into_jsonl", ".jsonl") + jsonlFile.deleteOnExit() + val writer1 = new java.io.PrintWriter(jsonlFile) + persons.foreach(writer1.println) + writer1.close() + + // 4. COPY INTO using JSONL (format inferred) + val copyJsonl = + s"""COPY INTO copy_into_test FROM "${jsonlFile.getAbsolutePath}";""" + + val jsonlResult = client.run(copyJsonl).futureValue + assertDml(jsonlResult, Some(DmlResult(inserted = persons.size))) + + // 5. Create a temporary JSON_ARRAY file + val jsonArrayFile = java.io.File.createTempFile("copy_into_array", ".json") + jsonArrayFile.deleteOnExit() + val writer2 = new java.io.PrintWriter(jsonArrayFile) + writer2.println("[") + writer2.println(persons.mkString(",\n")) + writer2.println("]") + writer2.close() + + // 6. COPY INTO using JSON_ARRAY + val copyArray = + s"""COPY INTO copy_into_test FROM "${jsonArrayFile.getAbsolutePath}" FILE_FORMAT = JSON_ARRAY ON CONFLICT DO UPDATE;""" + + val arrayResult = client.run(copyArray).futureValue + assertDml(arrayResult, Some(DmlResult(inserted = persons.size))) + + // 7. Final verification: SELECT all documents + val select = client.run("SELECT * FROM copy_into_test ORDER BY uuid ASC").futureValue + assertSelectResult( + select, + Seq( + Map( + "uuid" -> "A12", + "name" -> "Homer Simpson", + "birthDate" -> LocalDate.parse("1967-11-21"), + "childrenCount" -> 0 + ), + Map( + "uuid" -> "A14", + "name" -> "Moe Szyslak", + "birthDate" -> LocalDate.parse("1967-11-21"), + "childrenCount" -> 0 + ), + Map( + "uuid" -> "A16", + "name" -> "Barney Gumble", + "birthDate" -> LocalDate.parse("1969-05-09"), + "childrenCount" -> 2 + ) + ) + ) + } + // =========================================================================== // 4. DQL — SELECT, JOIN, UNNEST, GROUP BY, HAVING, ORDER BY, LIMIT/OFFSET, // functions, window functions, nested fields, UNION ALL From 62dd7b0e9c9b665057353145dacf3c1e028f1d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 09:46:49 +0100 Subject: [PATCH 72/95] update alter table statements --- documentation/sql/ddl_statements.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index a0a2f1c8..6ebc6036 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -68,6 +68,7 @@ Modify an existing table. Multiple statements can be grouped inside parentheses. - `ALTER COLUMN [IF EXISTS] column_name SET DATA TYPE new_type` → Change the data type of an existing column. - `ALTER COLUMN [IF EXISTS] column_name SET|ADD FIELD field_definition` → Add or update a field inside a STRUCT or multi‑field. - `ALTER COLUMN [IF EXISTS] column_name DROP FIELD field_name` → Remove a field from a STRUCT or multi‑field. +- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column. - `SET|ADD MAPPING (key = value)` → Set table‑level mapping. - `DROP MAPPING key` → Remove table‑level mapping. - `SET|ADD SETTING (key = value)` → Set table‑level setting. @@ -75,8 +76,6 @@ Modify an existing table. Multiple statements can be grouped inside parentheses. [//]: # (- `ALTER COLUMN [IF EXISTS] column_name SET OPTIONS (...)`) [//]: # ( → Set multiple options for an existing column.) -[//]: # (- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` ) -[//]: # ( → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column.) **Examples:** ```sql From 8ccb04cd7d21a23abfd9564e82c9b1b8c7e5a512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 10:12:52 +0100 Subject: [PATCH 73/95] fix parser keywords --- .../main/scala/app/softnetwork/elastic/sql/parser/Parser.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 8b1495fc..41e98d74 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -713,8 +713,6 @@ trait Parser "alter", "drop", "truncate", - "table", - "pipeline", "column", "from", "join", From d48363ff60fddb8797b8df1893b4718ee5e39ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 10:13:02 +0100 Subject: [PATCH 74/95] update ddl documentation --- documentation/sql/ddl_statements.md | 725 +++++++++++++--------------- 1 file changed, 338 insertions(+), 387 deletions(-) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 6ebc6036..37af398b 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -1,137 +1,103 @@ -# Data Definition Language (DDL) Support -[Back to index](README.md) - -This document describes the SQL statements supported by the API, focusing on **Data Definition Language (DDL)**. Each section provides syntax, examples, and notes on behavior. +# 📘 **DDL Statements — SQL Gateway for Elasticsearch** --- -## 📐 Table (DDLs) - -### CREATE TABLE -Create a new table with explicit column definitions or from a `SELECT` query. +## Introduction -**Syntax:** -```sql -CREATE [OR REPLACE] TABLE [IF NOT EXISTS] table_name ( - column_name data_type [SCRIPT AS (sql) | FIELDS (...)] [DEFAULT value] [NOT NULL] [COMMENT 'comment'] [OPTIONS (...)], - [... more columns ...], - [PRIMARY KEY (column1, column2, ...)] -) [PARTITION BY column_name] OPTIONS ([mappings = (...)] , [settings = (...)] ); -``` +The SQL Gateway provides a full Data Definition Language (DDL) layer on top of Elasticsearch. +It allows defining tables, schemas, pipelines, mappings, and settings using a relational syntax while generating the appropriate Elasticsearch structures: -- `FIELDS (...)` can define **multi‑fields** (alternative analyzers for text) or **STRUCT** (nested objects). +- **indices** (for non-partitioned tables) +- **index templates** (for partitioned tables) +- **ingest pipelines** (default or user-defined) +- **mappings** and **settings** +- **metadata** (primary key, generated columns, comments, options) -**Examples:** -```sql -CREATE TABLE IF NOT EXISTS users ( - id INT NOT NULL COMMENT 'user identifier', - name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), - birthdate DATE, - age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), - ingested_at TIMESTAMP DEFAULT _ingest.timestamp, -- special field - PRIMARY KEY (id) -) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false)); +The DDL engine is: -CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts; -``` +- **version-aware** (ES6 → ES9) +- **client-agnostic** (Jest, RHLC, Java Client) +- **schema-driven** +- **round-trip safe** (DESCRIBE returns normalized SQL) --- -## 📝 Notes -- **Types**: Supported SQL types include `INT`, `BIGINT`, `VARCHAR`, `BOOLEAN`, `DATE`, `TIMESTAMP`, etc. -- **Constraints**: `NOT NULL` and `DEFAULT` are supported. Other relational constraints (e.g., `PRIMARY KEY`) are not enforced by Elasticsearch. -- **SCRIPT AS (sql)**: Defines a scripted column computed at ingestion time. -- **FIELDS**: Dual purpose — multi‑fields for text analysis and STRUCT for nested data modeling. -- **Comments**: Column comments can be added via `COMMENT 'text'`. -- **Options**: Column options can be specified via `OPTIONS (...)`. -- The **partition key must be of type `DATE`** and the partition column must be explicitly defined in the table schema. +## Table Model ---- +A SQL table corresponds to: -### ALTER TABLE -Modify an existing table. Multiple statements can be grouped inside parentheses. +| SQL Definition | Elasticsearch Structure | +|-----------------------------------------|----------------------------------------------------| +| `CREATE TABLE` without `PARTITIONED BY` | **Concrete index** | +| `CREATE TABLE` with `PARTITIONED BY` | **Index template** (legacy ES6 or composable ES7+) | + +### Index-backed table (no partitioning) -**Supported statements:** -- `ADD COLUMN [IF NOT EXISTS] column_definition` → Add a new column. -- `DROP COLUMN [IF EXISTS] column_name` → Remove an existing column. -- `RENAME COLUMN old_name TO new_name` → Rename an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET SCRIPT AS (sql)` → Define or update a scripted column. -- `ALTER COLUMN [IF EXISTS] column_name DROP SCRIPT` → Remove a scripted column. -- `ALTER COLUMN [IF EXISTS] column_name SET|ADD OPTION (key = value)` → Set a specific option for an existing column. -- `ALTER COLUMN [IF EXISTS] column_name DROP OPTION key` → Remove a specific option from an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET COMMENT 'comment'` → Set or update the comment for an existing column. -- `ALTER COLUMN [IF EXISTS] column_name DROP COMMENT` → Remove the comment from an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET DEFAULT value` → Set or update the default value for an existing column. -- `ALTER COLUMN [IF EXISTS] column_name DROP DEFAULT` → Remove the default value from an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET NOT NULL` → Make an existing column NOT NULL. -- `ALTER COLUMN [IF EXISTS] column_name DROP NOT NULL` → Remove the NOT NULL constraint from an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET DATA TYPE new_type` → Change the data type of an existing column. -- `ALTER COLUMN [IF EXISTS] column_name SET|ADD FIELD field_definition` → Add or update a field inside a STRUCT or multi‑field. -- `ALTER COLUMN [IF EXISTS] column_name DROP FIELD field_name` → Remove a field from a STRUCT or multi‑field. -- `ALTER COLUMN [IF EXISTS] column_name SET FIELDS (...)` → Allows defining nested fields (STRUCT) or multi‑fields inside an existing column. -- `SET|ADD MAPPING (key = value)` → Set table‑level mapping. -- `DROP MAPPING key` → Remove table‑level mapping. -- `SET|ADD SETTING (key = value)` → Set table‑level setting. -- `DROP SETTING key` → Remove table‑level setting. - -[//]: # (- `ALTER COLUMN [IF EXISTS] column_name SET OPTIONS (...)`) -[//]: # ( → Set multiple options for an existing column.) - -**Examples:** ```sql -ALTER TABLE users - ADD COLUMN IF NOT EXISTS age INT DEFAULT 0; - -ALTER TABLE users - RENAME COLUMN name TO full_name; - -ALTER TABLE users ( - ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, - RENAME COLUMN name TO full_name, - ALTER COLUMN IF EXISTS status SET DEFAULT 'active', - ALTER COLUMN IF EXISTS profile SET FIELDS ( - description VARCHAR DEFAULT 'N/A', - visibility BOOLEAN DEFAULT true - ) +CREATE TABLE users ( + id INT, + name VARCHAR, + PRIMARY KEY (id) ); ``` ---- +Creates: -### DROP TABLE -Remove an existing table. +- index `users` +- default pipeline `users_ddl_default_pipeline` +- mapping + settings -**Syntax:** -```sql -DROP TABLE [IF EXISTS] table_name [CASCADE] -``` +### Template-backed table (with partitioning) -**Example:** ```sql -DROP TABLE IF EXISTS users CASCADE; +CREATE TABLE users ( + id INT, + birthdate DATE, + PRIMARY KEY (id) +) +PARTITIONED BY (birthdate MONTH); ``` +Creates: + +- template `users` +- default pipeline with `date_index_name` +- indices generated dynamically: + - `users-2025-01` + - `users-2025-02` + --- -### TRUNCATE TABLE -Delete all rows from a table without removing its definition. +## Column Types & Mapping -**Syntax:** -```sql -TRUNCATE TABLE table_name -``` +The SQL Gateway supports the following type system: -**Example:** -```sql -TRUNCATE TABLE users; -``` +| SQL Type | Elasticsearch Mapping | +|---------------------|--------------------------------------| +| `NULL` | `null` | +| `TINYINT` | `byte` | +| `SMALLINT` | `short` | +| `INT` | `integer` | +| `BIGINT` | `long` | +| `DOUBLE` | `double` | +| `REAL` | `float` | +| `BOOLEAN` | `boolean` | +| `VARCHAR` \| `TEXT` | `text` + optional `keyword` subfield | +| `KEYWORD` | `keyword` | +| `DATE` | `date` | +| `TIMESTAMP` | `date` | +| `STRUCT` | `object` with nested properties | +| `ARRAY` | `nested` | +| `GEO_POINT` | `geo_point` | --- -## 🧩 Nested and Structured Data +### 🧩 Nested and Structured Data -### FIELDS for Multi‑fields -`FIELDS (...)` can be used to define **multi‑fields** for text columns. This allows you to index the same column in multiple ways (e.g., with different analyzers). +#### FIELDS for Multi-fields + +`FIELDS (...)` can be used to define **multi-fields** for text columns. +This allows indexing the same column in multiple ways (e.g., with different analyzers). **Example:** ```sql @@ -142,14 +108,16 @@ CREATE TABLE docs ( ) ) ``` + - `content` is indexed as text. -- `content.keyword` is a keyword sub‑field. -- `content.english` is a text sub‑field with the English analyzer. +- `content.keyword` is a keyword sub-field. +- `content.english` is a text sub-field with the English analyzer. --- -### FIELDS for STRUCT or NESTED OBJECTS -`FIELDS (...)` also enables the definition of **STRUCT** types, which may represent either object or nested objects with their own fields. This is how you model hierarchical data. +#### FIELDS for STRUCT or NESTED OBJECTS + +`FIELDS (...)` also enables the definition of **STRUCT** types, representing hierarchical data. **Example:** ```sql @@ -172,7 +140,9 @@ CREATE TABLE users ( - `profile` is a `STRUCT` column containing multiple fields. - `address` is a nested `STRUCT` inside `profile`. -This maps naturally to **Elasticsearch** `object` type. +--- + +#### FIELDS for ARRAY **Example:** ```sql @@ -186,91 +156,88 @@ CREATE TABLE store ( ) ``` -- `products` is an `ARRAY` column containing multiple fields. +- `products` is an `ARRAY` column. +- Maps naturally to Elasticsearch `nested`. -This maps naturally to **Elasticsearch** `nested` type. +--- -### 📝 Notes -- The meaning of `FIELDS (...)` depends on the column type: - - On `VARCHAR` types, it defines **multi‑fields**. - - On `STRUCT`, it defines the **fields of the struct**. - - On `ARRAY`, it defines the **fields of each element**. -- Sub‑fields defined inside `FIELDS (...)` support the full DDL syntax: +#### Notes + +- On `VARCHAR` → defines **multi-fields** +- On `STRUCT` → defines **object fields** +- On `ARRAY` → defines **nested fields** +- Sub-fields support: - nested `FIELDS` - - `SCRIPT AS (sql)` (for scripted sub‑fields) - `DEFAULT` - `NOT NULL` - `COMMENT` - `OPTIONS` -- Multi‑level nesting is supported. -- `SCRIPT AS (sql)` can not be used for `ARRAY`. + - `SCRIPT AS` (except inside ARRAY) +- Multi-level nesting is supported. --- -## 🔄 MappingApi Migration Workflow - -The `MappingApi` provides intelligent mapping management with **automatic migration, validation, and rollback capabilities**. This ensures that SQL commands such as `ALTER TABLE … ALTER COLUMN SET TYPE …` are safely translated into Elasticsearch operations. +## Constraints & Column Options -### ✨ Features -- ✅ **Automatic Change Detection**: Compares existing mappings with new ones -- ✅ **Safe Migration Strategy**: Creates temporary indices, reindexes, and renames atomically -- ✅ **Automatic Rollback**: Reverts to original state if migration fails -- ✅ **Backup & Restore**: Preserves original mappings and settings -- ✅ **Progress Tracking**: Detailed logging of migration steps -- ✅ **Validation**: Strict JSON validation with error reporting - ---- +### Primary Key -### 📊 Migration Workflow - -``` -SQL Command: ALTER TABLE users ALTER COLUMN age SET TYPE BIGINT - │ - ▼ -MappingApi Execution: - 1. Backup current mapping and settings - 2. Create temporary index with new mapping (age: long) - 3. Reindex data from original → temporary - 4. Delete original index - 5. Recreate original index with new mapping - 6. Reindex data from temporary → original - 7. Delete temporary index - 8. Rollback if any step fails +```sql +id INT, +PRIMARY KEY (id) ``` ---- +Used for: -### 📝 Notes -- **Atomicity**: The workflow ensures that schema changes are applied safely without downtime. -- **Transparency**: Users only see the SQL command; the migration logic is handled internally. -- **Consistency**: All data is reindexed into the new mapping, guaranteeing type correctness. -- **Resilience**: Rollback and backup mechanisms prevent data loss in case of errors. +- document ID generation +- upsert semantics +- COPY INTO conflict resolution --- -## 📅 Partitioned Tables - -SoftClient4ES supports **partitioned tables** via the `PARTITION BY` clause in `CREATE TABLE`. -This feature allows automatic routing of documents into indices partitioned by the value of a date column. +### 🔑 Composite Primary Keys ---- +SoftClient4ES supports composite primary keys in SQL. -### ✅ Supported Syntax +#### Syntax ```sql -CREATE TABLE [IF NOT EXISTS] table_name ( - column_definitions -) PARTITION BY column_name [(granularity)] +CREATE TABLE users ( + id INT NOT NULL, + birthdate DATE NOT NULL, + name VARCHAR, + PRIMARY KEY (id, birthdate) +); +``` + +#### Elasticsearch Translation + +```json +{ + "processors": [ + { + "set": { + "field": "_id", + "value": "{{id}}||{{birthdate}}" + } + } + ] +} ``` -- The **partition key must be of type `DATE`**. -- The partition column must be explicitly defined in the table schema. -- Granularity is optional. If omitted, defaults to `DAY`. -- Granularity can be explicitly set with `PARTITION BY column (YEAR|MONTH|DAY|HOUR|MINUTE|SECOND)`. -- Partitioning is typically used for time‑based data (e.g., `birthdate`, `event_date`). +#### Notes + +- Composite PK fields must be immutable. +- Avoid long `_id` values. +- Mapping rules: + - `PRIMARY KEY (id)` → `_id = id` + - `PRIMARY KEY (id, birthdate)` → `_id = "{{id}}-{{birthdate}}"` --- +## Partitioning + +Partitioning routes documents to time-based indices using `date_index_name`. + ### Supported Granularities | SQL Granularity | ES `date_rounding` | Example Index Name | @@ -284,331 +251,315 @@ CREATE TABLE [IF NOT EXISTS] table_name ( --- -### 📌 Example +## Pipelines in DDL + +## CREATE PIPELINE ```sql -CREATE TABLE IF NOT EXISTS users ( - id INT NOT NULL, - name VARCHAR DEFAULT 'anonymous', - birthdate DATE -) PARTITION BY birthdate (MONTH); +CREATE OR REPLACE PIPELINE user_pipeline +WITH PROCESSORS ( + SET ( + field = "name", + if = "ctx.name == null", + description = "DEFAULT 'anonymous'", + ignore_failure = true, + value = "anonymous" + ), + SCRIPT ( + description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + lang = "painless", + source = "...", + ignore_failure = true + ), + DATE_INDEX_NAME ( + field = "birthdate", + index_name_prefix = "users-", + date_formats = ["yyyy-MM"], + date_rounding = "M", + separator = "-", + ignore_failure = true + ) +); ``` -- Creates a table `users` partitioned by the `birthdate` column. -- Documents are routed into indices partitioned by **month** of `birthdate`. - For `birthdate = 2025-12-10`, the target index will be `users-2025-12`. +## DROP PIPELINE ---- +```sql +DROP PIPELINE IF EXISTS user_pipeline; +``` -### ⚙️ Elasticsearch Translation +## ALTER PIPELINE -- **Mapping**: the partition key must be declared as a `date` field in the index mapping. -- **Pipeline**: SoftClient4ES uses the [`date_index_name`](https://www.elastic.co/guide/en/elasticsearch/reference/current/date-index-name-processor.html) processor to route documents into partitioned indices. -- **Index Templates**: SoftClient4ES uses it to ensure consistent mappings and settings across all partitioned indices. +```sql +ALTER PIPELINE IF EXISTS user_pipeline ( + ADD PROCESSOR SET ( + field = "status", + if = "ctx.status == null", + description = "status DEFAULT 'active'", + ignore_failure = true, + value = "active" + ), + DROP PROCESSOR SET (_id) +); +``` --- -### 📝 Notes - -- **Granularity**: controlled via `date_rounding` (`y`, `M`, `d`, etc.). -- **Migration**: existing documents are reindexed to be redistributed 🔀 into the correct partitions. -- **SQL vs ES**: in SQL, `PARTITION BY` is a logical clause; in Elasticsearch, it is implemented via ingest pipelines and index naming. - ---- +## SHOW PIPELINE -## 🔑 Composite Primary Keys +```sql +SHOW PIPELINE pipeline_name; +``` -SoftClient4ES supports composite primary keys in SQL. -In SQL, a primary key can be defined on multiple columns (`PRIMARY KEY (col1, col2)`), ensuring uniqueness across the combination of values. -In Elasticsearch, uniqueness is enforced by the special `_id` field. To emulate composite primary keys, `_id` is constructed from multiple fields using the `set` processor. +**Description** ---- +- Returns a high‑level view of the pipeline processors -### ✅ Syntax +**Example** ```sql -CREATE TABLE users ( - id INT NOT NULL, - birthdate DATE NOT NULL, - name VARCHAR, - PRIMARY KEY (id, birthdate) -); +SHOW PIPELINE user_pipeline; ``` --- -### ⚙️ Elasticsearch Translation - -A pipeline is automatically created to set `_id` as a concatenation of the primary key columns: +## DESCRIBE PIPELINE -```curl -PUT _ingest/pipeline/users-composite-id -{ - "processors": [ - { - "set": { - "field": "_id", - "value": "{{id}}||{{birthdate}}" - } - } - ] -} +```sql +DESCRIBE PIPELINE pipeline_name; ``` -- `_id` is built from the values of `id` and `birthdate`. -- Example: `id = 42`, `birthdate = 2025-12-10` → `_id = "42|2025-12-10"`. -- The separator (`||`) can be customized to avoid collisions (sql.composite-key-separator). - ---- +**Description** -### 📝 Notes -- **Stability**: chosen fields must be immutable to preserve uniqueness. -- **Performance**: avoid overly long `_id` values. -- **SQL ↔ ES Mapping**: - - `PRIMARY KEY (id)` → `_id = id` - - `PRIMARY KEY (id, birthdate)` → `_id = "{{id}}-{{birthdate}}"` +- Returns the full, normalized definition of the pipeline: + - processors in execution order + - full configuration of each processor (`SET`, `SCRIPT`, `REMOVE`, `RENAME`, `DATE_INDEX_NAME`, etc.) + - flags such as `ignore_failure`, `if`, `description` ---- +**Example** -## Scripted Columns 🧮 - -SoftClient4ES supports scripted columns in `CREATE TABLE`. -These columns are **computed at ingestion time** using an ingest pipeline `script` processor. -The value is persisted in `_source` and behaves like any other field. +```sql +DESCRIBE PIPELINE user_pipeline; +``` --- -### ✅ Syntax +## CREATE TABLE + +### Basic Example ```sql -CREATE TABLE table_name ( - column_definitions, - scripted_column TYPE SCRIPT AS (sql_expression) +CREATE TABLE users ( + id INT, + name VARCHAR DEFAULT 'anonymous', + birthdate DATE, + age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), + PRIMARY KEY (id) ); ``` -- `scripted_column` is a regular column name. -- `TYPE` defines the target type (`INT`, `VARCHAR`, etc.). -- `SCRIPT AS (sql_expression)` defines the computation in SQL syntax. -- The expression is translated into **Painless** for Elasticsearch. - ---- - -### 📌 SQL → ES Translation +### Partitioned Example -**SQL:** ```sql CREATE TABLE users ( - id INT NOT NULL, - name VARCHAR, + id INT, birthdate DATE, - age INT SCRIPT AS (YEAR(CURRENT_DATE) - YEAR(birthdate)), PRIMARY KEY (id) -) PARTITION BY birthdate (MONTH); -``` - -**Elasticsearch pipeline:** -```curl -PUT _ingest/pipeline/users-pipeline -{ - "processors": [ - { - "script": { - "source": "ctx.age = ChronoUnit.YEARS.between(ctx.birthdate, Instant.now())" - } - } - ] -} +) +PARTITIONED BY (birthdate MONTH); ``` --- -### 📖 Examples +## ALTER TABLE -| SQL Script Expression | ES Painless Translation (ingest) | -|--------------------------------------------|--------------------------------------------------------------------| -| `YEAR(CURRENT_DATE) - YEAR(birthdate)` | `ctx.age = ChronoUnit.YEARS.between(ctx.birthdate, Instant.now())` | -| `UPPER(name)` | `ctx.name_upper = ctx.name.toUpperCase()` | -| `CONCAT(firstname, ' ', lastname)` | `ctx.fullname = ctx.firstname + ' ' + ctx.lastname` | -| `CASE WHEN status = 'X' THEN 1 ELSE 0 END` | `ctx.flag = ctx.status == 'X' ? 1 : 0` | +**Supported statements:** ---- +- `ADD COLUMN [IF NOT EXISTS] column_definition` +- `DROP COLUMN [IF EXISTS] column_name` +- `RENAME COLUMN old_name TO new_name` +- `ALTER COLUMN column_name SET SCRIPT AS (sql)` +- `ALTER COLUMN column_name DROP SCRIPT` +- `ALTER COLUMN column_name SET|ADD OPTION (key = value)` +- `ALTER COLUMN column_name DROP OPTION key` +- `ALTER COLUMN column_name SET COMMENT 'comment'` +- `ALTER COLUMN column_name DROP COMMENT` +- `ALTER COLUMN column_name SET DEFAULT value` +- `ALTER COLUMN column_name DROP DEFAULT` +- `ALTER COLUMN column_name SET NOT NULL` +- `ALTER COLUMN column_name DROP NOT NULL` +- `ALTER COLUMN column_name SET DATA TYPE new_type` +- `ALTER COLUMN column_name SET|ADD FIELD field_definition` +- `ALTER COLUMN column_name DROP FIELD field_name` +- `ALTER COLUMN column_name SET FIELDS (...)` +- `SET|ADD MAPPING (key = value)` +- `DROP MAPPING key` +- `SET|ADD SETTING (key = value)` +- `DROP SETTING key` + +--- + +## DROP TABLE + +```sql +DROP TABLE IF EXISTS users; +``` -### 📝 Notes -- Scripted columns are **evaluated once at ingestion**. -- They are **persisted** in `_source`, unlike runtime fields. -- SQL expressions are translated into equivalent **Painless** code. -- This feature allows declarative enrichment directly in the DDL. +Deletes: + +- index (non-partitioned) +- template (partitioned) --- -## 🧩 Pipeline DDLs +## TRUNCATE TABLE -Pipelines define ordered ingestion processors that transform documents before they are indexed. -The SQL‑ES dialect supports three pipeline‑related statements: +```sql +TRUNCATE TABLE users; +``` -- `CREATE PIPELINE` -- `DROP PIPELINE` -- `ALTER PIPELINE` +Deletes all documents while keeping: -These statements allow users to declare, replace, remove, or modify ingestion pipelines in a declarative SQL‑style syntax. +- mapping +- settings +- pipeline +- template (if any) --- -### 🚀 CREATE PIPELINE - -#### **Syntax** +## SHOW TABLE ```sql -CREATE [OR REPLACE] PIPELINE pipeline_name -[IF NOT EXISTS] -WITH PROCESSORS ( - processor_1, - processor_2, - ... -); +SHOW TABLE users; ``` -#### **Description** +Returns: -- Creates a new ingestion pipeline. -- `OR REPLACE` overwrites an existing pipeline. -- `IF NOT EXISTS` prevents an error if the pipeline already exists. -- Processors are executed **in the order they are declared**. +- index or template metadata +- primary key +- partitioning +- pipeline +- mapping summary -#### **Supported Processor Types** - -| SQL Processor | Elasticsearch Equivalent | Purpose | -|--------------|---------------------------|---------| -| `SET` | `set` | Assigns a value to a field in `ctx` | -| `SCRIPT` | `script` | Executes a Painless script | -| `REMOVE` | `remove` | Removes a field | -| `RENAME` | `rename` | Renames a field | -| `DATE_INDEX_NAME` | `date_index_name` | Generates an index name based on a date field | +--- -#### **Example** +## DESCRIBE TABLE ```sql -CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( - SET ( - field = "name", - if = "ctx.name == null", - description = "DEFAULT 'anonymous'", - ignore_failure = true, - value = "anonymous" - ), - SCRIPT ( - description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", - lang = "painless", - source = "...", - ignore_failure = true - ), - DATE_INDEX_NAME ( - field = "birthdate", - index_name_prefix = "users-", - date_formats = ["yyyy-MM"], - date_rounding = "M", - separator = "-", - ignore_failure = true - ) -); +DESCRIBE TABLE users; ``` ---- +Returns the **normalized SQL schema**, including: + +- columns +- types +- defaults +- scripts +- STRUCT fields +- PK +- options +- comments -### 🗑️ DROP PIPELINE +--- -#### **Syntax** +## CREATE TABLE AS SELECT (CTAS) ```sql -DROP PIPELINE [IF EXISTS] pipeline_name; +CREATE TABLE new_users AS +SELECT id, name FROM users; ``` -#### **Description** +The gateway: -- Deletes a pipeline. -- `IF EXISTS` prevents an error if the pipeline does not exist. +- infers the schema +- generates mappings +- creates index or template +- **populates data using the Bulk API** -#### **Example** +--- + +## 🔄 Index Migration Workflow + +### Initial Creation ```sql -DROP PIPELINE IF EXISTS user_pipeline; +CREATE TABLE users (...); ``` +Creates: + +- index or template +- default pipeline +- mapping + settings +- metadata (PK, defaults, scripts) + --- -### 🔧 ALTER PIPELINE +### Schema Evolution -#### **Syntax** +#### Add a column ```sql -ALTER PIPELINE [IF EXISTS] pipeline_name ( - alter_action_1, - alter_action_2, - ... -); +ALTER TABLE users ADD COLUMN last_login TIMESTAMP; ``` -#### **Supported Actions** +#### Modify a column -| Action | Description | -|--------|-------------| -| `ADD PROCESSOR ` | Appends a processor to the pipeline | -| `DROP PROCESSOR ()` | Removes a processor identified by its type and field | -| *(Optional future extension)* `ALTER PROCESSOR` | Modify an existing processor | +```sql +ALTER TABLE users ALTER COLUMN name SET OPTIONS (analyzer = 'french'); +``` -#### **Example** +#### Add a STRUCT field ```sql -ALTER PIPELINE IF EXISTS user_pipeline ( - ADD PROCESSOR SET ( - field = "status", - if = "ctx.status == null", - description = "status DEFAULT 'active'", - ignore_failure = true, - value = "active" - ), - DROP PROCESSOR SET (_id) -); +ALTER TABLE users ALTER COLUMN profile ADD FIELD followers INT; ``` ---- +#### Drop a column -### 📐 Semantic Rules +```sql +ALTER TABLE users DROP COLUMN old_field; +``` -#### **Processor Identity** +--- -A processor is uniquely identified by: +### Migration Safety -``` -(processor_type, field) -``` +The Gateway ensures: -This means: +- non-destructive updates +- mapping compatibility checks +- pipeline regeneration when needed +- template updates for partitioned tables +- index updates for non-partitioned tables -- Changing the type or field is equivalent to removing the old processor and adding a new one. -- Property changes are treated as processor modifications. +--- -#### **Processor Ordering** +### Full Replacement (CTAS) -Ordering matters for certain processors: +```sql +CREATE OR REPLACE TABLE users AS +SELECT id, name FROM old_users; +``` -- `DATE_INDEX_NAME` must appear last. -- `SET _id` must appear last. -- `RENAME` and `REMOVE` should appear before `SCRIPT`. +Steps: -Other processors may be reordered without semantic impact. +1. infer schema +2. create new index/template +3. bulk-copy data +4. atomically replace --- -### 📦 Summary of Supported Pipeline DDLs +## Version Compatibility -| Statement | Purpose | -|----------|----------| -| `CREATE PIPELINE` | Define or replace a pipeline | -| `DROP PIPELINE` | Remove a pipeline | -| `ALTER PIPELINE` | Add or remove processors | +| Feature | ES6 | ES7 | ES8 | ES9 | +|----------------------|------|------|------|------| +| Legacy templates | ✔ | ✔ | ✖ | ✖ | +| Composable templates | ✖ | ✔ | ✔ | ✔ | +| date_index_name | ✔ | ✔ | ✔ | ✔ | +| Generated scripts | ✔ | ✔ | ✔ | ✔ | +| STRUCT | ✔ | ✔ | ✔ | ✔ | +| ARRAY | ✔ | ✔ | ✔ | ✔ | --- - -[Back to index](README.md) From a436afaedb403497bdcb37a02073334d2a6c878e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 10:59:57 +0100 Subject: [PATCH 75/95] update dml documentation --- documentation/sql/README.md | 2 +- documentation/sql/ddl_statements.md | 4 + documentation/sql/dml_statements.md | 351 +++++++++++++++++++++------- 3 files changed, 274 insertions(+), 83 deletions(-) diff --git a/documentation/sql/README.md b/documentation/sql/README.md index 48546750..fea27faa 100644 --- a/documentation/sql/README.md +++ b/documentation/sql/README.md @@ -14,5 +14,5 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: - [Conditional Functions](functions_conditional.md) - [Geo Functions](functions_geo.md) - [Keywords](keywords.md) -- [DML Support](dml_statements.md) - [DDL Support](ddl_statements.md) +- [DML Support](dml_statements.md) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 37af398b..54536131 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -1,3 +1,5 @@ +[Back to index](README.md) + # 📘 **DDL Statements — SQL Gateway for Elasticsearch** --- @@ -563,3 +565,5 @@ Steps: | ARRAY | ✔ | ✔ | ✔ | ✔ | --- + +[Back to index](README.md) diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md index f58e8f7d..5445c8c7 100644 --- a/documentation/sql/dml_statements.md +++ b/documentation/sql/dml_statements.md @@ -1,147 +1,334 @@ -# DML Support [Back to index](README.md) -This document describes the SQL statements supported by the API, focusing on **Data Manipulation Language (DML)**. Each section provides syntax, examples, and notes on behavior. +# 📘 DML Statements — SQL Gateway for Elasticsearch --- -## 📊 Data Manipulation Language (DML) +## Introduction -### INSERT -Insert new rows into a table, either with explicit values or from a `SELECT`. +The SQL Gateway provides a Data Manipulation Language (DML) layer on top of Elasticsearch. +It supports: -**Syntax:** -```sql -INSERT INTO table_name (col1, col2, ...) -VALUES (val1, val2, ...); +- **INSERT** +- **INSERT ... AS SELECT** +- **UPDATE** +- **DELETE** +- **COPY INTO** (bulk ingestion) -INSERT INTO table_name -SELECT ... -``` +The DML engine is: -**Examples:** -```sql -INSERT INTO users (id, name) VALUES (1, 'Alice'); +- **schema-aware** (PK, defaults, scripts, STRUCT, ARRAY) +- **pipeline-aware** (default pipeline + user pipelines) +- **partition-aware** (date-based routing) +- **primary-key-aware** (upsert semantics) +- **version-aware** (ES6 → ES9) -INSERT INTO users SELECT id, name FROM old_users; +Each DML statement returns: + +```scala + case class DmlResult( + inserted: Long = 0L, + updated: Long = 0L, + deleted: Long = 0L, + rejected: Long = 0L + ) extends QueryResult ``` --- -### UPDATE -Update existing rows in a table. +## INSERT + +### Standard Syntax -**Syntax:** ```sql -UPDATE table_name -SET col1 = val1, col2 = val2, ... -[WHERE condition] +INSERT INTO table_name (col1, col2, ...) +VALUES (v1, v2, ...), (v3, v4, ...), ...; ``` -**Example:** +INSERT operations: + +- use the table’s **primary key** to generate `_id` +- pass documents through the **table pipeline** +- support: + - STRUCT + - ARRAY + - DEFAULT values + - SCRIPT AS generated columns + - NOT NULL constraints + +--- + +### Full Example: INSERT INTO with STRUCT + ```sql -UPDATE users SET name = 'Bob', age = 42 WHERE id = 1; +CREATE TABLE IF NOT EXISTS dql_users ( + id INT NOT NULL, + name VARCHAR FIELDS( + raw KEYWORD + ) OPTIONS (fielddata = true), + age INT SCRIPT AS (YEAR(CURRENT_DATE) - YEAR(birthdate)), + birthdate DATE, + profile STRUCT FIELDS( + city VARCHAR OPTIONS (fielddata = true), + followers INT + ) +); + +INSERT INTO dql_users (id, name, birthdate, profile) VALUES + (1, 'Alice', '1994-01-01', {city = "Paris", followers = 100}), + (2, 'Bob', '1984-05-10', {city = "Lyon", followers = 50}), + (3, 'Chloe', '1999-07-20', {city = "Paris", followers = 200}), + (4, 'David', '1974-03-15', {city = "Marseille", followers = 10}); ``` +**Behavior** + +- `profile` → Elasticsearch `object` +- `name.raw` → multi-field +- DEFAULT, SCRIPT, NOT NULL → applied via pipeline +- PK → `_id` generated automatically + --- -### DELETE -Delete rows from a table. +### INSERT INTO ... AS SELECT -**Syntax:** ```sql -DELETE FROM table_name -[WHERE condition] +INSERT INTO orders (order_id, customer_id, order_date, total) +AS SELECT + id AS order_id, + cust AS customer_id, + date AS order_date, + amount AS total +FROM staging_orders; ``` -**Example:** -```sql -DELETE FROM users WHERE age > 30; +**Behavior** + +- The SELECT is executed by the Gateway +- Results are inserted using the Bulk API +- Table pipelines are applied +- PK ensures upsert semantics + +--- + +### INSERT INTO ... AS SELECT — Validation Workflow** + +Before executing an `INSERT ... AS SELECT`, the Gateway performs a full validation pipeline to ensure schema correctness and safe upsert behavior. + +### **1. Index Validation** +Ensures the target index name is valid. +Invalid names return a 400 error. + +### **2. SQL Parsing** +Extracts: +- target columns +- SELECT statement +- ON CONFLICT clause +- DO UPDATE flag +- conflict target columns + +### **3. Load Index Metadata** +Loads the real Elasticsearch schema: +- primary key +- partitioning +- mapping +- settings + +### **3.b Determine Effective Insert Columns** +If the INSERT column list is omitted, the Gateway derives it from the SELECT output. + +### **3.c Validate ON CONFLICT Rules** +- If the index has a primary key: + - conflict target must match the PK exactly + - INSERT must include all PK columns +- If the index has no primary key: + - ON CONFLICT requires an explicit conflict target + - all conflict target columns must be included in INSERT + +### **3.d Validate SELECT Output Columns** +Ensures: +- every INSERT column exists in the SELECT output +- aliases are resolved +- SELECT is valid + +Otherwise, a 400 error is returned. + +### **4. Derive Bulk Options** +Determines: +- `_id` generation (PK or composite PK) +- partitioning suffix +- upsert behavior (`update = DO UPDATE`) + +### **5. Build Document Source** +- For VALUES: convert to JSON array +- For SELECT: scroll the query and convert rows to JSON + +### **6. Bulk Insert** +Uses the Bulk API with: +- PK-based `_id` +- partitioning +- pipeline execution +- upsert if DO UPDATE + +Returns: + +```scala +DmlResult(inserted = N, rejected = M) ``` --- -## 🔄 DML Execution Strategy +## UPDATE + +### Syntax -The SQL DML statements (`INSERT`, `UPDATE`, `DELETE`) are automatically translated into Elasticsearch operations. -The execution path depends on the **number of impacted rows**: +```sql +UPDATE table_name +SET col1 = expr1, col2 = expr2, ... +WHERE condition; +``` -- **Single row impacted** → direct ES operation: - - `INSERT` → `_index` - - `UPDATE` → `_update` - - `DELETE` → `_delete` +**Behavior** -- **Multiple rows impacted** → bulk ingestion: - - All operations are batched and executed via the `_bulk` API. - - Bulk execution is implemented using **Akka Streams**, ensuring efficient back‑pressure handling, parallelism, and resilience for large datasets. +- UPDATE uses **update_by_query** +- Supports: + - nested fields + - STRUCT fields + - ARRAY (via full replacement) + - expressions + - PK-based filtering --- -### 📌 Example Translation +## DELETE + +**Syntax** -**SQL:** ```sql -INSERT INTO users (id, name) VALUES (1, 'Alice'); +DELETE FROM table_name +WHERE condition; ``` -**ES:** -```curl -PUT users/_doc/1 -{ - "id": 1, - "name": "Alice" -} -``` +**Behavior** + +- DELETE uses **delete_by_query** +- Supports: + - PK-based deletion + - nested conditions + - date filters --- -**SQL:** +## COPY INTO (Bulk Ingestion) + +COPY INTO is a **DML operator** that loads documents from external files into an Elasticsearch index. +It uses **only the Bulk API**, not the SQL engine. + +**Syntax** + ```sql -UPDATE users SET name = 'Bob' WHERE id = 1; +COPY INTO table_name +FROM 'path/to/file' +[FILE_FORMAT = 'JSON' | 'JSON_ARRAY' | 'PARQUET' | 'DELTA_LAKE'] +[ON CONFLICT (pk_column) DO UPDATE]; ``` -**ES:** -```curl -POST users/_update/1 -{ - "doc": { "name": "Bob" } -} -``` +**Behavior** ---- +COPY INTO performs: + +1. **Index name validation** +2. **Loading of the real Elasticsearch schema** + - mapping + - primary key + - partitioning +3. **Primary key extraction** + - PK → `_id` + - composite PK → concatenated `_id` +4. **Partitioning extraction** + - suffix index name based on date +5. **Bulk ingestion via `bulkFromFile`** +6. **Pipeline execution** +7. **Return of `DmlResult`** + - `inserted` = successfully indexed docs + - `rejected` = Bulk failures + +There are **no** strategies like `insertAfter`, `updateAfter`, or `deleteAfter`. +COPY INTO **does not** perform SQL-level operations — everything is Bulk. + +**Full Example** + +- **Table Definition** -**SQL:** ```sql -DELETE FROM users WHERE id = 1; +CREATE TABLE IF NOT EXISTS copy_into_test ( + uuid KEYWORD NOT NULL, + name VARCHAR, + birthDate DATE, + childrenCount INT, + PRIMARY KEY (uuid) +); ``` -**ES:** -```curl -DELETE users/_doc/1 +- **Data File** (`example_data.json`) + +``` +{"uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0} +{"uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0} +{"uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 2} ``` ---- +- **COPY INTO Statement** -**SQL (multi‑row):** ```sql -INSERT INTO users (id, name) VALUES (1, 'Alice'), (2, 'Bob'); +COPY INTO copy_into_test +FROM 's3://my-bucket/path/to/example_data.json' +FILE_FORMAT = 'JSON' +ON CONFLICT (uuid) DO UPDATE; ``` -**ES (bulk via Akka Streams):** -```curl -POST _bulk -{ "index": { "_index": "users", "_id": "1" } } -{ "id": 1, "name": "Alice" } -{ "index": { "_index": "users", "_id": "2" } } -{ "id": 2, "name": "Bob" } +**Behavior** + +- PK = `uuid` → `_id = uuid` +- ON CONFLICT DO UPDATE → Bulk upsert +- Table pipeline is applied +- Partitioning is applied if defined +- Returns `DmlResult(inserted = N, rejected = 0)` + +--- + +## DML Lifecycle Example + +```sql +INSERT INTO dml_chain (id, value) VALUES + (1, 10), + (2, 20), + (3, 30); + +UPDATE dml_chain +SET value = 50 +WHERE id IN (1, 3); + +DELETE FROM dml_chain +WHERE value > 40; + +SELECT * FROM dml_chain ORDER BY id ASC; ``` --- -### ✅ Notes -- The API automatically chooses between **single‑doc operations** and **bulk operations**. -- Bulk execution is stream‑based, scalable, and fault‑tolerant thanks to Akka Streams. -- This strategy ensures optimal performance while keeping SQL semantics transparent for the user. +## Version Compatibility + +| Feature | ES6 | ES7 | ES8 | ES9 | +|------------------|------|------|------|------| +| INSERT | ✔ | ✔ | ✔ | ✔ | +| INSERT AS SELECT | ✔ | ✔ | ✔ | ✔ | +| UPDATE | ✔ | ✔ | ✔ | ✔ | +| DELETE | ✔ | ✔ | ✔ | ✔ | +| COPY INTO | ✔ | ✔ | ✔ | ✔ | +| JSON_ARRAY | ✔ | ✔ | ✔ | ✔ | +| PARQUET | ✔ | ✔ | ✔ | ✔ | +| DELTA_LAKE | ✔ | ✔ | ✔ | ✔ | --- From cb56562f3692a79e1348765e78b284413470bf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 15:25:55 +0100 Subject: [PATCH 76/95] update dql documentation --- documentation/sql/dql_statements.md | 677 ++++++++++++++++-- .../client/GatewayApiIntegrationSpec.scala | 2 +- 2 files changed, 626 insertions(+), 53 deletions(-) diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 44c01070..820cdf80 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -1,112 +1,685 @@ [Back to index](README.md) -# Query Structure +# 📘 DQL Statements — SQL Gateway for Elasticsearch -**Navigation:** [Operators](operators.md) · [Functions — Aggregate](functions_aggregate.md) · [Keywords](keywords.md) +## Introduction -This page documents the SQL clauses supported by the engine and how they map to Elasticsearch. +The SQL Gateway provides a Data Query Language (DQL) on top of Elasticsearch, centered around the `SELECT` statement. +It offers a familiar SQL experience while translating queries into Elasticsearch search, aggregations, and scroll APIs. + +DQL supports: + +- `SELECT` with expressions, aliases, nested fields, STRUCT and ARRAY +- `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, `LIMIT`, `OFFSET` +- `UNION ALL` +- `JOIN UNNEST` on `ARRAY` +- aggregations, parent-level aggregations on nested arrays +- window functions with `OVER` +- rich function support (numeric, string, date/time, geo, conditional, type conversion) --- ## SELECT -**Description:** -Projection of fields, expressions and computed values. -**Behavior:** -- `_source` includes for plain fields. -- Computed expressions are translated into `script_fields` (Painless) when push-down is not otherwise possible. -- Aggregates are translated to ES aggregations and the top-level `size` is often set to `0` for aggregation-only queries. +#### Basic syntax + +```sql +SELECT [DISTINCT] expr1, expr2, ... +FROM table_name [alias] +[WHERE condition] +[GROUP BY expr1, expr2, ...] +[HAVING condition] +[ORDER BY expr1 [ASC|DESC], ...] +[LIMIT n] +[OFFSET m]; +``` + +#### Nested fields and aliases -**Example:** ```sql -SELECT department, COUNT(*) AS cnt -FROM emp -GROUP BY department; +SELECT id, + name AS full_name, + profile.city AS city, + profile.followers AS followers +FROM dql_users +ORDER BY id ASC; ``` +- `profile` is a `STRUCT` column. +- `profile.city` and `profile.followers` access nested fields. +- Aliases (`AS full_name`, `AS city`) are returned as column names. + --- -## FROM -**Description:** -Source index (one or more). Translates to the Elasticsearch index parameter. +## WHERE + +The `WHERE` clause supports: + +- comparison operators: `=`, `!=`, `<`, `<=`, `>`, `>=` +- logical operators: `AND`, `OR`, `NOT` +- `IN`, `NOT IN` +- `BETWEEN` +- `IS NULL`, `IS NOT NULL` +- `LIKE`, `RLIKE` (regex) +- conditions on nested fields (`profile.city`, `profile.followers`) + +**Example** -**Example:** ```sql -SELECT * FROM employees; +SELECT id, name, age +FROM dql_users +WHERE (age > 20 AND profile.followers >= 100) + OR (profile.city = 'Lyon' AND age < 50) +ORDER BY age DESC; +``` + +Another example with multiple operators: + +```sql +SELECT id, + age + 10 AS age_plus_10, + name +FROM dql_users +WHERE age BETWEEN 20 AND 50 + AND name IN ('Alice', 'Bob', 'Chloe') + AND name IS NOT NULL + AND (name LIKE 'A%' OR name RLIKE '.*o.*'); ``` --- -## UNNEST -**Description:** -Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and inner hits where necessary. +## ORDER BY + +`ORDER BY` supports: + +- single or multiple fields +- `ASC` / `DESC` +- expressions and nested fields + +**Example** -**Example:** ```sql -SELECT id, phone -FROM customers -JOIN UNNEST(customers.phones) AS phone; +SELECT id, name, age +FROM dql_users +ORDER BY age DESC, name ASC +LIMIT 2 OFFSET 1; ``` --- -## WHERE -**Description:** -Row-level predicates. Mapped to `bool` queries; complex expressions become `script` queries (Painless). +## LIMIT / OFFSET + +- `LIMIT n` restricts the number of returned rows. +- `OFFSET m` skips the first `m` rows. +- Translated to Elasticsearch `from` + `size`. + +Example: + +```sql +SELECT id, name, age +FROM dql_users +ORDER BY age DESC +LIMIT 10 OFFSET 20; +``` + +--- + +`UNION ALL` combines the results of multiple `SELECT` queries **without removing duplicates**. + +All SELECT statements in a UNION ALL must be **strictly compatible**: + +- **same number of columns** +- **same column names** (after alias resolution) +- **same or implicitly compatible types** + +If these conditions are not met, the Gateway raises a validation error before executing the query. + +**Example** + +```sql +SELECT id, name FROM dql_users WHERE age > 30 +UNION ALL +SELECT id, name FROM dql_users WHERE age <= 30; +``` + +### Execution model + +The SQL Gateway executes `UNION ALL` using **Elasticsearch Multi‑Search (`_msearch`)**: + +1. Each SELECT query is translated into an independent ES search request. +2. All requests are sent in a single `_msearch` call. +3. The Gateway concatenates the results **in order**, without deduplication. +4. ORDER BY, LIMIT, OFFSET apply **per SELECT**, not globally (unless wrapped in a subquery, which is not supported). + +### Notes + +- `UNION ALL` does **not** sort or deduplicate results. +- Column names in the final output are taken from the **first SELECT**. +- All subsequent SELECTs must produce columns with the **same names**. +- Type mismatches should result in a validation error before execution. (⚠️ not implemented yet) + +--- + +## JOIN UNNEST + +The Gateway supports a specific form of join: `JOIN UNNEST` on `ARRAY` columns. + +### Table definition + +```sql +CREATE TABLE IF NOT EXISTS dql_orders ( + id INT NOT NULL, + customer_id INT, + items ARRAY FIELDS( + product VARCHAR OPTIONS (fielddata = true), + quantity INT, + price DOUBLE + ) OPTIONS (include_in_parent = false) +); +``` + +### Query with JOIN UNNEST and window function + +```sql +SELECT + o.id, + items.product, + items.quantity, + SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price +FROM dql_orders o +JOIN UNNEST(o.items) AS items +WHERE items.quantity >= 1 +ORDER BY o.id ASC; +``` + +- `JOIN UNNEST(o.items)` flattens the `items` array. +- Each element should become a row with `items.product`, `items.quantity`, `items.price`. (⚠️ not implemented yet) +- Window function computes per-order totals. + +--- + +## Aggregations + +Supported aggregate functions include: + +- `COUNT(*)`, `COUNT(expr)` +- `SUM(expr)` +- `AVG(expr)` +- `MIN(expr)` +- `MAX(expr)` + +### GROUP BY and HAVING + +```sql +SELECT profile.city AS city, + COUNT(*) AS cnt, + AVG(age) AS avg_age +FROM dql_users +GROUP BY profile.city +HAVING COUNT(*) >= 1 +ORDER BY COUNT(*) DESC; +``` + +- `GROUP BY` supports nested fields (`profile.city`). +- `HAVING` filters groups based on aggregate conditions. +- Translated to Elasticsearch aggregations. + +--- + +## Parent-Level Aggregations on Nested Arrays + +The SQL Gateway supports computing aggregations **over nested arrays** (e.g., `ARRAY`) **without exploding them into multiple rows**. + +This pattern: + +- reads the nested array (`JOIN UNNEST`) +- computes aggregations per parent document (`PARTITION BY parent_id`) +- **returns one row per parent** +- **preserves the original nested array** +- **adds the aggregated value as a top-level field** + +**Example** + +```sql +SELECT + o.id, + o.items, + SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price +FROM dql_orders o +JOIN UNNEST(o.items) AS items +WHERE items.quantity >= 1 +ORDER BY o.id ASC; +``` + +**Result** + +```json +[ + { + "id": 1, + "items": [ + {"product": "A", "quantity": 2, "price": 10.0}, + {"product": "B", "quantity": 1, "price": 20.0} + ], + "total_price": 40.0 + }, + { + "id": 2, + "items": [ + {"product": "C", "quantity": 3, "price": 5.0} + ], + "total_price": 15.0 + } +] +``` + +### Notes + +- This is **not** a standard SQL window function (which would return one row per item). +- This is **not** an Elasticsearch nested aggregation (which would not return the items). +- This is a **hybrid parent-level aggregation**, unique to the SQL Gateway. + +--- + +## Window Functions + +Window functions operate over a logical window of rows defined by `OVER (PARTITION BY ... ORDER BY ...)`. + +Supported window functions include: + +- `SUM(expr) OVER (...)` +- `COUNT(expr) OVER (...)` +- `FIRST_VALUE(expr) OVER (...)` +- `LAST_VALUE(expr) OVER (...)` +- `ARRAY_AGG(expr) OVER (...)` + +#### Basic window example + +```sql +SELECT + product, + customer, + amount, + SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product +FROM dql_sales +ORDER BY product, ts; +``` + +#### FIRST_VALUE / LAST_VALUE / ARRAY_AGG + +```sql +SELECT + product, + customer, + amount, + SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product, + FIRST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS first_amount, + LAST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS last_amount, + ARRAY_AGG(amount) OVER (PARTITION BY product ORDER BY ts ASC LIMIT 10) AS amounts_array +FROM dql_sales +ORDER BY product, ts; +``` + +Notes: + +- `PARTITION BY` defines the grouping key. +- `ORDER BY` inside `OVER` defines the window ordering. +- `LIMIT` inside `ARRAY_AGG` restricts the collected values. +- Frame clauses (`ROWS BETWEEN ...`) are not exposed; the engine uses a default frame per function semantics. + +--- + +## Functions + +The SQL Gateway provides a rich set of SQL functions covering: + +- numeric and trigonometric operations +- string manipulation +- date and time extraction, arithmetic, formatting and parsing +- geospatial functions +- conditional expressions +- type conversion + +All functions operate on Elasticsearch documents and are evaluated by the SQL engine. + +--- + +#### Numeric & Trigonometric + +##### **Arithmetic:** + +| Function | Description | +|---------------|----------------------| +| `ABS(x)` | Absolute value | +| `CEIL(x)` | Round up | +| `FLOOR(x)` | Round down | +| `ROUND(x, n)` | Round to n decimals | +| `SQRT(x)` | Square root | +| `POW(x, y)` | Power | +| `EXP(x)` | Exponential | +| `LOG(x)` | Natural logarithm | +| `LOG10(x)` | Base-10 logarithm | +| `SIGN(x)` | Sign of x (−1, 0, 1) | + + +##### **Trigonometric:** + +| Function | Description | +|---------------|----------------------------------| +| `SIN(x)` | Sine | +| `COS(x)` | Cosine | +| `TAN(x)` | Tangent | +| `ASIN(x)` | Arc-sine | +| `ACOS(x)` | Arc-cosine | +| `ATAN(x)` | Arc-tangent | +| `ATAN2(y, x)` | Arc-tangent of y/x with quadrant | +| `PI()` | π constant | +| `RADIANS(x)` | Degrees → radians | +| `DEGREES(x)` | Radians → degrees | + +**Example** -**Example:** ```sql -SELECT * FROM emp WHERE salary > 50000 AND department = 'IT'; +SELECT id, + ABS(age) AS abs_age, + SQRT(age) AS sqrt_age, + POW(age, 2) AS pow_age, + LOG(age) AS log_age, + SIN(age) AS sin_age, + ATAN2(age, 10) AS atan2_val +FROM dql_users; ``` --- -## GROUP BY -**Description:** -Aggregation buckets. Mapped to `terms`/`date_histogram` and nested sub-aggregations. -Non-aggregated selected fields are disallowed unless included in the `GROUP BY` (standard SQL semantics). +#### String + +##### **Manipulation:** + + +| Function | Description | +|------------------------------|-------------------------------| +| `CONCAT(a, b, ...)` | Concatenate strings | +| `SUBSTRING(str, start, len)` | Extract substring | +| `LOWER(str)` | Lowercase | +| `UPPER(str)` | Uppercase | +| `TRIM(str)` | Trim both sides | +| `LTRIM(str)` | Trim left | +| `RTRIM(str)` | Trim right | +| `LENGTH(str)` | String length | +| `REPLACE(str, from, to)` | Replace substring | +| `LEFT(str, n)` | Left n chars | +| `RIGHT(str, n)` | Right n chars | +| `REVERSE(str)` | Reverse string | +| `POSITION(substr IN str)` | 1-based position of substring | + +##### **Pattern matching:** + +| Function | Description | +|------------------------------|-----------------------------------------------------| +| `REGEXP_LIKE(str, pattern)` | True if regex matches | +| `MATCH(str) AGAINST (query)` | Full-text match (backed by ES query_string / match) | **Example:** + ```sql -SELECT department, AVG(salary) AS avg_salary -FROM emp -GROUP BY department; +SELECT id, + CONCAT(name.raw, '_suffix') AS name_concat, + SUBSTRING(name.raw, 1, 2) AS name_sub, + LOWER(name.raw) AS name_lower, + LTRIM(name.raw) AS name_ltrim, + POSITION('o' IN name.raw) AS pos_o, + REGEXP_LIKE(name.raw, '.*o.*') AS has_o +FROM dql_users +ORDER BY id ASC; ``` --- -## HAVING -**Description:** -Filter groups using aggregate expressions. Implemented with pipeline aggregations and `bucket_selector` where possible, or client-side filtering if required. +#### Date & Time + +##### **Current time:** + +| Function | Description | +|-------------------------------------------|-----------------------------| +| `CURRENT_DATE` | Current date (UTC) | +| `TODAY()` | Alias for CURRENT_DATE | +| `CURRENT_TIMESTAMP` \| `CURRENT_DATETIME` | Current timestamp (UTC) | +| `NOW()` | Alias for CURRENT_TIMESTAMP | +| `CURRENT_TIME` | Current time (UTC) | + +##### **Extraction:** + +| Function | Description | +|-------------------|--------------| +| `YEAR(date)` | Year | +| `MONTH(date)` | Month | +| `DAY(date)` | Day of month | +| `WEEKDAY(date)` | Day of week | +| `YEARDAY(date)` | Day of year | +| `HOUR(ts)` | Hour | +| `MINUTE(ts)` | Minute | +| `SECOND(ts)` | Second | +| `MILLISECOND(ts)` | Millisecond | +| `MICROSECOND(ts)` | Microsecond | +| `NANOSECOND(ts)` | Nanosecond | + +##### **EXTRACT:** + +```sql +EXTRACT(unit FROM date_or_timestamp) +``` + +Supported units include: `YEAR`, `MONTH`, `DAY`, `HOUR`, `MINUTE`, `SECOND`, etc. + +**Example:** + +```sql +SELECT id, + EXTRACT(YEAR FROM birthdate) AS year_b, + EXTRACT(MONTH FROM birthdate) AS month_b +FROM dql_users; +``` + +##### **Arithmetic:** + +| Function | Description | +|-------------------------------------|----------------------------------| +| `DATE_ADD(date, INTERVAL n unit)` | Add interval | +| `DATE_SUB(date, INTERVAL n unit)` | Subtract interval | +| `DATETIME_ADD(ts, INTERVAL n unit)` | Add interval to timestamp | +| `DATETIME_SUB(ts, INTERVAL n unit)` | Subtract interval from timestamp | +| `DATE_DIFF(date1, date2, unit)` | Difference in units | +| `DATE_TRUNC(date, unit)` | Truncate to unit | + +##### **Formatting & parsing:** + +| Function | Description | +|--------------------------------|-----------------------------| +| `DATE_FORMAT(ts, pattern)` | Format date as string | +| `DATE_PARSE(str, pattern)` | Parse string into date | +| `DATETIME_FORMAT(ts, pattern)` | Format timestamp as string | +| `DATETIME_PARSE(str, pattern)` | Parse string into timestamp | + +**Supported MySQL-style Date/Time Patterns** + +##### **Special functions:** + +| Function | Description | +|------------------------|-------------------------------| +| `LAST_DAY(date)` | Last day of month | +| `EPOCHDAY(date)` | Days since epoch (1970-01-01) | +| `OFFSET_SECONDS(date)` | Epoch seconds | **Example:** + ```sql -SELECT department, COUNT(*) AS cnt -FROM emp -GROUP BY department -HAVING COUNT(*) > 10; +SELECT id, + YEAR(CURRENT_DATE) AS current_year, + MONTH(CURRENT_DATE) AS current_month, + DAY(CURRENT_DATE) AS current_day, + YEAR(birthdate) AS year_b, + DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years, + DATE_TRUNC(birthdate, MONTH) AS trunc_month, + FORMAT_DATETIME(birthdate, '%Y-%m-%d') AS birth_str +FROM dql_users; ``` --- -## ORDER BY -**Description:** -Sorting of final rows or ordering used inside window/aggregations (pushed to `sort` or `top_hits`). +#### Geospatial + +##### **POINT:** + +```sql +POINT(longitude, latitude) +``` + +##### **ST_DISTANCE:** + +```sql +ST_DISTANCE(location, POINT(2.3522, 48.8566)) +``` + +Example: + +```sql +CREATE TABLE IF NOT EXISTS dql_geo ( + id INT NOT NULL, + location GEO_POINT, + PRIMARY KEY (id) +); + +SELECT id, + ST_DISTANCE(location, POINT(2.3522, 48.8566)) AS dist_paris +FROM dql_geo; +``` + +--- + +#### Conditional + +##### CASE WHEN + +```sql +CASE + WHEN condition THEN value + [WHEN condition2 THEN value2] + [ELSE default] +END +``` + +##### COALESCE + +```sql +COALESCE(a, b, c) +``` + +Returns the first non-null value. + +##### NULLIF + +```sql +NULLIF(a, b) +``` + +Returns NULL if `a = b`, otherwise `a`. **Example:** + ```sql -SELECT name, salary FROM emp ORDER BY salary DESC; +SELECT id, + CASE + WHEN age >= 50 THEN 'senior' + WHEN age >= 30 THEN 'adult' + ELSE 'young' + END AS age_group, + COALESCE(name, 'unknown') AS safe_name +FROM dql_users; ``` --- -## LIMIT / OFFSET -**Description:** -Limit and paging. For pure aggregations, `size` is typically set to 0 and `limit` applies to aggregations or outer rows. +#### Type Conversion + +##### CAST + +```sql +CAST(value AS TYPE) +``` + +##### TRY_CAST + +Returns NULL instead of failing on invalid conversion. + +```sql +TRY_CAST('123' AS INT) +``` + +##### SAFE_CAST + +Alias for TRY_CAST. + +##### PostgreSQL-style operator + +```sql +value::TYPE +``` **Example:** + ```sql -SELECT * FROM emp ORDER BY hire_date DESC LIMIT 10 OFFSET 20; +SELECT id, + age::BIGINT AS age_bigint, + CAST(age AS DOUBLE) AS age_double, + TRY_CAST('123' AS INT) AS try_cast_ok, + SAFE_CAST('abc' AS INT) AS safe_cast_null +FROM dql_users; ``` +--- + +## Scroll & Pagination + +For large result sets, the Gateway uses Elasticsearch scroll or search-after mechanisms ([Scroll Search](../client/scroll.md)). + +- `ORDER BY` is strongly recommended for deterministic pagination. +- `LIMIT`/`OFFSET` are translated to `from`/`size` when feasible; deep pagination may rely on scroll. + +--- + +## Version Compatibility + +| Feature | ES6 | ES7 | ES8 | ES9 | +|--------------------------------|-----|-----|-----|-----| +| Basic SELECT | ✔ | ✔ | ✔ | ✔ | +| Nested fields | ✔ | ✔ | ✔ | ✔ | +| UNION ALL | ✔ | ✔ | ✔ | ✔ | +| JOIN UNNEST | ✔ | ✔ | ✔ | ✔ | +| Aggregations | ✔ | ✔ | ✔ | ✔ | +| Parent-level nested array aggs | ✔ | ✔ | ✔ | ✔ | +| Window functions | ✔ | ✔ | ✔ | ✔ | +| Geospatial functions | ✔ | ✔ | ✔ | ✔ | +| Date/time functions | ✔ | ✔ | ✔ | ✔ | +| String / math functions | ✔ | ✔ | ✔ | ✔ | + +--- + +## Limitations + +Even though the DQL engine is powerful, some SQL features are not (yet) supported: + +- No traditional SQL joins (only `JOIN UNNEST` on `ARRAY`) +- No correlated subqueries +- No arbitrary subqueries in `SELECT` or `WHERE` (except `INSERT ... AS SELECT` in DML) +- No `GROUPING SETS`, `CUBE`, `ROLLUP` +- No `DISTINCT ON` +- No explicit window frame clauses (`ROWS BETWEEN ...`) + +These constraints keep the translation to Elasticsearch efficient and predictable. + +--- + [Back to index](README.md) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index b6fda660..5f89350b 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -1378,7 +1378,7 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala } // =========================================================================== - // 5. PIPELINES — CREATE / ALTER / DROP + // 5. PIPELINES — CREATE / ALTER / DROP / SHOW // =========================================================================== behavior of "PIPELINE statements" From 8e3b8cc37f4f853f6c74e81eb5818a6e9db3e054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 15:57:20 +0100 Subject: [PATCH 77/95] update dql documentation --- documentation/sql/dql_statements.md | 37 +++++++++++++++++++---------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 820cdf80..71d84a7c 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -90,11 +90,12 @@ WHERE age BETWEEN 20 AND 50 ## ORDER BY -`ORDER BY` supports: +`ORDER BY` sorts the result set by one or more expressions. -- single or multiple fields -- `ASC` / `DESC` -- expressions and nested fields +- Supports multiple sort keys +- Supports `ASC` and `DESC` +- Supports expressions and nested fields (e.g., `profile.city`) +- When used inside a window function (`OVER`), `ORDER BY` defines the logical ordering of the window **Example** @@ -192,9 +193,17 @@ WHERE items.quantity >= 1 ORDER BY o.id ASC; ``` -- `JOIN UNNEST(o.items)` flattens the `items` array. -- Each element should become a row with `items.product`, `items.quantity`, `items.price`. (⚠️ not implemented yet) -- Window function computes per-order totals. +`JOIN UNNEST` is intended to behave like a standard SQL UNNEST operation, where each element of an `ARRAY` becomes a logical row that can participate in expressions, filters, and window functions. + +⚠️ **Current status:** +The SQL Gateway already supports reading and filtering nested array elements through `JOIN UNNEST`, and supports parent-level aggregations using window functions. +However, **full row-level expansion (one output row per array element)** is **not implemented yet**. + +This means: +- expressions such as `items.price` and `items.quantity` are fully usable +- window functions over `PARTITION BY parent_id` work +- parent-level aggregations can be computed +- but the final output still returns **one row per parent**, not one row per item --- @@ -440,7 +449,7 @@ ORDER BY id ASC; #### Date & Time -##### **Current time:** +##### **Current :** | Function | Description | |-------------------------------------------|-----------------------------| @@ -523,7 +532,7 @@ SELECT id, YEAR(birthdate) AS year_b, DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years, DATE_TRUNC(birthdate, MONTH) AS trunc_month, - FORMAT_DATETIME(birthdate, '%Y-%m-%d') AS birth_str + DATETIME_FORMAT(birthdate, '%Y-%m-%d') AS birth_str FROM dql_users; ``` @@ -643,10 +652,14 @@ FROM dql_users; ## Scroll & Pagination -For large result sets, the Gateway uses Elasticsearch scroll or search-after mechanisms ([Scroll Search](../client/scroll.md)). +For large result sets, the Gateway uses Elasticsearch scroll or search-after mechanisms depending on backend capabilities ([Scroll Search](../client/scroll.md)). + +Notes: -- `ORDER BY` is strongly recommended for deterministic pagination. -- `LIMIT`/`OFFSET` are translated to `from`/`size` when feasible; deep pagination may rely on scroll. +- `LIMIT` and `OFFSET` are applied by the SQL engine after retrieving documents from Elasticsearch +- Deep pagination may require scroll +- When using `search_after`, an explicit `ORDER BY` clause is required for deterministic pagination +- Without `ORDER BY`, result ordering is not guaranteed --- From c9499823edb168adcb1d40b5c00867a503f1141f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 16:41:37 +0100 Subject: [PATCH 78/95] update dml documentation --- documentation/sql/dml_statements.md | 79 +++++++++++++++++++---------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md index 5445c8c7..c2350264 100644 --- a/documentation/sql/dml_statements.md +++ b/documentation/sql/dml_statements.md @@ -6,14 +6,13 @@ ## Introduction -The SQL Gateway provides a Data Manipulation Language (DML) layer on top of Elasticsearch. -It supports: +The SQL Gateway supports the following Data Manipulation Language (DML) operations: -- **INSERT** -- **INSERT ... AS SELECT** -- **UPDATE** -- **DELETE** -- **COPY INTO** (bulk ingestion) +- **INSERT INTO ... VALUES** +- **INSERT INTO ... AS SELECT ... [ON CONFLICT ...]** +- **UPDATE ... SET ... [WHERE]** +- **DELETE FROM ... [WHERE]** +- **COPY INTO ...** (bulk ingestion) The DML engine is: @@ -23,6 +22,16 @@ The DML engine is: - **primary-key-aware** (upsert semantics) - **version-aware** (ES6 → ES9) +All DML operations are translated into Elasticsearch write APIs: + +- `INSERT` → index API (bulk) +- `INSERT ... AS SELECT` → search + bulk index +- `UPDATE` → update-by-query +- `DELETE` → delete-by-query +- `COPY INTO` → bulk from file + +`DELETE` without a `WHERE` clause is allowed and behaves like a **TRUNCATE TABLE**, removing all documents from the index. + Each DML statement returns: ```scala @@ -38,7 +47,7 @@ Each DML statement returns: ## INSERT -### Standard Syntax +### INSERT INTO ... VALUES ```sql INSERT INTO table_name (col1, col2, ...) @@ -90,7 +99,16 @@ INSERT INTO dql_users (id, name, birthdate, profile) VALUES --- -### INSERT INTO ... AS SELECT +### Notes: + +- Fields not provided in the `VALUES` clause are simply omitted from the document (Elasticsearch does not store `NULL`). +- Unknown fields are rejected if the index mapping is strict. +- All values are validated against the Elasticsearch mapping (type, format, date patterns, etc.). +- Nested structures (`STRUCT`, `ARRAY`) must match the expected JSON shape. + +--- + +### INSERT INTO ... AS SELECT ... [ON CONFLICT ...] ```sql INSERT INTO orders (order_id, customer_id, order_date, total) @@ -111,7 +129,7 @@ FROM staging_orders; --- -### INSERT INTO ... AS SELECT — Validation Workflow** +### INSERT INTO ... AS SELECT ... [ON CONFLICT ...] — Validation Workflow** Before executing an `INSERT ... AS SELECT`, the Gateway performs a full validation pipeline to ensure schema correctness and safe upsert behavior. @@ -146,7 +164,7 @@ If the INSERT column list is omitted, the Gateway derives it from the SELECT out - all conflict target columns must be included in INSERT ### **3.d Validate SELECT Output Columns** -Ensures: +Ensures : - every INSERT column exists in the SELECT output - aliases are resolved - SELECT is valid @@ -178,7 +196,17 @@ DmlResult(inserted = N, rejected = M) --- -## UPDATE +### Notes + +- All columns listed in the `INSERT` target must be present in the `SELECT` list. +- The order of columns does not matter; matching is done by column name. +- Extra columns in the `SELECT` that are not part of the target table are ignored. +- Nested structures (`STRUCT`, `ARRAY`) are supported as long as the selected expressions produce valid JSON structures for the target fields. +- No type validation is performed at the SQL layer; type errors are reported by Elasticsearch at indexing time. + +--- + +## UPDATE ... SET ... [WHERE] ### Syntax @@ -190,17 +218,16 @@ WHERE condition; **Behavior** -- UPDATE uses **update_by_query** -- Supports: - - nested fields - - STRUCT fields - - ARRAY (via full replacement) - - expressions - - PK-based filtering +- `UPDATE` is implemented using Elasticsearch `update-by-query`. +- Only top-level scalar fields or entire nested objects can be replaced. +- Updating a specific element inside an `ARRAY` is not supported. +- `SET field = NULL` removes the field from the document (Elasticsearch does not store SQL NULL). +- Expressions such as `SET x = x + 1` are not supported (no script-based incremental updates). +- Fields not present in the mapping cannot be added unless dynamic mapping is enabled. --- -## DELETE +## DELETE FROM ... [WHERE] **Syntax** @@ -211,15 +238,15 @@ WHERE condition; **Behavior** -- DELETE uses **delete_by_query** -- Supports: - - PK-based deletion - - nested conditions - - date filters +- `DELETE` is implemented using Elasticsearch `delete-by-query`. +- `WHERE` is optional. + - If provided, only matching documents are deleted. + - If omitted, **all documents in the index are deleted** (equivalent to `TRUNCATE TABLE`). +- Deleting a nested field or an element inside an array is not supported; only whole documents can be removed. --- -## COPY INTO (Bulk Ingestion) +## COPY INTO ... COPY INTO is a **DML operator** that loads documents from external files into an Elasticsearch index. It uses **only the Bulk API**, not the SQL engine. From f718d21092c4b5bbd9587feb3778a9e2af7bf883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 17:06:31 +0100 Subject: [PATCH 79/95] update ddl documentation --- documentation/sql/ddl_statements.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 54536131..d0ba8481 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -399,6 +399,24 @@ PARTITIONED BY (birthdate MONTH); - `SET|ADD SETTING (key = value)` - `DROP SETTING key` +### Type Changes and Safety + +When applying `ALTER COLUMN column_name SET DATA TYPE new_type`, the SQL Gateway computes a structural diff between the current schema and the target schema. + +Type changes fall into two categories: + +- **Convertible types** (`SQLTypeUtils.canConvert(from, to) = true`) + The change is allowed but requires a full **reindex** of the underlying data. + The Gateway automatically performs the reindex operation and swaps aliases when complete. + These changes are classified as `UnsafeReindex`. + +- **Incompatible types** (`SQLTypeUtils.canConvert(from, to) = false`) + The change is **not allowed** and the `ALTER TABLE` statement fails. + These changes are classified as `Impossible`. + +This is the only case where an `ALTER TABLE` operation can be rejected for safety reasons. +All other ALTER operations (adding/dropping columns, renaming, modifying options, modifying nested fields, etc.) are allowed. + --- ## DROP TABLE From 69b62578a0b04925d524f579191a74254d7561a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 17:08:38 +0100 Subject: [PATCH 80/95] minor update --- .../main/scala/app/softnetwork/elastic/sql/query/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index a176a9f9..d4e2d136 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -425,7 +425,7 @@ package object query { case class Delete(table: Table, where: Option[Where]) extends DmlStatement { override def sql: String = - s"DELETE FROM ${table.name}${where.map(w => s" ${w.sql}").getOrElse("")}" + s"DELETE FROM ${table.name}${asString(where)}" } sealed trait FileFormat extends Token { From 661b1cd6c2880821ae6bd16709305dda8ec321fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 18:22:57 +0100 Subject: [PATCH 81/95] add documentation for Gateway api - implements SHOW CREATE TABLE / PIPELINE --- .../elastic/client/GatewayApi.scala | 40 +- .../elastic/client/result/package.scala | 7 +- documentation/client/README.md | 1 + documentation/client/gateway.md | 381 ++++++++++++++++++ documentation/sql/README.md | 1 + .../elastic/sql/parser/Parser.scala | 12 + .../elastic/sql/query/package.scala | 8 + .../client/GatewayApiIntegrationSpec.scala | 32 +- 8 files changed, 469 insertions(+), 13 deletions(-) create mode 100644 documentation/client/gateway.md diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index a3b3d1c5..69b8f80c 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -24,12 +24,13 @@ import app.softnetwork.elastic.client.result.{ ElasticFailure, ElasticResult, ElasticSuccess, - QueryPipeline, + PipelineResult, QueryResult, QueryRows, QueryStream, QueryStructured, - QueryTable + SQLResult, + TableResult } import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query.{ @@ -48,6 +49,8 @@ import app.softnetwork.elastic.sql.query.{ MultiSearch, PipelineStatement, SelectStatement, + ShowCreatePipeline, + ShowCreateTable, ShowPipeline, ShowTable, SingleSearch, @@ -245,7 +248,22 @@ class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[Pip api.loadPipeline(show.name) match { case ElasticSuccess(pipeline) => logger.info(s"✅ Retrieved pipeline ${show.name}.") - Future.successful(ElasticResult.success(QueryPipeline(pipeline))) + Future.successful(ElasticResult.success(PipelineResult(pipeline))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case showCreate: ShowCreatePipeline => + // handle SHOW CREATE PIPELINE statement + api.loadPipeline(showCreate.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"✅ Retrieved pipeline ${showCreate.name}.") + Future.successful( + ElasticResult.success(SQLResult(pipeline.sql)) + ) case ElasticFailure(elasticError) => Future.successful( ElasticFailure( @@ -303,7 +321,21 @@ class TableExecutor( api.loadSchema(show.table) match { case ElasticSuccess(schema) => logger.info(s"✅ Retrieved schema for index ${show.table}.") - Future.successful(ElasticResult.success(QueryTable(schema))) + Future.successful(ElasticResult.success(TableResult(schema))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case showCreate: ShowCreateTable => + api.loadSchema(showCreate.table) match { + case ElasticSuccess(schema) => + logger.info(s"✅ Retrieved schema for index ${showCreate.table}.") + Future.successful( + ElasticResult.success(SQLResult(schema.sql)) + ) case ElasticFailure(elasticError) => Future.successful( ElasticFailure( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 61f41e3b..3582acfe 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -418,7 +418,10 @@ package object result { // -------------------- case class DdlResult(success: Boolean) extends QueryResult - case class QueryTable(table: Table) extends QueryResult + case class TableResult(table: Table) extends QueryResult + + case class PipelineResult(pipeline: IngestPipeline) extends QueryResult + + case class SQLResult(sql: String) extends QueryResult - case class QueryPipeline(pipeline: IngestPipeline) extends QueryResult } diff --git a/documentation/client/README.md b/documentation/client/README.md index 5d4893c0..da497288 100644 --- a/documentation/client/README.md +++ b/documentation/client/README.md @@ -19,3 +19,4 @@ Welcome to the Client Engine Documentation. Navigate through the sections below: - [Scroll Search](scroll.md) - [Aggregations](aggregations.md) - [Template Management](templates.md) +- [SQL Gateway Usage](gateway.md) diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md new file mode 100644 index 00000000..5586e87d --- /dev/null +++ b/documentation/client/gateway.md @@ -0,0 +1,381 @@ +[Back to index](README.md) + +# 📘 GatewayApi + +The `GatewayApi` provides a unified entry point for executing SQL statements (DQL, DML, DDL, and Pipeline operations) against Elasticsearch. +It exposes a single method: + +```scala +def run(sql: String): Future[ElasticResult[QueryResult]] +``` + +which accepts any SQL statement supported by the SQL Gateway. + +This API is designed to offer a familiar SQL interface while leveraging Elasticsearch’s indexing, search, aggregation, and mapping capabilities under the hood. + +--- + +## Table of Contents + +- [Overview](#overview) +- [Statement Hierarchy](#statement-hierarchy) +- [Supported SQL Statements](#supported-sql-statements) +- [Executing SQL](#executing-sql) + - [Single Statement](#single-statement) + - [Multiple Statements](#multiple-statements) +- [QueryResult Types](#queryresult-types) + - [DQL Results](#dql-results) + - [DML Results](#dml-results) + - [DDL Results](#ddl-results) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Notes](#notes) + +--- + +## Overview + +`GatewayApi` is a high‑level SQL interface built on top of the Softnetwork Elasticsearch client. +It routes SQL statements to the appropriate executor: + +| SQL Type | Executor | +|----------------------------------------------------------|---------------------| +| DQL (SELECT) | `DqlExecutor` | +| DML (INSERT / UPDATE / DELETE / COPY INTO) | `DmlExecutor` | +| DDL (CREATE / ALTER / DROP / TRUNCATE / SHOW / DESCRIBE) | `DdlRouterExecutor` | + +The API automatically: + +- normalizes SQL (removes comments, trims whitespace) +- splits multiple statements separated by `;` +- parses SQL into AST nodes +- dispatches to the correct executor +- returns a typed `QueryResult` + +--- + +## Statement Hierarchy + +The SQL Gateway classifies SQL statements into three main categories: + +- **DQL** — SELECT queries +- **DML** — INSERT, UPDATE, DELETE, COPY INTO +- **DDL** — schema and pipeline definitions + +DDL statements are further divided into: + +- **Table statements** (`TableStatement`) + CREATE TABLE, ALTER TABLE, DROP TABLE, TRUNCATE TABLE, DESCRIBE TABLE, SHOW TABLE, SHOW CREATE TABLE + +- **Pipeline statements** (`PipelineStatement`) + CREATE PIPELINE, ALTER PIPELINE, DROP PIPELINE, DESCRIBE PIPELINE, SHOW PIPELINE, SHOW CREATE PIPELINE + +Both kinds of statements extend `DdlStatement`. + +--- + +## Supported SQL Statements + +### DQL + +[DQL Statements](../sql/dql_statements.md) supported by the SQL Gateway include: +- `SELECT … FROM …` +- `JOIN UNNEST` +- `GROUP BY`, `HAVING` +- `ORDER BY`, `LIMIT`, `OFFSET` +- Window functions (`OVER`) +- Parent‑level aggregations on nested arrays + +### DML + +[DML Statements](../sql/dml_statements.md) supported by the SQL Gateway include: +- `INSERT INTO … VALUES` +- `INSERT INTO … AS SELECT` +- `UPDATE … SET …` +- `DELETE FROM …` +- `COPY INTO table FROM file` + +### DDL + +[DDL Statements](../sql/ddl_statements.md) supported by the SQL Gateway include: +- `CREATE TABLE` +- `ALTER TABLE` +- `DROP TABLE` +- `TRUNCATE TABLE` +- `DESCRIBE TABLE` +- `SHOW TABLE` +- `SHOW CREATE TABLE` + +- `CREATE PIPELINE` +- `ALTER PIPELINE` +- `DROP PIPELINE` +- `DESCRIBE PIPELINE` +- `SHOW PIPELINE` +- `SHOW CREATE PIPELINE` + +--- + +## Executing SQL + +### Single Statement + +```scala +gateway.run("SELECT * FROM dql_users") +``` + +### Multiple Statements + +Statements separated by `;` are executed **sequentially**: + +```scala +gateway.run(""" + CREATE TABLE users (...); + INSERT INTO users VALUES (...); + SELECT * FROM users; +""") +``` + +The result of the **last** statement is returned. + +--- + +## QueryResult Types + +The Gateway returns one of the following: + +--- + +### DQL Results + +#### `QueryRows` + +Materialized rows: + +```scala +QueryRows(Seq(Map("id" -> 1, "name" -> "Alice"))) +``` + +#### `QueryStream` + +Streaming rows using scroll: + +```scala +QueryStream(Source[(Map[String, Any], ScrollMetrics)]) +``` + +#### `QueryStructured` + +Raw Elasticsearch response: + +```scala +QueryStructured(ElasticResponse) +``` + +--- + +### DML Results + +#### `DmlResult` + +```scala +DmlResult( + inserted = 10, + updated = 0, + deleted = 0, + rejected = 0 +) +``` + +--- + +### DDL Results + +DDL operations return one of the following: + +- `DdlResult(success = true)` + Returned for CREATE / ALTER / DROP / TRUNCATE (tables or pipelines). + +- `TableResult(table: Table` + Returned only by `SHOW TABLE`. + +- `PipelineResult(pipeline: IngestPipeline)` + Returned only by `SHOW PIPELINE`. + +- `QueryRows` + Returned by: + - `DESCRIBE TABLE` + - `DESCRIBE PIPELINE` + +- `SQLResult(sql: String)` + Returned by: + - `SHOW CREATE TABLE` + - `SHOW CREATE PIPELINE` + +--- + +## Examples + +--- + +### SELECT + +```scala +gateway.run(""" + SELECT id, name, age + FROM dql_users + WHERE age >= 18 + ORDER BY age DESC +""") +``` + +Result: + +```scala +ElasticSuccess(QueryRows(rows)) +``` + +--- + +### INSERT + +```scala +gateway.run(""" + INSERT INTO dml_users (id, name, age) + VALUES (1, 'Alice', 30) +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(inserted = 1)) +``` + +--- + +### UPDATE + +```scala +gateway.run(""" + UPDATE dml_users + SET age = 31 + WHERE id = 1 +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(updated = 1)) +``` + +--- + +### DELETE + +```scala +gateway.run("DELETE FROM dml_users WHERE id = 1") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(deleted = 1)) +``` + +--- + +### CREATE TABLE + +```scala +gateway.run(""" + CREATE TABLE dml_users ( + id INT PRIMARY KEY, + name TEXT, + age INT + ) +""") +``` + +--- + +### ALTER TABLE + +```scala +gateway.run(""" + ALTER TABLE dml_users + ADD COLUMN email TEXT +""") +``` + +--- + +### DROP TABLE + +```scala +gateway.run("DROP TABLE dml_users") +``` + +--- + +### TRUNCATE TABLE + +```scala +gateway.run("TRUNCATE TABLE dml_users") +``` + +--- + +### COPY INTO + +```scala +gateway.run(""" + COPY INTO dml_users + FROM 'classpath:/data/users.json' +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(inserted = 100)) +``` + +--- + +## Error Handling + +Errors are returned as: + +```scala +ElasticFailure(ElasticError(...)) +``` + +Typical cases: + +- SQL parsing errors +- Unsupported statements +- Mapping conflicts +- Invalid DDL operations +- Elasticsearch indexing errors + +Example: + +```scala +gateway.run("BAD SQL") +→ ElasticFailure(ElasticError(message = "Error parsing schema DDL statement: ...")) +``` + +--- + +## Notes + +- SQL comments (`-- ...`) are stripped automatically. +- Empty statements return `EmptyResult`. +- Multiple statements are executed sequentially. +- The last statement’s result is returned. +- Streaming queries (`QueryStream`) require an active `ActorSystem`. + +--- + +[Back to index](README.md) diff --git a/documentation/sql/README.md b/documentation/sql/README.md index fea27faa..bd2cbdad 100644 --- a/documentation/sql/README.md +++ b/documentation/sql/README.md @@ -16,3 +16,4 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: - [Keywords](keywords.md) - [DDL Support](ddl_statements.md) - [DML Support](dml_statements.md) +- [DQL Support](dql_statements.md) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index 41e98d74..81ddacf5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -168,6 +168,11 @@ object Parser ShowPipeline(pipeline) } + def showCreatePipeline: PackratParser[ShowCreatePipeline] = + ("SHOW" ~ "CREATE" ~ "PIPELINE") ~ ident ^^ { case _ ~ _ ~ _ ~ pipeline => + ShowCreatePipeline(pipeline) + } + def describePipeline: PackratParser[DescribePipeline] = ("DESCRIBE" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => DescribePipeline(pipeline) @@ -363,6 +368,11 @@ object Parser ShowTable(table) } + def showCreateTable: PackratParser[ShowCreateTable] = + ("SHOW" ~ "CREATE" ~ "TABLE") ~ ident ^^ { case _ ~ _ ~ _ ~ table => + ShowCreateTable(table) + } + def describeTable: PackratParser[DescribeTable] = ("DESCRIBE" ~ "TABLE") ~ ident ^^ { case _ ~ table => DescribeTable(table) @@ -572,9 +582,11 @@ object Parser dropTable | truncateTable | showTable | + showCreateTable | describeTable | dropPipeline | showPipeline | + showCreatePipeline | describePipeline def onConflict: PackratParser[OnConflict] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index d4e2d136..e314c673 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -551,6 +551,10 @@ package object query { override def sql: String = s"SHOW PIPELINE $name" } + case class ShowCreatePipeline(name: String) extends PipelineStatement { + override def sql: String = s"SHOW CREATE PIPELINE $name" + } + case class DescribePipeline(name: String) extends PipelineStatement { override def sql: String = s"DESCRIBE PIPELINE $name" } @@ -871,6 +875,10 @@ package object query { override def sql: String = s"SHOW TABLE $table" } + case class ShowCreateTable(table: String) extends TableStatement { + override def sql: String = s"SHOW CREATE TABLE $table" + } + case class DescribeTable(table: String) extends TableStatement { override def sql: String = s"DESCRIBE TABLE $table" } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index 5f89350b..d48178ee 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -22,13 +22,13 @@ import app.softnetwork.elastic.client.result.{ DdlResult, DmlResult, ElasticResult, - ElasticSuccess, - QueryPipeline, + PipelineResult, QueryResult, QueryRows, QueryStream, QueryStructured, - QueryTable + SQLResult, + TableResult } import app.softnetwork.elastic.client.scroll.ScrollMetrics import app.softnetwork.elastic.scalatest.ElasticTestKit @@ -161,8 +161,8 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala def assertShowTable(res: ElasticResult[QueryResult]): Table = { res.isSuccess shouldBe true - res.toOption.get shouldBe a[QueryTable] - res.toOption.get.asInstanceOf[QueryTable].table + res.toOption.get shouldBe a[TableResult] + res.toOption.get.asInstanceOf[TableResult].table } // ------------------------------------------------------------------------- @@ -171,8 +171,18 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala def assertShowPipeline(res: ElasticResult[QueryResult]): IngestPipeline = { res.isSuccess shouldBe true - res.toOption.get shouldBe a[QueryPipeline] - res.toOption.get.asInstanceOf[QueryPipeline].pipeline + res.toOption.get shouldBe a[PipelineResult] + res.toOption.get.asInstanceOf[PipelineResult].pipeline + } + + // ------------------------------------------------------------------------- + // Helper: assert SHOW CREATE result type + // ------------------------------------------------------------------------- + + def assertShowCreate(res: ElasticResult[QueryResult]): String = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[SQLResult] + res.toOption.get.asInstanceOf[SQLResult].sql } // ------------------------------------------------------------------------- @@ -194,6 +204,10 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val show = client.run("SHOW TABLE show_users").futureValue val table = assertShowTable(show) + val showCreate = client.run("SHOW CREATE TABLE show_users").futureValue + val sql = assertShowCreate(showCreate) + sql should include("CREATE OR REPLACE TABLE show_users") + val ddl = table.ddl ddl should include("CREATE OR REPLACE TABLE show_users") ddl should include("id INT NOT NULL") @@ -1439,6 +1453,10 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala val pipeline = assertShowPipeline(client.run("SHOW PIPELINE user_pipeline").futureValue) pipeline.name shouldBe "user_pipeline" pipeline.processors.size shouldBe 6 + + val showCreate = client.run("SHOW CREATE PIPELINE user_pipeline").futureValue + val ddl = assertShowCreate(showCreate) + ddl should include("CREATE OR REPLACE PIPELINE user_pipeline") } // --------------------------------------------------------------------------- From a9e1b77e05de108fc0dd978af73b22a0352f2cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 18:32:00 +0100 Subject: [PATCH 82/95] fix create table with primary key --- documentation/client/gateway.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index 5586e87d..8ad62bbc 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -290,9 +290,10 @@ ElasticSuccess(DmlResult(deleted = 1)) ```scala gateway.run(""" CREATE TABLE dml_users ( - id INT PRIMARY KEY, + id INT, name TEXT, - age INT + age INT, + PRIMARY KEY (id) ) """) ``` From 9cb1f6223afda68aba98fe5ae8c33bf98b4727e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 18:57:51 +0100 Subject: [PATCH 83/95] add Gateway api to main README.md --- README.md | 185 ++++++++++++++++++++---------------------------------- 1 file changed, 68 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index a2b953b5..f8d4cb58 100644 --- a/README.md +++ b/README.md @@ -30,22 +30,23 @@ SoftClient4ES provides a trait-based interface (`ElasticClientApi`) that aggrega #### **Core APIs** -| API | Description | Documentation | -|------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------| -| **[RefreshApi](documentation/client/refresh.md)** | Control index refresh for real-time search | [📖 Docs](documentation/client/refresh.md) | -| **[IndicesApi](documentation/client/indices.md)** | Create, update, and manage indices with settings and mappings | [📖 Docs](documentation/client/indices.md) | -| **[SettingsApi](documentation/client/settings.md)** | Dynamic index settings management | [📖 Docs](documentation/client/settings.md) | -| **[AliasApi](documentation/client/aliases.md)** | Manage index aliases for zero-downtime deployments | [📖 Docs](documentation/client/aliases.md) | -| **[MappingApi](documentation/client/mappings.md)** | Smart mapping management with automatic migration and rollback | [📖 Docs](documentation/client/mappings.md) | -| **[IndexApi](documentation/client/index.md)** | Index documents | [📖 Docs](documentation/client/index.md) | -| **[UpdateApi](documentation/client/update.md)** | Partial document updates with script support | [📖 Docs](documentation/client/update.md) | -| **[DeleteApi](documentation/client/delete.md)** | Delete documents by ID or query | [📖 Docs](documentation/client/delete.md) | -| **[BulkApi](documentation/client/bulk.md)** | High-performance bulk operations with Akka Streams | [📖 Docs](documentation/client/bulk.md) | -| **[GetApi](documentation/client/get.md)** | Get documents by ID | [📖 Docs](documentation/client/get.md) | -| **[SearchApi](documentation/client/search.md)** | Advanced search with SQL and aggregations support | [📖 Docs](documentation/client/search.md) | -| **[ScrollApi](documentation/client/scroll.md)** | Stream large datasets with automatic strategy detection (PIT, search_after, scroll) | [📖 Docs](documentation/client/scroll.md) | +| API | Description | Documentation | +|--------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------| +| **[RefreshApi](documentation/client/refresh.md)** | Control index refresh for real-time search | [📖 Docs](documentation/client/refresh.md) | +| **[IndicesApi](documentation/client/indices.md)** | Create, update, and manage indices with settings and mappings | [📖 Docs](documentation/client/indices.md) | +| **[SettingsApi](documentation/client/settings.md)** | Dynamic index settings management | [📖 Docs](documentation/client/settings.md) | +| **[AliasApi](documentation/client/aliases.md)** | Manage index aliases for zero-downtime deployments | [📖 Docs](documentation/client/aliases.md) | +| **[MappingApi](documentation/client/mappings.md)** | Smart mapping management with automatic migration and rollback | [📖 Docs](documentation/client/mappings.md) | +| **[IndexApi](documentation/client/index.md)** | Index documents | [📖 Docs](documentation/client/index.md) | +| **[UpdateApi](documentation/client/update.md)** | Partial document updates with script support | [📖 Docs](documentation/client/update.md) | +| **[DeleteApi](documentation/client/delete.md)** | Delete documents by ID or query | [📖 Docs](documentation/client/delete.md) | +| **[BulkApi](documentation/client/bulk.md)** | High-performance bulk operations with Akka Streams | [📖 Docs](documentation/client/bulk.md) | +| **[GetApi](documentation/client/get.md)** | Get documents by ID | [📖 Docs](documentation/client/get.md) | +| **[SearchApi](documentation/client/search.md)** | Advanced search with SQL and aggregations support | [📖 Docs](documentation/client/search.md) | +| **[ScrollApi](documentation/client/scroll.md)** | Stream large datasets with automatic strategy detection (PIT, search_after, scroll) | [📖 Docs](documentation/client/scroll.md) | | **[AggregationApi](documentation/client/aggregations.md)** | Type-safe way to execute aggregations using SQL queries | [📖 Docs](documentation/client/aggregations.md) | -| **[TemplateApi](documentation/client/templates.md)** | Templates management | [📖 Docs](documentation/client/templates.md) | +| **[TemplateApi](documentation/client/templates.md)** | Templates management | [📖 Docs](documentation/client/templates.md) | +| **[GatewayApi](documentation/client/gateway.md)** | Unified SQL interface for DQL, DML, and DDL statements | [📖 Docs](documentation/client/gateway.md) | #### **Client Implementations** @@ -182,7 +183,57 @@ result match { ### **3. SQL compatible** -### **3.1 SQL to Elasticsearch Query DSL** +### **3.1 SQL Gateway — Unified SQL Interface for Elasticsearch** + +SoftClient4ES includes a high‑level SQL Gateway that allows you to execute **DQL, DML, DDL, and Pipeline statements** directly against Elasticsearch using standard SQL syntax. + +The Gateway exposes a single entry point: + +```scala +gateway.run(sql: String): Future[ElasticResult[QueryResult]] +``` + +It automatically: + +- normalizes SQL (removes comments, trims whitespace) +- parses SQL into AST nodes +- routes statements to the appropriate executor +- returns a typed `QueryResult` (`QueryRows`, `TableResult`, `PipelineResult`, `DmlResult`, `DdlResult`, `SQLResult`) + +#### **Supported SQL Categories** + +| Category | Examples | +|---------|----------| +| **DQL** | `SELECT`, `JOIN UNNEST`, `GROUP BY`, `HAVING`, window functions | +| **DML** | `INSERT`, `UPDATE`, `DELETE`, `COPY INTO` | +| **DDL (Tables)** | `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `DESCRIBE TABLE`, `SHOW TABLE`, `SHOW CREATE TABLE` | +| **DDL (Pipelines)** | `CREATE PIPELINE`, `ALTER PIPELINE`, `DROP PIPELINE`, `DESCRIBE PIPELINE`, `SHOW PIPELINE`, `SHOW CREATE PIPELINE` | + +#### **Example** + +```scala +gateway.run(""" + CREATE TABLE users ( + id INT, + name TEXT, + age INT, + PRIMARY KEY (id) + ); + INSERT INTO users VALUES (1, 'Alice', 30); + SELECT * FROM users; +""") +``` + +#### **Documentation** + +- 📘 **Gateway API** — `documentation/client/gateway.md` +- 📘 **DQL** — `documentation/sql/dql_statements.md` +- 📘 **DML** — `documentation/sql/dml_statements.md` +- 📘 **DDL** — `documentation/sql/ddl_statements.md` + +--- + +### **3.2 SQL to Elasticsearch Query DSL** SoftClient4ES includes a powerful SQL parser that translates standard SQL `SELECT` queries into native Elasticsearch queries. @@ -1194,104 +1245,6 @@ client.scrollAsUnchecked[Product](dynamicQuery) --- -### **3.4 DML Support** - -SoftClient4ES supports **SQL Data Manipulation Language (DML)** statements for interacting with Elasticsearch indices. - -#### **Supported DML Statements** -- ✅ `INSERT INTO … VALUES (…)` -- ✅ `INSERT INTO … SELECT …` -- ✅ `UPDATE … SET … [WHERE …]` -- ✅ `DELETE FROM … [WHERE …]` - -**Examples:** -```sql -INSERT INTO users (id, name) VALUES (1, 'Alice'); -INSERT INTO users SELECT id, name FROM old_users; - -UPDATE users SET name = 'Bob', age = 42 WHERE id = 1; - -DELETE FROM users WHERE age > 30; -``` - ---- - -#### 🔄 DML Execution Strategy - -The SQL DML statements (`INSERT`, `UPDATE`, `DELETE`) are automatically translated into Elasticsearch operations. -The execution path depends on the **number of impacted rows**: - -- **Single row impacted** → direct ES operation: - - `INSERT` → `_index` - - `UPDATE` → `_update` - - `DELETE` → `_delete` - -- **Multiple rows impacted** → bulk ingestion: - - All operations are batched and executed via the `_bulk` API. - - Bulk execution is implemented using **Akka Streams**, ensuring efficient back‑pressure handling, parallelism, and resilience for large datasets. - ---- - -### **3.5 DDL Support** - -SoftClient4ES also supports **SQL Data Definition Language (DDL)** statements to manage table schemas mapped to Elasticsearch indices. - -#### **Supported DDL Statements** -- ✅ `CREATE [OR REPLACE] TABLE [IF NOT EXISTS] …` with column definitions, `DEFAULT`, `NOT NULL`, `OPTIONS`, `FIELDS` (multi‑fields or STRUCT) and `PARTITION BY …` -- ✅ `CREATE [OR REPLACE] TABLE … AS SELECT …` -- ✅ `ALTER TABLE …` with multiple sub‑statements: - - `ADD COLUMN [IF NOT EXISTS] …` - - `DROP COLUMN [IF EXISTS] …` - - `RENAME COLUMN … TO …` - - `ALTER COLUMN [IF EXISTS] … SET OPTIONS (…)` - - `ALTER COLUMN [IF EXISTS] … SET DEFAULT … / DROP DEFAULT` - - `ALTER COLUMN [IF EXISTS] … SET NOT NULL / DROP NOT NULL` - - `ALTER COLUMN [IF EXISTS] … SET DATA TYPE …` - - `ALTER COLUMN [IF EXISTS] … SET FIELDS (…)` (define nested STRUCT or multi‑fields) -- ✅ `DROP TABLE [IF EXISTS] … [CASCADE]` -- ✅ `TRUNCATE TABLE …` -- ✅ `CREATE [OR REPLACE] PIPELINE [IF NOT EXISTS] … WITH PROCESSORS (…)` -- ✅ `ALTER PIPELINE … [(]ADD|DROP PROCESSOR …[)]` -- ✅ `DROP PIPELINE [IF EXISTS] …` - -**Examples:** -```sql -CREATE TABLE IF NOT EXISTS users ( - id INT NOT NULL, - name VARCHAR DEFAULT 'anonymous', - birthdate DATE -) PARTITION BY birthdate (MONTH); - -ALTER TABLE users ( - ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, - RENAME COLUMN name TO full_name, - ALTER COLUMN IF EXISTS status SET DEFAULT 'active', - ALTER COLUMN IF EXISTS profile SET FIELDS ( - description VARCHAR DEFAULT 'N/A', - visibility BOOLEAN DEFAULT true - ) -); - -DROP TABLE IF EXISTS users CASCADE; -TRUNCATE TABLE users; -``` - ---- - -#### 🔄 MappingApi Migration Workflow - -The `MappingApi` provides intelligent mapping management with **automatic migration, validation, and rollback capabilities**. This ensures that SQL commands such as `ALTER TABLE … ALTER COLUMN SET TYPE …` are safely translated into Elasticsearch operations. - -##### ✨ Features -- ✅ **Automatic Change Detection**: Compares existing mappings with new ones -- ✅ **Safe Migration Strategy**: Creates temporary indices, reindexes, and renames atomically -- ✅ **Automatic Rollback**: Reverts to original state if migration fails -- ✅ **Backup & Restore**: Preserves original mappings and settings -- ✅ **Progress Tracking**: Detailed logging of migration steps -- ✅ **Validation**: Strict JSON validation with error reporting - ---- - 📖 **[Full SQL Validation Documentation](documentation/sql/validation.md)** 📖 **[Full SQL Documentation](documentation/sql/README.md)** @@ -1671,12 +1624,10 @@ client.createIndex("users", mapping) match { ### **Short-term** -- [ ] Support for `INSERT`, `UPDATE`, `DELETE` SQL operations -- [ ] Support for `CREATE TABLE`, `ALTER TABLE` SQL operations +- [ ] Full **JDBC connector for Elasticsearch** ### **Long-term** -- [ ] Full **JDBC connector for Elasticsearch** - [ ] Advanced monitoring and metrics --- From e9e5a9d80ea6f996bd812a54e2c45e5191d53185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Wed, 7 Jan 2026 19:07:38 +0100 Subject: [PATCH 84/95] add Gateway diagrams --- README.md | 58 ++++++++++-- documentation/client/gateway.md | 154 ++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f8d4cb58..962a6560 100644 --- a/README.md +++ b/README.md @@ -200,13 +200,61 @@ It automatically: - routes statements to the appropriate executor - returns a typed `QueryResult` (`QueryRows`, `TableResult`, `PipelineResult`, `DmlResult`, `DdlResult`, `SQLResult`) +#### **Architecture Diagram — SQL Gateway** + +```text + +----------------------+ + | GatewayApi | + | run(sql: String) | + +----------+-----------+ + | + v + +----------------------+ + | Parser | + | SQL → Statement | + +----------+-----------+ + | + ------------------------------------------------ + | | | + v v v ++---------------+ +---------------+ +----------------+ +| DqlStatement | | DmlStatement | | DdlStatement | ++-------+-------+ +-------+-------+ +--------+-------+ + | | | + v v v ++---------------+ +---------------+ +----------------------+ +| DqlExecutor | | DmlExecutor | | DdlRouterExecutor | ++-------+-------+ +-------+-------+ +----------+-----------+ + / \ + / \ + v v + +----------------+ +------------------+ + | TableExecutor | | PipelineExecutor | + +--------+-------+ +--------+---------+ + | | + v v + +-----------+ +-----------+ + |ElasticSearch| |ElasticSearch| + +-----------+ +-----------+ + + +-----------------------------------------------+ + | QueryResult | + +-----------------------------------------------+ + | QueryRows | QueryStream | QueryStructured | + | DmlResult | DdlResult | TableResult | + | PipelineResult | SQLResult (SHOW CREATE) | + +-----------------------------------------------+ +``` + +--- + #### **Supported SQL Categories** -| Category | Examples | -|---------|----------| -| **DQL** | `SELECT`, `JOIN UNNEST`, `GROUP BY`, `HAVING`, window functions | -| **DML** | `INSERT`, `UPDATE`, `DELETE`, `COPY INTO` | -| **DDL (Tables)** | `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `DESCRIBE TABLE`, `SHOW TABLE`, `SHOW CREATE TABLE` | +| Category | Examples | +|---------------------|--------------------------------------------------------------------------------------------------------------------| +| **DQL** | `SELECT`, `JOIN UNNEST`, `GROUP BY`, `HAVING`, window functions | +| **DML** | `INSERT`, `UPDATE`, `DELETE`, `COPY INTO` | +| **DDL (Tables)** | `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `DESCRIBE TABLE`, `SHOW TABLE`, `SHOW CREATE TABLE` | | **DDL (Pipelines)** | `CREATE PIPELINE`, `ALTER PIPELINE`, `DROP PIPELINE`, `DESCRIBE PIPELINE`, `SHOW PIPELINE`, `SHOW CREATE PIPELINE` | #### **Example** diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md index 8ad62bbc..67d19115 100644 --- a/documentation/client/gateway.md +++ b/documentation/client/gateway.md @@ -213,6 +213,160 @@ DDL operations return one of the following: --- +## **Architecture Diagram** + +```text ++-------------------------------------------------------------+ +| GatewayApi | +|-------------------------------------------------------------| +| - run(sql: String): Future[ElasticResult[QueryResult]] | ++-------------------------------+-----------------------------+ + | + v + Normalize / strip comments / split on ';' + | + v + +----------------+ + | Parser | + | (SQL → AST) | + +----------------+ + | + v + +------------------------+ + | Statement | + +------------------------+ + / | \ + / | \ + v v v + +----------------+ +----------------+ +----------------+ + | DqlStatement | | DmlStatement | | DdlStatement | + +----------------+ +----------------+ +----------------+ + | + v + +-------------------------------+ + | DdlRouterExecutor | + +-------------------------------+ + | - execute(ddl: DdlStatement) | + +-------------------------------+ + / \ + / \ + v v + +---------------------------+ +---------------------------+ + | TableStatement | | PipelineStatement | + +---------------------------+ +---------------------------+ + | TableExecutor | | PipelineExecutor | + +---------------------------+ +---------------------------+ + | | + v v + +----------------+ +----------------+ + | Elasticsearch | | Elasticsearch | + +----------------+ +----------------+ + +DQL path: +--------- + +DqlStatement + | + v ++----------------+ +| DqlExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++---------------------------------------------+ +| DQL QueryResult: | +| - QueryRows | +| - QueryStream | +| - QueryStructured | ++---------------------------------------------+ + + +DML path: +--------- + +DmlStatement + | + v ++----------------+ +| DmlExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++---------------------------------------------+ +| DML QueryResult: | +| - DmlResult | ++---------------------------------------------+ + + +DDL (Tables) path: +------------------ + +TableStatement + | + v ++----------------+ +| TableExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++------------------------------------------------------+ +| DDL Table QueryResult: | +| - DdlResult (CREATE / ALTER / DROP / TRUNC) | +| - TableResult (SHOW TABLE) | +| - SQLResult (SHOW CREATE TABLE) | +| - QueryRows (DESCRIBE TABLE, etc.) | ++------------------------------------------------------+ + + +DDL (Pipelines) path: +--------------------- + +PipelineStatement + | + v ++--------------------+ +| PipelineExecutor | ++--------------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++------------------------------------------------------+ +| DDL Pipeline QueryResult: | +| - DdlResult (CREATE / ALTER / DROP) | +| - PipelineResult (SHOW PIPELINE) | +| - SQLResult (SHOW CREATE PIPELINE) | +| - QueryRows (DESCRIBE PIPELINE, etc.) | ++------------------------------------------------------+ + + +Final API surface: +------------------ + +gateway.run(sql: String) + → Future[ElasticResult[QueryResult]] +``` + +--- + ## Examples --- From 0c78af3d0302ed34a5b7be4269ad5f91a78c47a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 9 Jan 2026 08:11:32 +0100 Subject: [PATCH 85/95] rename default value processor to set processor, replace sql field with optional description field for all processors --- .../softnetwork/elastic/schema/package.scala | 4 +- .../elastic/sql/query/package.scala | 10 +- .../elastic/sql/schema/package.scala | 275 +++++++++++++----- .../elastic/sql/parser/ParserSpec.scala | 46 ++- 4 files changed, 240 insertions(+), 95 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 417e1f0a..897b8392 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -21,13 +21,13 @@ import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{ Column, DateIndexNameProcessor, - DefaultValueProcessor, IngestPipelineType, IngestProcessor, PartitionDate, PrimaryKeyProcessor, Schema, ScriptProcessor, + SetProcessor, Table } import app.softnetwork.elastic.sql.serialization._ @@ -433,7 +433,7 @@ package object schema { enrichedCols.update(col, c.copy(script = Some(p))) } - case p: DefaultValueProcessor => + case p: SetProcessor => val col = p.column enrichedCols.get(col).foreach { c => enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index e314c673..a14ce264 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -19,7 +19,6 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.schema.{ Column, - DefaultValueProcessor, IngestPipeline, IngestPipelineType, IngestProcessor, @@ -29,6 +28,7 @@ import app.softnetwork.elastic.sql.schema.{ RenameProcessor, Schema, ScriptProcessor, + SetProcessor, Table => DdlTable } import app.softnetwork.elastic.sql.function.aggregate.WindowFunction @@ -413,8 +413,7 @@ package object query { s"update-$table-${Instant.now}", IngestPipelineType.Custom, values.map { case (k, v) => - DefaultValueProcessor( - sql = s"SET DEFAULT $k = ${v.value}", + SetProcessor( column = k, value = v ) @@ -678,7 +677,7 @@ package object query { s"DROP COLUMN$ifExistsClause $columnName" } override def ddlProcessor: Option[IngestProcessor] = Some( - RemoveProcessor(sql = sql, column = columnName) + RemoveProcessor(column = columnName) ) } case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { @@ -755,8 +754,7 @@ package object query { } override def ddlProcessor: Option[IngestProcessor] = Some( - DefaultValueProcessor( - sql = sql, + SetProcessor( column = columnName, value = defaultValue ) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 3f1969c6..7120645b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -41,22 +41,22 @@ package object schema { object IngestProcessorType { case object Script extends IngestProcessorType { - def name: String = "script" + val name: String = "script" } case object Rename extends IngestProcessorType { - def name: String = "rename" + val name: String = "rename" } case object Remove extends IngestProcessorType { - def name: String = "remove" + val name: String = "remove" } case object Set extends IngestProcessorType { - def name: String = "set" + val name: String = "set" } case object DateIndexName extends IngestProcessorType { def name: String = "date_index_name" } def apply(n: String): IngestProcessorType = new IngestProcessorType { - override def name: String = n + override val name: String = n } } @@ -71,7 +71,8 @@ package object schema { def json: String = mapper.writeValueAsString(node) def pipelineType: IngestPipelineType def processorType: IngestProcessorType - def description: String = sql.trim + def sql: String // = s"${processorType.name.toUpperCase}${Value(properties).ddl}" + def description: Option[String] def name: String = processorType.name def properties: Map[String, Any] @@ -155,15 +156,15 @@ package object schema { val props = processor.get(processorType) processorType match { - case "set" => + case IngestProcessorType.Set.name => val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") - val valueNode = props.get("value") + val desc = Option(props.get("description")).map(_.asText()) + val valueNode = Option(props.get("value")) val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) - if (field == "_id") { + if (field == "_id" && valueNode.isDefined) { val value = - valueNode + valueNode.get .asText() .trim .stripPrefix("{{") @@ -172,33 +173,42 @@ package object schema { // DdlPrimaryKeyProcessor val cols = value.split(sqlConfig.compositeKeySeparator).toSet PrimaryKeyProcessor( - sql = desc, + pipelineType = pipelineType, + description = desc, column = "_id", value = cols, ignoreFailure = ignoreFailure ) } else { - DefaultValueProcessor( - sql = desc, + val copyFrom = Option(props.get("copy_from")).map(_.asText()) + val doOverride = Option(props.get("override")).map(_.asBoolean()) + val ignoreEmptyValue = Option(props.get("ignore_empty_value")).map(_.asBoolean()) + val doIf = Option(props.get("if")).map(_.asText()) + SetProcessor( + pipelineType = pipelineType, + description = desc, column = field, - value = Value(valueNode.asText()), + value = valueNode.map(v => Value(v.asText())).getOrElse(Null), + copyFrom = copyFrom, + doOverride = doOverride, + ignoreEmptyValue = ignoreEmptyValue, + doIf = doIf, ignoreFailure = ignoreFailure ) } - case "script" => - val desc = { - if (props.has("description")) props.get("description").asText() - else "" - } + case IngestProcessorType.Script.name => + val desc = Option(props.get("description")).map(_.asText()) val lang = props.get("lang").asText() require(lang == "painless", s"Only painless supported, got $lang") val source = props.get("source").asText() val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) desc match { - case ScriptDescRegex(col, dataType, script) => + case Some(ScriptDescRegex(col, dataType, script)) => ScriptProcessor( + pipelineType = pipelineType, + description = desc, script = script, column = col, dataType = SQLTypes(dataType), @@ -207,15 +217,16 @@ package object schema { ) case _ => GenericProcessor( + pipelineType = pipelineType, processorType = IngestProcessorType.Script, properties = mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap ) } - case "date_index_name" => + case IngestProcessorType.DateIndexName.name => val field = props.get("field").asText() - val desc = Option(props.get("description")).map(_.asText()).getOrElse("") + val desc = Option(props.get("description")).map(_.asText()) val rounding = props.get("date_rounding").asText() val formats = Option(props.get("date_formats")) .map(_.elements().asScala.toList.map(_.asText())) @@ -223,13 +234,51 @@ package object schema { val prefix = props.get("index_name_prefix").asText() DateIndexNameProcessor( - sql = desc, + pipelineType = pipelineType, + description = desc, column = field, dateRounding = rounding, dateFormats = formats, prefix = prefix ) + case IngestProcessorType.Remove.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + + RemoveProcessor( + pipelineType = pipelineType, + description = desc, + column = field + ) + + case IngestProcessorType.Rename.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val targetField = props.get("target_field").asText() + + RenameProcessor( + pipelineType = pipelineType, + description = desc, + column = field, + newName = targetField + ) + + case IngestProcessorType.Enrich.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val policyName = props.get("policy_name").asText() + val targetField = props.get("target_field").asText() + val maxMatches = Option(props.get("max_matches")).map(_.asInt()).getOrElse(1) + EnrichProcessor( + pipelineType = pipelineType, + description = desc, + column = targetField, + policyName = policyName, + field = field, + maxMatches = maxMatches + ) + case other => GenericProcessor( pipelineType = pipelineType, @@ -247,6 +296,11 @@ package object schema { processorType: IngestProcessorType, properties: Map[String, Any] ) extends IngestProcessor { + override def description: Option[String] = properties.get("description") match { + case Some(s: String) => Some(s) + case _ => None + } + override def sql: String = ddl override def column: String = properties.get("field") match { case Some(s: String) => s @@ -260,6 +314,7 @@ package object schema { case class ScriptProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, script: String, column: String, dataType: SQLType, @@ -273,7 +328,7 @@ package object schema { def processorType: IngestProcessorType = IngestProcessorType.Script override def properties: Map[String, Any] = Map( - "description" -> description, + "description" -> description.getOrElse(sql), "lang" -> "painless", "source" -> source, "ignore_failure" -> ignoreFailure @@ -283,97 +338,142 @@ package object schema { case class RenameProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, column: String, newName: String, ignoreFailure: Boolean = true, - ignoreMissing: Boolean = true + ignoreMissing: Option[Boolean] = None ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Rename - def sql: String = s"$column RENAME TO $newName" + override def sql: String = s"$column RENAME TO $newName" override def properties: Map[String, Any] = Map( - "description" -> description, + "description" -> description.getOrElse(sql), "field" -> column, "target_field" -> newName, - "ignore_failure" -> ignoreFailure, - "ignore_missing" -> ignoreMissing - ) + "ignore_failure" -> ignoreFailure + ) ++ ignoreMissing + .map("ignore_missing" -> _) + .toMap } case class RemoveProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, - sql: String, + description: Option[String] = None, column: String, ignoreFailure: Boolean = true, - ignoreMissing: Boolean = true + ignoreMissing: Option[Boolean] = None ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Remove + override def sql: String = s"REMOVE $column" + override def properties: Map[String, Any] = Map( - "description" -> description, + "description" -> description.getOrElse(sql), "field" -> column, - "ignore_failure" -> ignoreFailure, - "ignore_missing" -> ignoreMissing - ) + "ignore_failure" -> ignoreFailure + ) ++ ignoreMissing + .map("ignore_missing" -> _) + .toMap } case class PrimaryKeyProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, - sql: String, + description: Option[String] = None, column: String, value: Set[String], ignoreFailure: Boolean = false, - ignoreEmptyValue: Boolean = false, + ignoreEmptyValue: Option[Boolean] = Some(false), separator: String = "\\|\\|" ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Set + override def sql: String = s"PRIMARY KEY (${value.mkString(", ")})" + override def properties: Map[String, Any] = Map( - "description" -> description, - "field" -> column, - "value" -> value.mkString("{{", separator, "}}"), - "ignore_failure" -> ignoreFailure, - "ignore_empty_value" -> ignoreEmptyValue - ) + "description" -> description.getOrElse(sql), + "field" -> column, + "value" -> value.mkString("{{", separator, "}}"), + "ignore_failure" -> ignoreFailure + ) ++ ignoreEmptyValue + .map("ignore_empty_value" -> _) + .toMap } - case class DefaultValueProcessor( + case class SetProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, - sql: String, + description: Option[String] = None, column: String, value: Value[_], + copyFrom: Option[String] = None, + doOverride: Option[Boolean] = None, + ignoreEmptyValue: Option[Boolean] = None, + doIf: Option[String] = None, ignoreFailure: Boolean = true ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.Set - def _if: String = { - if (column.contains(".")) - s"""ctx.${column.split("\\.").mkString("?.")} == null""" - else - s"""ctx.$column == null""" + override def sql: String = { + val base = copyFrom match { + case Some(source) => s"$column COPY FROM $source" + case None => s"$column SET VALUE ${value.sql}" + } + val withOverride = doOverride match { + case Some(overrideValue) => s"$base OVERRIDE $overrideValue" + case None => base + } + val withIgnoreEmpty = ignoreEmptyValue match { + case Some(ignoreEmpty) => s"$withOverride IGNORE EMPTY VALUE $ignoreEmpty" + case None => withOverride + } + val withIf = doIf match { + case Some(condition) => s"$withIgnoreEmpty IF $condition" + case None => withIgnoreEmpty + } + withIf } - override def properties: Map[String, Any] = Map( - "description" -> description, - "field" -> column, - "value" -> { + lazy val defaultValue: Option[Any] = { + if (copyFrom.isDefined) None + else value match { - case IdValue | IngestTimestampValue => s"{{${value.value}}}" - case _ => value.value + case IdValue | IngestTimestampValue => Some(s"{{${value.value}}}") + case Null => None + case _ => Some(value.value) } - }, - "ignore_failure" -> ignoreFailure, - "if" -> _if - ) + } + + override def properties: Map[String, Any] = { + Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "ignore_failure" -> ignoreFailure + ) ++ defaultValue.map("value" -> _) ++ + copyFrom.map("copy_from" -> _) ++ + doOverride.map("override" -> _) ++ + doIf.map("if" -> _) ++ + ignoreEmptyValue.map("ignore_empty_value" -> _) + } + + override def validate(): Either[String, Unit] = { + for { + _ <- + if (value != Null && copyFrom.isDefined) + Left( + s"Only one of value or copy_from should be defined for SET processor on column $column" + ) + else Right(()) + } yield () + } } case class DateIndexNameProcessor( pipelineType: IngestPipelineType = IngestPipelineType.Default, - sql: String, + description: Option[String] = None, column: String, dateRounding: String, dateFormats: List[String], @@ -382,8 +482,10 @@ package object schema { ) extends IngestProcessor { def processorType: IngestProcessorType = IngestProcessorType.DateIndexName + override def sql: String = s"PARTITION BY $column (${TimeUnit(dateRounding).sql})" + override def properties: Map[String, Any] = Map( - "description" -> description, + "description" -> description.getOrElse(sql), "field" -> column, "date_rounding" -> dateRounding, "date_formats" -> dateFormats, @@ -399,7 +501,6 @@ package object schema { if (primaryKey.nonEmpty) { Seq( PrimaryKeyProcessor( - sql = s"PRIMARY KEY (${primaryKey.mkString(", ")})", column = "_id", value = primaryKey.toSet, separator = sqlConfig.compositeKeySeparator @@ -410,19 +511,45 @@ package object schema { } } + case class EnrichProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + policyName: String, + field: String, + maxMatches: Int = 1, + ignoreFailure: Boolean = true + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Enrich + + def targetField: String = column + + override def sql: String = s"ENRICH USING POLICY $policyName FROM $field INTO $targetField" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "policy_name" -> policyName, + "field" -> field, + "target_field" -> targetField, + "max_matches" -> maxMatches, + "ignore_failure" -> ignoreFailure + ) + + } + sealed trait IngestPipelineType { def name: String } object IngestPipelineType { case object Default extends IngestPipelineType { - def name: String = "DEFAULT" + val name: String = "DEFAULT" } case object Final extends IngestPipelineType { - def name: String = "FINAL" + val name: String = "FINAL" } case object Custom extends IngestPipelineType { - def name: String = "CUSTOM" + val name: String = "CUSTOM" } } @@ -664,10 +791,15 @@ package object schema { def processors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ defaultValue.map { dv => - DefaultValueProcessor( - sql = s"$path DEFAULT $dv", + SetProcessor( column = path, - value = dv + value = dv, + doIf = Some { + if (path.contains(".")) + s"""ctx.${path.split("\\.").mkString("?.")} == null""" + else + s"""ctx.$path == null""" + } ) }.toSeq ++ multiFields.flatMap(_.processors) @@ -888,7 +1020,6 @@ package object schema { def processor(table: Table): DateIndexNameProcessor = DateIndexNameProcessor( - sql = sql, column = column, dateRounding = dateRounding, dateFormats = dateFormats, diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 51f8977e..1178abda 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -7,12 +7,12 @@ import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.schema.{ mapper, DateIndexNameProcessor, - DefaultValueProcessor, IngestPipelineType, IngestProcessorType, PartitionDate, PrimaryKeyProcessor, - ScriptProcessor + ScriptProcessor, + SetProcessor } import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec @@ -994,7 +994,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" val indexMappings = schema.indexMappings println(indexMappings) indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin @@ -1003,7 +1003,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name DEFAULT 'anonymous', age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name DEFAULT 'anonymous'","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at DEFAULT _ingest.timestamp","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) @@ -1342,20 +1342,25 @@ class ParserSpec extends AnyFlatSpec with Matchers { processors.size shouldBe 6 processors.find(_.column == "name") match { case Some( - DefaultValueProcessor( + SetProcessor( IngestPipelineType.Default, - "DEFAULT 'anonymous'", + Some("DEFAULT 'anonymous'"), "name", StringValue("anonymous"), + None, + None, + None, + Some("ctx.name == null"), true ) ) => - case other => fail(s"Expected DdlDefaultValueProcessor for name, got $other") + case other => fail(s"Expected SetProcessor for name, got $other") } processors.find(_.column == "age") match { case Some( ScriptProcessor( IngestPipelineType.Default, + Some("age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))"), "DATE_DIFF(birthdate, CURRENT_DATE, YEAR)", "age", SQLTypes.Int, @@ -1370,11 +1375,15 @@ class ParserSpec extends AnyFlatSpec with Matchers { } processors.find(_.column == "ingested_at") match { case Some( - DefaultValueProcessor( + SetProcessor( IngestPipelineType.Default, - "DEFAULT _ingest.timestamp", + Some("DEFAULT _ingest.timestamp"), "ingested_at", IngestTimestampValue, + None, + None, + None, + _, true ) ) => @@ -1384,6 +1393,9 @@ class ParserSpec extends AnyFlatSpec with Matchers { case Some( ScriptProcessor( IngestPipelineType.Default, + Some( + "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))" + ), "DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)", "profile.seniority", SQLTypes.Int, @@ -1400,7 +1412,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { case Some( DateIndexNameProcessor( IngestPipelineType.Default, - "PARTITION BY birthdate (MONTH)", + Some("PARTITION BY birthdate (MONTH)"), "birthdate", "M", List("yyyy-MM"), @@ -1414,11 +1426,11 @@ class ParserSpec extends AnyFlatSpec with Matchers { case Some( PrimaryKeyProcessor( IngestPipelineType.Default, - "PRIMARY KEY (id)", + Some("PRIMARY KEY (id)"), "_id", cols, false, - false, + Some(false), "\\|\\|" ) ) => @@ -1453,14 +1465,18 @@ class ParserSpec extends AnyFlatSpec with Matchers { ) if ie => statements.size shouldBe 2 statements.collect { case AddPipelineProcessor(p) => p } match { - case DefaultValueProcessor( + case SetProcessor( IngestPipelineType.Default, - "status DEFAULT 'active'", + Some("status DEFAULT 'active'"), "status", StringValue("active"), + None, + None, + None, + Some("ctx.status == null"), true ) :: Nil => - case other => fail(s"Expected AddPipelineProcessor with DdlSetProcessor, got $other") + case other => fail(s"Expected AddPipelineProcessor with SetProcessor, got $other") } statements.collect { case DropPipelineProcessor(IngestProcessorType.Set, f) => f From de6523e948237df51c7c5dd44404fe919de295da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 9 Jan 2026 08:13:04 +0100 Subject: [PATCH 86/95] fix DataIndexName --- .../main/scala/app/softnetwork/elastic/sql/schema/package.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 7120645b..b93925a2 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -54,6 +54,7 @@ package object schema { } case object DateIndexName extends IngestProcessorType { def name: String = "date_index_name" + val name: String = "date_index_name" } def apply(n: String): IngestProcessorType = new IngestProcessorType { override val name: String = n From f3d2ddd7bf171bddbaed0fc681f9b87caf7e6cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 9 Jan 2026 08:13:43 +0100 Subject: [PATCH 87/95] fix DataIndexName --- .../main/scala/app/softnetwork/elastic/sql/schema/package.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index b93925a2..b43f8fb9 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -53,7 +53,6 @@ package object schema { val name: String = "set" } case object DateIndexName extends IngestProcessorType { - def name: String = "date_index_name" val name: String = "date_index_name" } def apply(n: String): IngestProcessorType = new IngestProcessorType { From 9119b3f9265dd9bb75536c1ec010ea60cee98e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Fri, 9 Jan 2026 16:06:11 +0100 Subject: [PATCH 88/95] add artificial primary key while creating a new table if no primary keys have been defined - fix column search within a table --- sql/src/main/resources/softnetwork-sql.conf | 1 + .../elastic/sql/config/ElasticSqlConfig.scala | 3 ++- .../elastic/sql/config/ElasticSqlConfig.scala | 3 ++- .../elastic/sql/query/package.scala | 22 ++++++++++++++++--- .../elastic/sql/schema/package.scala | 12 ++-------- .../client/GatewayApiIntegrationSpec.scala | 22 ++++++++++++++++--- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/sql/src/main/resources/softnetwork-sql.conf b/sql/src/main/resources/softnetwork-sql.conf index dfb62c17..74fdf8e4 100644 --- a/sql/src/main/resources/softnetwork-sql.conf +++ b/sql/src/main/resources/softnetwork-sql.conf @@ -1,3 +1,4 @@ sql { composite-key-separator = "\\|\\|" # regex separator for composite keys in SQL ddl queries + artificial-primary-key-column-name = "id" # name of the artificial primary key column added to tables without primary key(s) defined } diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index 15ec4b20..75c56386 100644 --- a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -5,7 +5,8 @@ import com.typesafe.scalalogging.StrictLogging import configs.Configs case class ElasticSqlConfig( - compositeKeySeparator: String + compositeKeySeparator: String, + artificialPrimaryKeyColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala index d0c9c75e..c777f29d 100644 --- a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala +++ b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -21,7 +21,8 @@ import com.typesafe.scalalogging.StrictLogging import configs.ConfigReader case class ElasticSqlConfig( - compositeKeySeparator: String + compositeKeySeparator: String, + artificialPrimaryKeyColumnName: String ) object ElasticSqlConfig extends StrictLogging { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index a14ce264..4b18a3d4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -16,8 +16,9 @@ package app.softnetwork.elastic.sql -import app.softnetwork.elastic.sql.`type`.SQLType +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.schema.{ + sqlConfig, Column, IngestPipeline, IngestPipelineType, @@ -584,8 +585,23 @@ package object query { } } + private val artificialPkColumnName: String = + s"${table}_${sqlConfig.artificialPrimaryKeyColumnName}" + lazy val columns: Seq[Column] = { - ddl match { + val artificialPkColumn = if (primaryKey.isEmpty) { + Seq( + Column( + name = artificialPkColumnName, + dataType = SQLTypes.Keyword, + defaultValue = Some(IdValue), + comment = Some("Artificial primary key column") + ) + ) + } else { + Nil + } + (ddl match { case Left(select) => select match { case s: SingleSearch => @@ -599,7 +615,7 @@ package object query { case _ => Nil } case Right(cols) => cols - } + }).filterNot(_.name == artificialPkColumnName) ++ artificialPkColumn } lazy val mappings: Map[String, Value[_]] = options.get("mappings") match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index b43f8fb9..68859fd0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -696,11 +696,7 @@ package object schema { def find(path: String): Option[Column] = { if (path.contains(".")) { val parts = path.split("\\.") - cols.get(parts.head).flatMap { col => - col.multiFields.toStream - .flatMap(_.find(parts.tail.mkString("."))) - .headOption - } + cols.get(parts.head).flatMap(col => col.find(parts.tail.mkString("."))) } else { cols.get(path) } @@ -1136,11 +1132,7 @@ package object schema { def find(path: String): Option[Column] = { if (path.contains(".")) { val parts = path.split("\\.") - cols.get(parts.head).flatMap { col => - col.multiFields.toStream - .flatMap(_.find(parts.tail.mkString("."))) - .headOption - } + cols.get(parts.head).flatMap(col => col.find(parts.tail.mkString("."))) } else { cols.get(path) } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala index d48178ee..ce8fa93b 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -32,6 +32,7 @@ import app.softnetwork.elastic.client.result.{ } import app.softnetwork.elastic.client.scroll.ScrollMetrics import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.{DoubleValue, IdValue} import app.softnetwork.elastic.sql.`type`.SQLTypes import app.softnetwork.elastic.sql.schema.{IngestPipeline, Table} import app.softnetwork.persistence.generateUUID @@ -527,8 +528,20 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala assertDdl(client.run(alter).futureValue) val table = assertShowTable(client.run("SHOW TABLE users_alter5").futureValue) - table.ddl should include("reputation DOUBLE DEFAULT 0.0") - table.defaultPipeline.processors.size shouldBe 2 + table.find("profile.reputation") match { + case Some(col) => + col.dataType shouldBe SQLTypes.Double + col.defaultValue shouldBe Some(DoubleValue(0.0)) + case _ => fail("Column 'profile.reputation' not found") + } + table.defaultPipeline.processors.size shouldBe 3 // added processor to create an artificial primary key + table.find("users_alter5_id") match { + case Some(col) => + col.dataType shouldBe SQLTypes.Keyword + col.defaultValue shouldBe Some(IdValue) + col.processors.size shouldBe 1 + case _ => fail("Column 'users_alter5_id' not found") + } } // --------------------------------------------------------------------------- @@ -556,7 +569,10 @@ trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with Scala assertDdl(client.run(dropNotNull).futureValue) val table = assertShowTable(client.run("SHOW TABLE users_alter6").futureValue) - table.ddl should not include "NOT NULL" + table.find("status") match { + case Some(col) => col.nullable shouldBe true + case _ => fail("Column 'status' not found") + } } // --------------------------------------------------------------------------- From 9ed9e091c356dee52f210b724330dbccdffd2461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 10 Jan 2026 14:14:18 +0100 Subject: [PATCH 89/95] to fix infinite loops --- .../elastic/sql/function/math/package.scala | 6 +----- .../app/softnetwork/elastic/sql/function/package.scala | 4 +--- .../elastic/sql/function/string/package.scala | 7 +------ .../elastic/sql/function/time/package.scala | 4 +--- .../scala/app/softnetwork/elastic/sql/package.scala | 3 ++- .../elastic/sql/parser/function/string/package.scala | 2 +- .../elastic/sql/parser/function/time/package.scala | 10 +++++----- .../scala/app/softnetwork/elastic/sql/query/From.scala | 6 ++++-- .../app/softnetwork/elastic/sql/query/GroupBy.scala | 4 +++- .../app/softnetwork/elastic/sql/query/Select.scala | 4 +++- 10 files changed, 22 insertions(+), 28 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 97fb54f5..9d7fb5f3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -69,9 +69,7 @@ package object math { case object Atan extends Expr("ATAN") with Trigonometric case object Atan2 extends Expr("ATAN2") with Trigonometric - sealed trait MathematicalFunction - extends TransformFunction[SQLNumeric, SQLNumeric] - with FunctionWithIdentifier { + sealed trait MathematicalFunction extends TransformFunction[SQLNumeric, SQLNumeric] { override def inputType: SQLNumeric = SQLTypes.Numeric override def outputType: SQLNumeric = mathOp.baseType @@ -80,8 +78,6 @@ package object math { override def fun: Option[PainlessScript] = Some(mathOp) - override def identifier: Identifier = Identifier(this) - override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 9f99cf29..064e6b25 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -45,10 +45,8 @@ package object function { override def shouldBeScripted: Boolean = identifier.shouldBeScripted } - trait FunctionWithValue[+T] extends FunctionWithIdentifier with TokenValue { + trait FunctionWithValue[+T] extends Function with TokenValue { def value: T - - override def identifier: Identifier = Identifier(this) } object FunctionUtils { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 894a74c1..316b7a3e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -18,7 +18,6 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ Expr, - Identifier, IntValue, PainlessContext, PainlessScript, @@ -108,17 +107,13 @@ package object string { override def nullable: Boolean = false } - sealed trait StringFunction[Out <: SQLType] - extends TransformFunction[SQLVarchar, Out] - with FunctionWithIdentifier { + sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] { override def inputType: SQLVarchar = SQLTypes.Varchar override def outputType: Out def stringOp: StringOp - override def identifier: Identifier = Identifier(this) - override def toSQL(base: String): String = if (base.nonEmpty) s"$sql($base)" else sql diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 4a96eaee..1856002f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -129,10 +129,8 @@ package object time { override def baseType: SQLType = SQLTypes.Time } - sealed trait SystemFunction extends FunctionWithIdentifier { + sealed trait SystemFunction extends Function { override def system: Boolean = true - - override def identifier: Identifier = Identifier(this) } sealed trait CurrentFunction extends SystemFunction with PainlessScript with DateMathScript { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 1f7a333e..96819c09 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -1124,7 +1124,8 @@ package object sql { } val parts: Seq[String] = name.split("\\.").toSeq val tableAlias = parts.head - if (request.tableAliases.values.toSeq.contains(tableAlias)) { + val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._2) + if (table.nonEmpty) { request.unnestAliases.find(_._1 == tableAlias) match { case Some(tuple) if !nested => val nestedElement = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index d924e914..2268fb5e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -125,7 +125,7 @@ package object string { trim | ltrim | rtrim) ^^ { sf => - sf.identifier + Identifier(sf) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index cd8525d9..7bb9e950 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -37,25 +37,25 @@ package object time { def current_date: PackratParser[Identifier] = CurrentDate.regex ~ parens.? ^^ { case _ ~ p => - CurrentDate(p.isDefined).identifier + Identifier(CurrentDate(p.isDefined)) } def current_time: PackratParser[Identifier] = CurrentTime.regex ~ parens.? ^^ { case _ ~ p => - CurrentTime(p.isDefined).identifier + Identifier(CurrentTime(p.isDefined)) } def current_timestamp: PackratParser[Identifier] = CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => - CurrentTimestamp(p.isDefined).identifier + Identifier(CurrentTimestamp(p.isDefined)) } def now: PackratParser[Identifier] = Now.regex ~ parens.? ^^ { case _ ~ p => - Now(p.isDefined).identifier + Identifier(Now(p.isDefined)) } def today: PackratParser[Identifier] = Today.regex ~ parens.? ^^ { case _ ~ p => - Today(p.isDefined).identifier + Identifier(Today(p.isDefined)) } private[this] def current_function: PackratParser[Identifier] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index a953e211..f4c0333b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -71,8 +71,6 @@ sealed trait Join extends Updateable { _ <- this match { case j if joinType.isDefined && on.isEmpty && joinType.get != CrossJoin => Left(s"JOIN $j requires an ON clause") - case j if joinType.isEmpty && on.isDefined => - Left(s"JOIN $j requires a JOIN type") case j if alias.isEmpty => Left(s"JOIN $j requires an alias") case _ => Right(()) @@ -186,6 +184,8 @@ case class From(tables: Seq[Table]) extends Updateable { override def validate(): Either[String, Unit] = { if (tables.isEmpty) { Left("At least one table is required in FROM clause") + } else if (tables.size > 1) { + Left("Only one table is supported in FROM clause") } else { for { _ <- tables.map(_.validate()).filter(_.isLeft) match { @@ -195,6 +195,8 @@ case class From(tables: Seq[Table]) extends Updateable { } yield () } } + + lazy val mainTable: Table = tables.head } case class NestedElement( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index c02aace2..4437309d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -55,6 +55,8 @@ case class Bucket( size: Option[Int] = None ) extends Updateable with PainlessScript { + def tableAlias: Option[String] = identifier.tableAlias + def table: Option[String] = identifier.table override def sql: String = s"$identifier" def update(request: SingleSearch): Bucket = { identifier.functions.headOption match { @@ -76,7 +78,7 @@ case class Bucket( lazy val sourceBucket: String = if (identifier.nested) { - identifier.tableAlias + tableAlias .map(a => s"$a.") .getOrElse("") + identifier.name.split("\\.").tail.mkString(".") } else { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index bfe8adf4..45d47194 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -45,12 +45,14 @@ case class Field( with FunctionChain with PainlessScript with DateMathScript { + def tableAlias: Option[String] = identifier.tableAlias + def table: Option[String] = identifier.table def isScriptField: Boolean = functions.nonEmpty && !hasAggregation && identifier.bucket.isEmpty override def sql: String = s"$identifier${asString(fieldAlias)}" lazy val sourceField: String = { if (identifier.nested) { - identifier.tableAlias + tableAlias .orElse(fieldAlias.map(_.alias)) .map(a => s"$a.") .getOrElse("") + identifier.name From b2b10b0775ec5bd899c2a684a3be1df8205d21de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sat, 10 Jan 2026 14:15:53 +0100 Subject: [PATCH 90/95] clean import --- .../app/softnetwork/elastic/sql/function/math/package.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 9d7fb5f3..adcc4fae 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -18,7 +18,6 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ Expr, - Identifier, IntValue, PainlessContext, PainlessParam, From 1c2a550cf1b88c245871af3d0f987b7d4fddfa88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 11 Jan 2026 05:30:01 +0100 Subject: [PATCH 91/95] fix scala fmt --- .../softnetwork/elastic/sql/function/string/package.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 316b7a3e..bbd98d01 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -16,13 +16,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{ - Expr, - IntValue, - PainlessContext, - PainlessScript, - TokenRegex -} +import app.softnetwork.elastic.sql.{Expr, IntValue, PainlessContext, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{ SQLBigInt, SQLBool, From 370ea4192763e8dabd4d8e7acc905d2d185a2000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 11 Jan 2026 05:32:50 +0100 Subject: [PATCH 92/95] add optional table to identifier --- .../app/softnetwork/elastic/sql/package.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index 96819c09..ad54ce26 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -835,6 +835,7 @@ package object sql { def update(request: SingleSearch): Identifier def tableAlias: Option[String] + def table: Option[String] def distinct: Boolean def nested: Boolean def nestedElement: Option[NestedElement] @@ -1095,7 +1096,8 @@ package object sql { bucket: Option[Bucket] = None, nestedElement: Option[NestedElement] = None, bucketPath: String = "", - col: Option[Column] = None + col: Option[Column] = None, + table: Option[String] = None ) extends Identifier { def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) @@ -1144,7 +1146,8 @@ package object sql { bucket = request.bucketNames.get(identifierName).orElse(bucket), nestedElement = nestedElement, bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(colName)) + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) .withFunctions(this.updateFunctions(request)) case Some(tuple) if nested => @@ -1157,7 +1160,8 @@ package object sql { fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(colName)) + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) .withFunctions(this.updateFunctions(request)) case None if nested => @@ -1167,7 +1171,8 @@ package object sql { fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(name)) + col = request.schema.flatMap(schema => schema.find(name)), + table = table ) .withFunctions(this.updateFunctions(request)) case _ => @@ -1178,7 +1183,8 @@ package object sql { fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), bucketPath = bucketPath, - col = request.schema.flatMap(schema => schema.find(colName)) + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) } } else { From 8a49badaa9156cf8f546dc793a28450337a6c72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 11 Jan 2026 05:33:34 +0100 Subject: [PATCH 93/95] add enrich processor type --- .../elastic/sql/schema/package.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index 68859fd0..dc0d7a68 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -55,6 +55,9 @@ package object schema { case object DateIndexName extends IngestProcessorType { val name: String = "date_index_name" } + case object Enrich extends IngestProcessorType { + val name: String = "enrich" + } def apply(n: String): IngestProcessorType = new IngestProcessorType { override val name: String = n } @@ -1117,6 +1120,25 @@ package object schema { } } + /** Definition of a table within the schema + * + * @param name + * the table name + * @param columns + * the list of columns within the table + * @param primaryKey + * optional list of columns composing the primary key + * @param partitionBy + * optional partition by date definition + * @param mappings + * optional index mappings + * @param settings + * optional index settings + * @param processors + * optional list of ingest processors associated to this table (apart from column processors) + * @param aliases + * optional map of aliases associated to this table + */ case class Table( name: String, columns: List[Column], @@ -1127,6 +1149,7 @@ package object schema { processors: Seq[IngestProcessor] = Seq.empty, aliases: Map[String, Value[_]] = Map.empty ) extends DdlToken { + lazy val indexName: String = name.toLowerCase private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap def find(path: String): Option[Column] = { From 06aada6b65ff4c8f0e05db73ea3321d8f600a8ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Sun, 11 Jan 2026 08:15:34 +0100 Subject: [PATCH 94/95] fix from validation when multiple tables are involved without joins --- .../main/scala/app/softnetwork/elastic/sql/query/From.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index f4c0333b..3ad17781 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -184,8 +184,8 @@ case class From(tables: Seq[Table]) extends Updateable { override def validate(): Either[String, Unit] = { if (tables.isEmpty) { Left("At least one table is required in FROM clause") - } else if (tables.size > 1) { - Left("Only one table is supported in FROM clause") + } else if (tables.filter(_.joins.nonEmpty).size > 1) { + Left("Only one table with joins is supported in FROM clause") } else { for { _ <- tables.map(_.validate()).filter(_.isLeft) match { From 38ae45c3fba6f356456e691ec5e8e2bc63137e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Mon, 12 Jan 2026 13:41:55 +0100 Subject: [PATCH 95/95] to fix set processor --- .../softnetwork/elastic/sql/schema/package.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index dc0d7a68..5b2ebf93 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -186,7 +186,13 @@ package object schema { val copyFrom = Option(props.get("copy_from")).map(_.asText()) val doOverride = Option(props.get("override")).map(_.asBoolean()) val ignoreEmptyValue = Option(props.get("ignore_empty_value")).map(_.asBoolean()) - val doIf = Option(props.get("if")).map(_.asText()) + val doIf = + if (props.has("if")) { + val cond = props.get("if") + if (cond.has("source")) { + Option(cond.get("source")).map(_.asText()) + } else Option(cond).map(_.asText()) + } else None SetProcessor( pipelineType = pipelineType, description = desc, @@ -195,7 +201,10 @@ package object schema { copyFrom = copyFrom, doOverride = doOverride, ignoreEmptyValue = ignoreEmptyValue, - doIf = doIf, + doIf = doIf match { + case Some(condition) if condition.nonEmpty => Some(condition) + case _ => None + }, ignoreFailure = ignoreFailure ) }