diff --git a/README.md b/README.md index 4eee333d..962a6560 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,23 @@ SoftClient4ES provides a trait-based interface (`ElasticClientApi`) that aggrega #### **Core APIs** -| API | Description | Documentation | -|------------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------| -| **[RefreshApi](documentation/client/refresh.md)** | Control index refresh for real-time search | [πŸ“– Docs](documentation/client/refresh.md) | -| **[IndicesApi](documentation/client/indices.md)** | Create, update, and manage indices with settings and mappings | [πŸ“– Docs](documentation/client/indices.md) | -| **[SettingsApi](documentation/client/settings.md)** | Dynamic index settings management | [πŸ“– Docs](documentation/client/settings.md) | -| **[AliasApi](documentation/client/aliases.md)** | Manage index aliases for zero-downtime deployments | [πŸ“– Docs](documentation/client/aliases.md) | -| **[MappingApi](documentation/client/mappings.md)** | Smart mapping management with automatic migration and rollback | [πŸ“– Docs](documentation/client/mappings.md) | -| **[IndexApi](documentation/client/index.md)** | Index documents | [πŸ“– Docs](documentation/client/index.md) | -| **[UpdateApi](documentation/client/update.md)** | Partial document updates with script support | [πŸ“– Docs](documentation/client/update.md) | -| **[DeleteApi](documentation/client/delete.md)** | Delete documents by ID or query | [πŸ“– Docs](documentation/client/delete.md) | -| **[BulkApi](documentation/client/bulk.md)** | High-performance bulk operations with Akka Streams | [πŸ“– Docs](documentation/client/bulk.md) | -| **[GetApi](documentation/client/get.md)** | Get documents by ID | [πŸ“– Docs](documentation/client/get.md) | -| **[SearchApi](documentation/client/search.md)** | Advanced search with SQL and aggregations support | [πŸ“– Docs](documentation/client/search.md) | -| **[ScrollApi](documentation/client/scroll.md)** | Stream large datasets with automatic strategy detection (PIT, search_after, scroll) | [πŸ“– Docs](documentation/client/scroll.md) | +| API | Description | Documentation | +|--------------------------------------------------------|-------------------------------------------------------------------------------------|-------------------------------------------------| +| **[RefreshApi](documentation/client/refresh.md)** | Control index refresh for real-time search | [πŸ“– Docs](documentation/client/refresh.md) | +| **[IndicesApi](documentation/client/indices.md)** | Create, update, and manage indices with settings and mappings | [πŸ“– Docs](documentation/client/indices.md) | +| **[SettingsApi](documentation/client/settings.md)** | Dynamic index settings management | [πŸ“– Docs](documentation/client/settings.md) | +| **[AliasApi](documentation/client/aliases.md)** | Manage index aliases for zero-downtime deployments | [πŸ“– Docs](documentation/client/aliases.md) | +| **[MappingApi](documentation/client/mappings.md)** | Smart mapping management with automatic migration and rollback | [πŸ“– Docs](documentation/client/mappings.md) | +| **[IndexApi](documentation/client/index.md)** | Index documents | [πŸ“– Docs](documentation/client/index.md) | +| **[UpdateApi](documentation/client/update.md)** | Partial document updates with script support | [πŸ“– Docs](documentation/client/update.md) | +| **[DeleteApi](documentation/client/delete.md)** | Delete documents by ID or query | [πŸ“– Docs](documentation/client/delete.md) | +| **[BulkApi](documentation/client/bulk.md)** | High-performance bulk operations with Akka Streams | [πŸ“– Docs](documentation/client/bulk.md) | +| **[GetApi](documentation/client/get.md)** | Get documents by ID | [πŸ“– Docs](documentation/client/get.md) | +| **[SearchApi](documentation/client/search.md)** | Advanced search with SQL and aggregations support | [πŸ“– Docs](documentation/client/search.md) | +| **[ScrollApi](documentation/client/scroll.md)** | Stream large datasets with automatic strategy detection (PIT, search_after, scroll) | [πŸ“– Docs](documentation/client/scroll.md) | | **[AggregationApi](documentation/client/aggregations.md)** | Type-safe way to execute aggregations using SQL queries | [πŸ“– Docs](documentation/client/aggregations.md) | +| **[TemplateApi](documentation/client/templates.md)** | Templates management | [πŸ“– Docs](documentation/client/templates.md) | +| **[GatewayApi](documentation/client/gateway.md)** | Unified SQL interface for DQL, DML, and DDL statements | [πŸ“– Docs](documentation/client/gateway.md) | #### **Client Implementations** @@ -181,7 +183,105 @@ result match { ### **3. SQL compatible** -### **3.1 SQL to Elasticsearch Query DSL** +### **3.1 SQL Gateway β€” Unified SQL Interface for Elasticsearch** + +SoftClient4ES includes a high‑level SQL Gateway that allows you to execute **DQL, DML, DDL, and Pipeline statements** directly against Elasticsearch using standard SQL syntax. + +The Gateway exposes a single entry point: + +```scala +gateway.run(sql: String): Future[ElasticResult[QueryResult]] +``` + +It automatically: + +- normalizes SQL (removes comments, trims whitespace) +- parses SQL into AST nodes +- routes statements to the appropriate executor +- returns a typed `QueryResult` (`QueryRows`, `TableResult`, `PipelineResult`, `DmlResult`, `DdlResult`, `SQLResult`) + +#### **Architecture Diagram β€” SQL Gateway** + +```text + +----------------------+ + | GatewayApi | + | run(sql: String) | + +----------+-----------+ + | + v + +----------------------+ + | Parser | + | SQL β†’ Statement | + +----------+-----------+ + | + ------------------------------------------------ + | | | + v v v ++---------------+ +---------------+ +----------------+ +| DqlStatement | | DmlStatement | | DdlStatement | ++-------+-------+ +-------+-------+ +--------+-------+ + | | | + v v v ++---------------+ +---------------+ +----------------------+ +| DqlExecutor | | DmlExecutor | | DdlRouterExecutor | ++-------+-------+ +-------+-------+ +----------+-----------+ + / \ + / \ + v v + +----------------+ +------------------+ + | TableExecutor | | PipelineExecutor | + +--------+-------+ +--------+---------+ + | | + v v + +-----------+ +-----------+ + |ElasticSearch| |ElasticSearch| + +-----------+ +-----------+ + + +-----------------------------------------------+ + | QueryResult | + +-----------------------------------------------+ + | QueryRows | QueryStream | QueryStructured | + | DmlResult | DdlResult | TableResult | + | PipelineResult | SQLResult (SHOW CREATE) | + +-----------------------------------------------+ +``` + +--- + +#### **Supported SQL Categories** + +| Category | Examples | +|---------------------|--------------------------------------------------------------------------------------------------------------------| +| **DQL** | `SELECT`, `JOIN UNNEST`, `GROUP BY`, `HAVING`, window functions | +| **DML** | `INSERT`, `UPDATE`, `DELETE`, `COPY INTO` | +| **DDL (Tables)** | `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `DESCRIBE TABLE`, `SHOW TABLE`, `SHOW CREATE TABLE` | +| **DDL (Pipelines)** | `CREATE PIPELINE`, `ALTER PIPELINE`, `DROP PIPELINE`, `DESCRIBE PIPELINE`, `SHOW PIPELINE`, `SHOW CREATE PIPELINE` | + +#### **Example** + +```scala +gateway.run(""" + CREATE TABLE users ( + id INT, + name TEXT, + age INT, + PRIMARY KEY (id) + ); + INSERT INTO users VALUES (1, 'Alice', 30); + SELECT * FROM users; +""") +``` + +#### **Documentation** + +- πŸ“˜ **Gateway API** β€” `documentation/client/gateway.md` +- πŸ“˜ **DQL** β€” `documentation/sql/dql_statements.md` +- πŸ“˜ **DML** β€” `documentation/sql/dml_statements.md` +- πŸ“˜ **DDL** β€” `documentation/sql/ddl_statements.md` + +--- + +### **3.2 SQL to Elasticsearch Query DSL** SoftClient4ES includes a powerful SQL parser that translates standard SQL `SELECT` queries into native Elasticsearch queries. @@ -210,6 +310,9 @@ SoftClient4ES includes a powerful SQL parser that translates standard SQL `SELEC - βœ… Date / Time functions (`YEAR`, `QUARTER`, `MONTH`, `WEEK`, `DAY`, `HOUR`, `MINUTE`, `SECOND`, `MILLISECOND`, `MICROSECOND`, `NANOSECOND`, `EPOCHDAY`, `OFFSET_SECONDS`, `LAST_DAY`, `WEEKDAY`, `YEARDAY`, `INTERVAL`, `CURRENT_DATE`, `CURDATE`, `TODAY`, `NOW`, `CURRENT_TIME`, `CURTIME`, `CURRENT_DATETIME`, `CURRENT_TIMESTAMP`, `DATE_ADD`, `DATEADD`, `DATE_SUB`, `DATESUB`, `DATETIME_ADD`, `DATETIMEADD`, `DATETIME_SUB`, `DATETIMESUB`, `DATE_DIFF`, `DATEDIFF`, `DATE_FORMAT`, `DATE_PARSE`, `DATETIME_FORMAT`, `DATETIME_PARSE`, `DATE_TRUNC`, `EXTRACT`) - βœ… Geospatial functions (`POINT`, `ST_DISTANCE`) - βœ… Aggregate functions (`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `DISTINCT`, `FIRST_VALUE`, `LAST_VALUE`, `ARRAY_AGG`) +- βœ… [Window functions](#32-window-functions-support) with `OVER` clause +- βœ… [DML Support](#34-dml-support) (`INSERT`, `UPDATE`, `DELETE`) +- βœ… [DDL Support](#35-ddl-support) (`CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `TRUNCATE TABLE`, `CREATE PIPELINE`, `ALTER PIPELINE`, `DROP PIPELINE`) **Example:** @@ -1188,6 +1291,8 @@ client.searchAsUnchecked[Product](SQLQuery(dynamicQuery)) client.scrollAsUnchecked[Product](dynamicQuery) ``` +--- + πŸ“– **[Full SQL Validation Documentation](documentation/sql/validation.md)** πŸ“– **[Full SQL Documentation](documentation/sql/README.md)** @@ -1517,18 +1622,18 @@ ThisBuild / resolvers ++= Seq( // For Elasticsearch 6 // Using Jest client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-jest-client" % 0.15.0 // Or using Rest High Level client -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es6-rest-client" % 0.15.0 // For Elasticsearch 7 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es7-rest-client" % 0.15.0 // For Elasticsearch 8 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es8-java-client" % 0.15.0 // For Elasticsearch 9 -libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.14.2 +libraryDependencies += "app.softnetwork.elastic" %% s"softclient4es9-java-client" % 0.15.0 ``` ### **Quick Example** @@ -1567,12 +1672,10 @@ client.createIndex("users", mapping) match { ### **Short-term** -- [ ] Support for `INSERT`, `UPDATE`, `DELETE` SQL operations -- [ ] Support for `CREATE TABLE`, `ALTER TABLE` SQL operations +- [ ] Full **JDBC connector for Elasticsearch** ### **Long-term** -- [ ] Full **JDBC connector for Elasticsearch** - [ ] Advanced monitoring and metrics --- diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d6c11fd8..72132c8d 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -22,7 +22,6 @@ import app.softnetwork.elastic.sql.query.{ Asc, BucketIncludesExcludes, BucketNode, - BucketTree, Criteria, Desc, Field, @@ -101,7 +100,7 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - ): ElasticAggregation = { + )(implicit timestamp: Long): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -156,14 +155,14 @@ object ElasticAggregation { if (transformFuncs.nonEmpty) { val context = PainlessContext() val scriptSrc = identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => val context = PainlessContext() val scriptSrc = th.identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) case _ => buildField(aggName, sourceField) } @@ -231,7 +230,7 @@ object ElasticAggregation { .filter(_.isScriptField) .groupBy(_.sourceField) .map(_._2.head) - .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) + .map(f => f.sourceField -> now(Script(f.painless(None)).lang("painless"))) .toMap, size = limit, sorts = th.orderBy @@ -269,14 +268,14 @@ object ElasticAggregation { val painless = script.identifier.painless(None) bucketScriptAggregation( aggName, - Script(s"$painless").lang("painless"), + now(Script(s"$painless").lang("painless")), params.toMap ) case _ => throw new IllegalArgumentException(s"Unsupported aggregation type: $aggType") } - val nestedElement = identifier.nestedElement + val nestedElement = sqlAgg.nestedElement val nestedElements: Seq[NestedElement] = nestedElement.map(n => NestedElements.buildNestedTrees(Seq(n))).getOrElse(Nil) @@ -349,11 +348,7 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Seq[Aggregation] = { - val trees = BucketTree(buckets.flatMap(_.headOption)) - println( - s"[DEBUG] buildBuckets called with buckets: \n$trees" - ) + )(implicit timestamp: Long): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => @@ -371,7 +366,7 @@ object ElasticAggregation { if (!bucket.isBucketScript && bucket.shouldBeScripted) { val context = PainlessContext() val painless = bucket.painless(Some(context)) - Some(Script(s"$context$painless").lang("painless")) + Some(now(Script(s"$context$painless").lang("painless"))) } else { None } @@ -520,7 +515,7 @@ object ElasticAggregation { val bucketSelector = bucketSelectorAggregation( "having_filter", - Script(script), + now(Script(script)), extractMetricsPathForBucket( criteria, nested, diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 124326dc..c013e82c 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -45,7 +45,7 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - ): Query = { + )(implicit timestamp: Long): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ @@ -77,13 +77,10 @@ case class ElasticBridge(filter: ElasticFilter) { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.map { source => - (n.path.split('.').toSeq ++ Seq(source)).mkString(".") - }.toArray - ) + inner = inner.docValueFields( + n.sources.map { source => + (n.path.split('.').toSeq ++ Seq(source)).mkString(".") + } ) } inner diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 0473a2eb..32305e18 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -21,7 +21,9 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit + timestamp: Long + ): Query = { val query = criteria.boolQuery.copy(group = group) query .filter(criteria.asFilter(Option(query))) diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 69be62e9..d8de91e1 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -51,10 +51,17 @@ import scala.language.implicitConversions package object bridge { + def now(script: Script)(implicit timestamp: Long): Script = { + if (!script.script.contains("params.__now__")) { + return script + } + script.param("__now__", timestamp) + } + implicit def requestToNestedFilterAggregation( - request: SQLSearchRequest, + request: SingleSearch, innerHitsName: String - ): Option[FilterAggregation] = { + )(implicit timestamp: Long): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -129,8 +136,8 @@ package object bridge { } implicit def requestToFilterAggregation( - request: SQLSearchRequest - ): Option[FilterAggregation] = + request: SingleSearch + )(implicit timestamp: Long): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -146,9 +153,9 @@ package object bridge { } implicit def requestToRootAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[AbstractAggregation] = { + )(implicit timestamp: Long): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -198,9 +205,9 @@ package object bridge { } implicit def requestToScopedAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[NestedAggregation] = { + )(implicit timestamp: Long): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -330,7 +337,7 @@ package object bridge { scopedAggregations } - implicit def requestToNestedWithoutCriteriaQuery(request: SQLSearchRequest): Option[Query] = + implicit def requestToNestedWithoutCriteriaQuery(request: SingleSearch): Option[Query] = NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match { case Nil => None case nestedTrees => @@ -342,12 +349,7 @@ package object bridge { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.toArray - ) - ) + inner = inner.docValueFields(n.sources) } inner } @@ -410,7 +412,9 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit + timestamp: Long + ): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -425,7 +429,9 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + implicit def requestToSearchRequest( + request: SingleSearch + )(implicit timestamp: Long): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -467,7 +473,11 @@ package object bridge { case _ => nestedWithoutCriteriaQuery.getOrElse(matchAllQuery()) } - } sourceFiltering (fields, excludes) + } + + if (!request.deleteByQuery && !request.updateByQuery) { + _search = _search sourceFiltering (fields, excludes) + } _search = if (allAggregations.nonEmpty) { _search aggregations { @@ -485,13 +495,15 @@ package object bridge { val script = field.painless(Some(context)) scriptField( field.scriptName, - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") - .params(field.identifier.functions.headOption match { - case Some(f: PainlessParams) => f.params - case _ => Map.empty[String, Any] - }) + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + .params(field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + }) + ) ) } } @@ -523,9 +535,11 @@ package object bridge { } val scriptSort = ScriptSort( - script = Script(script = script) - .lang("painless") - .scriptType(Source), + script = now( + Script(script = script) + .lang("painless") + .scriptType(Source) + ), scriptSortType = sort.field.out match { case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number case _ => ScriptSortType.String @@ -556,8 +570,8 @@ package object bridge { } implicit def requestToMultiSearchRequest( - request: SQLMultiSearchRequest - ): MultiSearchRequest = { + request: MultiSearch + )(implicit timestamp: Long): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -568,7 +582,7 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -581,7 +595,7 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) return scriptQuery( - Script(script = s"$context$script").lang("painless").scriptType("source") + now(Script(script = s"$context$script").lang("painless").scriptType("source")) ) } // Geo distance special case @@ -799,18 +813,22 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => matchAllQuery() @@ -866,7 +884,7 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - ): Query = { + )(implicit timestamp: Long): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -996,12 +1014,12 @@ package object bridge { } implicit def sqlQueryToAggregations( - query: SQLQuery - ): Seq[ElasticAggregation] = { + query: SelectStatement + )(implicit timestamp: Long): Seq[ElasticAggregation] = { import query._ - request + statement .map { - case Left(l) => + case l: SingleSearch => val filteredAgg: Option[FilterAggregation] = requestToFilterAggregation(l) l.aggregates .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts, l.sqlAggregations)) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 8da795a9..4749d58a 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -7,16 +7,20 @@ import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { - import Queries._ + import parser.Queries._ import scala.language.implicitConversions def asQuery(sql: String): String = { import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 245bd887..cd630515 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1,20 +1,24 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.Queries._ +import app.softnetwork.elastic.sql.parser.Queries._ import app.softnetwork.elastic.sql.query._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions - implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { - sqlQuery.request match { - case Some(Left(value)) => + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) case None => throw new IllegalArgumentException( @@ -23,9 +27,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } } - "SQLQuery" should "perform native count" in { + "SelectStatement" should "perform native count" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(t.id) c2 from Table t where t.nom = 'Nom'") + SelectStatement("select count(t.id) c2 from Table t where t.nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -61,7 +65,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform count distinct" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") + SelectStatement("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -97,7 +101,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as email from index i join unnest(i.emails) as inner_emails where i.nom = 'Nom'" ) results.size shouldBe 1 @@ -142,7 +146,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with nested criteria" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" ) results.size shouldBe 1 @@ -205,7 +209,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with filter" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\"" ) results.size shouldBe 1 @@ -279,7 +283,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with \"and not\" operator" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" ) results.size shouldBe 1 @@ -353,7 +357,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with date filtering" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" ) results.size shouldBe 1 @@ -421,21 +425,21 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested select" in { val select: ElasticSearchRequest = - SQLQuery(""" - |SELECT - |profileId, - |profile_ccm.email as email, - |profile_ccm.city as city, - |profile_ccm.firstName as firstName, - |profile_ccm.lastName as lastName, - |profile_ccm.postalCode as postalCode, - |profile_ccm.birthYear as birthYear - |FROM index join unnest(index.profiles) as profile_ccm - |WHERE - |((profile_ccm.postalCode BETWEEN "10" AND "99999") - |AND - |(profile_ccm.birthYear <= 2000)) - |limit 100""".stripMargin) + SelectStatement(""" + |SELECT + |profileId, + |profile_ccm.email as email, + |profile_ccm.city as city, + |profile_ccm.firstName as firstName, + |profile_ccm.lastName as lastName, + |profile_ccm.postalCode as postalCode, + |profile_ccm.birthYear as birthYear + |FROM index join unnest(index.profiles) as profile_ccm + |WHERE + |((profile_ccm.postalCode BETWEEN "10" AND "99999") + |AND + |(profile_ccm.birthYear <= 2000)) + |limit 100""".stripMargin) val query = select.query println(query) query shouldBe @@ -470,17 +474,15 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "profile_ccm", | "from": 0, - | "_source": { - | "includes": [ - | "profiles.email", - | "profiles.postalCode", - | "profiles.firstName", - | "profiles.lastName", - | "profiles.birthYear", - | "profiles.city" - | ] - | }, - | "size": 100 + | "size": 100, + | "docvalue_fields": [ + | "profiles.email", + | "profiles.city", + | "profiles.firstName", + | "profiles.lastName", + | "profiles.postalCode", + | "profiles.birthYear" + | ] | } | } | } @@ -499,7 +501,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "exclude fields from select" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( except ) select.query shouldBe @@ -517,7 +519,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform query with group by and having" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHaving) + SelectStatement(groupByWithHaving) val query = select.query println(query) query shouldBe @@ -577,7 +579,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform complex query" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( s"""SELECT | inner_products.category as cat, | min(inner_products.price) as min_price, @@ -839,7 +841,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "add script fields" in { val select: ElasticSearchRequest = - SQLQuery(fieldsWithInterval) + SelectStatement(fieldsWithInterval) val query = select.query println(query) query shouldBe @@ -851,7 +853,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -879,7 +881,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateTimeAndInterval) + SelectStatement(filterWithDateTimeAndInterval) val query = select.query println(query) query shouldBe @@ -914,7 +916,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateAndInterval) + SelectStatement(filterWithDateAndInterval) val query = select.query println(query) query shouldBe @@ -949,7 +951,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithTimeAndInterval) + SelectStatement(filterWithTimeAndInterval) val query = select.query println(query) query shouldBe @@ -997,10 +999,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle having with date functions" in { val select: ElasticSearchRequest = - SQLQuery("""SELECT userId, MAX(createdAt) as lastSeen - |FROM table - |GROUP BY userId - |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) + SelectStatement("""SELECT userId, MAX(createdAt) as lastSeen + |FROM table + |GROUP BY userId + |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) val query = select.query println(query) query shouldBe @@ -1028,7 +1030,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1041,11 +1046,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by with having and date time functions" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHavingAndDateTimeFunctions) + SelectStatement(groupByWithHavingAndDateTimeFunctions) val query = select.query println(query) query shouldBe @@ -1090,7 +1096,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1106,11 +1115,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by index" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( groupByWithHavingAndDateTimeFunctions.replace("GROUP BY Country, City", "GROUP BY 3, 2") ) val query = select.query @@ -1157,7 +1167,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1173,56 +1186,57 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle date_parse function" in { val select: ElasticSearchRequest = - SQLQuery(dateParse) + SelectStatement(dateParse) val query = select.query println(query) query shouldBe """{ - | "query": { - | "bool": { - | "filter": [ - | { - | "exists": { - | "field": "identifier2" - | } - | } - | ] - | } - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "identifier": { - | "terms": { - | "field": "identifier", - | "min_doc_count": 1, - | "order": { - | "ct": "desc" - | } - | }, - | "aggs": { - | "ct": { - | "value_count": { - | "field": "identifier2" - | } - | }, - | "lastSeen": { - | "max": { - | "field": "createdAt", - | "script": { - | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" - | } - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "identifier2" + | } + | } + | ] + | } + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "identifier": { + | "terms": { + | "field": "identifier", + | "min_doc_count": 1, + | "order": { + | "ct": "desc" + | } + | }, + | "aggs": { + | "ct": { + | "value_count": { + | "field": "identifier2" + | } + | }, + | "lastSeen": { + | "max": { + | "field": "createdAt", + | "script": { + | "lang": "painless", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : LocalDate.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + | } + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("defp", "def p") .replaceAll("defe", "def e") @@ -1246,7 +1260,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateFormat) + SelectStatement(dateFormat) val query = select.query println(query) query shouldBe @@ -1266,49 +1280,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z'))); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1345,52 +1359,52 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_parse function" in { // #25 val select: ElasticSearchRequest = - SQLQuery(dateTimeParse) + SelectStatement(dateTimeParse) val query = select.query println(query) query shouldBe """{ - | "query": { - | "bool": { - | "filter": [ - | { - | "exists": { - | "field": "identifier2" - | } - | } - | ] - | } - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "identifier": { - | "terms": { - | "field": "identifier", - | "min_doc_count": 1, - | "order": { - | "ct": "desc" - | } - | }, - | "aggs": { - | "lastSeen": { - | "max": { - | "field": "createdAt", - | "script": { - | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" - | } - | } - | }, - | "ct": { - | "value_count": { - | "field": "identifier2" - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "identifier2" + | } + | } + | ] + | } + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "identifier": { + | "terms": { + | "field": "identifier", + | "min_doc_count": 1, + | "order": { + | "ct": "desc" + | } + | }, + | "aggs": { + | "lastSeen": { + | "max": { + | "field": "createdAt", + | "script": { + | "lang": "painless", + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).truncatedTo(ChronoUnit.MINUTES).get(ChronoField.YEAR)" + | } + | } + | }, + | "ct": { + | "value_count": { + | "field": "identifier2" + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("defp", "def p") .replaceAll("defe", "def e") @@ -1415,7 +1429,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeFormat) + SelectStatement(dateTimeFormat) val query = select.query println(query) query shouldBe @@ -1435,7 +1449,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1469,7 +1483,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_diff function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateDiff) + SelectStatement(dateDiff) val query = select.query println(query) query shouldBe @@ -1481,7 +1495,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" | } | } | }, @@ -1511,7 +1525,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with date_diff function" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithDateDiff) + SelectStatement(aggregationWithDateDiff) val query = select.query println(query) query shouldBe @@ -1532,7 +1546,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1564,7 +1578,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateAdd) + SelectStatement(dateAdd) val query = select.query println(query) query shouldBe @@ -1615,7 +1629,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_sub function as script field" in { // 30 val select: ElasticSearchRequest = - SQLQuery(dateSub) + SelectStatement(dateSub) val query = select.query println(query) query shouldBe @@ -1666,7 +1680,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeAdd) + SelectStatement(dateTimeAdd) val query = select.query println(query) query shouldBe @@ -1717,7 +1731,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_sub function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeSub) + SelectStatement(dateTimeSub) val query = select.query println(query) query shouldBe @@ -1768,7 +1782,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnull) + SelectStatement(isnull) val query = select.query println(query) query shouldBe @@ -1806,7 +1820,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnotnull) + SelectStatement(isnotnull) val query = select.query println(query) query shouldBe @@ -1848,7 +1862,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null criteria as must_not exists" in { val select: ElasticSearchRequest = - SQLQuery(isNullCriteria) + SelectStatement(isNullCriteria) val query = select.query println(query) query shouldBe @@ -1880,7 +1894,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull criteria as exists" in { val select: ElasticSearchRequest = - SQLQuery(isNotNullCriteria) + SelectStatement(isNotNullCriteria) val query = select.query println(query) query shouldBe @@ -1906,7 +1920,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle coalesce function as script field" in { val select: ElasticSearchRequest = - SQLQuery(coalesce) + SelectStatement(coalesce) val query = select.query println(query) query shouldBe @@ -1918,7 +1932,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -1953,11 +1970,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle nullif function as script field" in { val select: ElasticSearchRequest = - SQLQuery(nullif) + SelectStatement(nullif) val query = select.query println(query) query shouldBe @@ -1969,7 +1987,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2012,11 +2033,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(conversion) + SelectStatement(conversion) val query = select.query println(query) query shouldBe @@ -2028,19 +2050,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c2": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toInstant().toEpochMilli()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c3": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c4": { @@ -2097,11 +2128,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case function as script field" in { // 40 val select: ElasticSearchRequest = - SQLQuery(caseWhen) + SelectStatement(caseWhen) val query = select.query println(query) query shouldBe @@ -2113,7 +2145,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2150,11 +2185,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case with expression function as script field" in { val select: ElasticSearchRequest = - SQLQuery(caseWhenExpr) + SelectStatement(caseWhenExpr) val query = select.query println(query) query shouldBe @@ -2166,7 +2202,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2205,11 +2244,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle extract function as script field" in { val select: ElasticSearchRequest = - SQLQuery(extract) + SelectStatement(extract) val query = select.query println(query) query shouldBe @@ -2221,91 +2261,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2333,7 +2373,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle arithmetic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(arithmetic.replace("as group1", "")) + SelectStatement(arithmetic.replace("as group1", "")) val query = select.query println(query) query shouldBe @@ -2345,7 +2385,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2418,11 +2461,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("-", " - ") .replaceAll("==", " == ") .replaceAll("\\|\\|", " || ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle mathematic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(mathematical) + SelectStatement(mathematical) val query = select.query println(query) query shouldBe @@ -2434,7 +2478,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1)) > 100.0" | } | } | } @@ -2445,61 +2489,61 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Double.valueOf(Math.abs(param1)) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.ceil(param1))" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.floor(param1))" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1))" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.exp(param1))" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log(param1))" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log10(param1))" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.pow(param1, 3))" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "sign_identifier": { @@ -2511,43 +2555,43 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.cos(param1))" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.acos(param1))" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sin(param1))" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.asin(param1))" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.tan(param1))" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan(param1))" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan2(param1, 3.0))" | } | } | }, @@ -2592,7 +2636,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle string function as script field and condition" in { // 45 val select: ElasticSearchRequest = - SQLQuery(string) + SelectStatement(string) val query = select.query println(query) query shouldBe @@ -2633,7 +2677,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(3, ((String)param1).length()))" | } | }, | "tr": { @@ -2663,13 +2707,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(5, ((String)param1).length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(((String)param1).length() - (int)Math.min(3, ((String)param1).length()))" | } | }, | "rep": { @@ -2711,7 +2755,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("def_", "def _") .replaceAll("=_", " = _") .replaceAll(",_", ", _") - .replaceAll(",\\(", ", (") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") .replaceAll(":\\(", " : (") @@ -2748,7 +2791,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle top hits aggregation" in { val select: ElasticSearchRequest = - SQLQuery(topHits) + SelectStatement(topHits) val query = select.query println(query) query shouldBe @@ -2865,7 +2908,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle last day function" in { val select: ElasticSearchRequest = - SQLQuery(lastDay) + SelectStatement(lastDay) val query = select.query println(query) query shouldBe @@ -2877,7 +2920,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2928,11 +2974,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle all extractors" in { val select: ElasticSearchRequest = - SQLQuery(extractors) + SelectStatement(extractors) val query = select.query println(query) query shouldBe @@ -2944,91 +2991,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z'))); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3069,7 +3116,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = - SQLQuery(geoDistance) + SelectStatement(geoDistance) val query = select.query println(query) query shouldBe @@ -3084,7 +3131,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3108,7 +3155,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon) < 2000000.0" | } | } | }, @@ -3127,7 +3174,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3137,7 +3184,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3191,7 +3238,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle between with temporal" in { // 50 val select: ElasticSearchRequest = - SQLQuery(betweenTemporal) + SelectStatement(betweenTemporal) val query = select.query println(query) query shouldBe @@ -3276,7 +3323,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested of nested" in { val select: ElasticSearchRequest = - SQLQuery(nestedOfNested) + SelectStatement(nestedOfNested) val query = select.query println(query) query shouldBe @@ -3314,26 +3361,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "comments.replies.reply_author", - | "comments.replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.replies.reply_author", + | "comments.replies.reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3384,7 +3427,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle predicate with distinct nested" in { val select: ElasticSearchRequest = - SQLQuery(predicateWithDistinctNested) + SelectStatement(predicateWithDistinctNested) val query = select.query println(query) query shouldBe @@ -3409,13 +3452,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "replies.reply_author", - | "replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "replies.reply_author", + | "replies.reply_text" + | ] | } | } | } @@ -3434,13 +3475,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3494,7 +3533,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested without criteria" in { val select: ElasticSearchRequest = - SQLQuery(nestedWithoutCriteria) + SelectStatement(nestedWithoutCriteria) val query = select.query println(query) query shouldBe @@ -3509,7 +3548,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -3528,26 +3570,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "reply_author", - | "reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "reply_author", + | "reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "author", - | "comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "author", + | "comments" + | ] | } | } | } @@ -3559,8 +3597,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") - .replaceAll("\\s+", "") - .replaceAll("\\s+", "") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3594,11 +3630,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") .replaceAll("false:", "false : ") .replaceAll("DateTimeFormatter", " DateTimeFormatter") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "determine the aggregation context" in { val select: ElasticSearchRequest = - SQLQuery(determinationOfTheAggregationContext) + SelectStatement(determinationOfTheAggregationContext) val query = select.query println(query) query shouldBe @@ -3632,7 +3669,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with nested of nested context" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithNestedOfNestedContext) + SelectStatement(aggregationWithNestedOfNestedContext) val query = select.query println(query) query shouldBe @@ -3668,7 +3705,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle where filters according to scope" in { val select: ElasticSearchRequest = - SQLQuery(whereFiltersAccordingToScope) + SelectStatement(whereFiltersAccordingToScope) val query = select.query println(query) query shouldBe diff --git a/build.sbt b/build.sbt index 056a2bd6..7fe63779 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.14.3" +ThisBuild / version := "0.15.0" ThisBuild / scalaVersion := scala213 diff --git a/core/src/main/resources/softnetwork-elastic.conf b/core/src/main/resources/softnetwork-elastic.conf index 25f1644b..4b59eec4 100644 --- a/core/src/main/resources/softnetwork-elastic.conf +++ b/core/src/main/resources/softnetwork-elastic.conf @@ -39,4 +39,5 @@ elastic { latency-threshold = 1000.0 # Alert if average latency > 1000ms } } + } \ No newline at end of file diff --git a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala index a4b010e9..bcc562b6 100644 --- a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala @@ -44,8 +44,7 @@ case class ElasticConfig( discovery: DiscoveryConfig, connectionTimeout: Duration, socketTimeout: Duration, - metrics: MetricsConfig -) + metrics: MetricsConfig) object ElasticConfig extends StrictLogging { def apply(config: Config): ElasticConfig = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala index caa3325c..3bb04c7d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AggregateApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import java.time.temporal.Temporal import scala.annotation.tailrec @@ -37,7 +37,7 @@ trait AggregateApi[T <: AggregateResult] { * @return * a sequence of aggregated results */ - def aggregate(sqlQuery: SQLQuery)(implicit + def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[_root_.scala.collection.Seq[T]]] } @@ -57,7 +57,7 @@ trait SingleValueAggregateApi * a sequence of aggregated results */ override def aggregate( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit ec: ExecutionContext ): Future[ElasticResult[_root_.scala.collection.Seq[SingleValueAggregateResult]]] = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala index ccc76be6..3aad94bf 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/AliasApi.scala @@ -22,7 +22,9 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } -import com.google.gson.JsonParser +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode import scala.jdk.CollectionConverters._ import scala.util.Try @@ -60,7 +62,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * @param index * the index name * @param alias - * the alias name to add + * the alias to add * @return * ElasticSuccess(true) if added, ElasticFailure otherwise * @@ -78,7 +80,40 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * An index can have multiple aliases */ //format:on + @deprecated("Use addAlias(TableAlias) instead", "0.15.0") def addAlias(index: String, alias: String): ElasticResult[Boolean] = { + addAlias(TableAlias(index, alias)) + } + + //format:off + /** Add an alias to an index. + * + * This operation: + * 1. Validates the index and alias names 2. Checks that the index exists 3. Adds the alias + * + * @param alias + * the TableAlias to add + * @return + * ElasticSuccess(true) if added, ElasticFailure otherwise + * + * @example + * {{{ + * val alias = TableAlias(table = "my-index-2024", alias = "my-index-current") + * addAlias(alias) match { + * case ElasticSuccess(_) => println("Alias added") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + * + * @note + * An alias can point to multiple indexes (useful for searches) + * @note + * An index can have multiple aliases + */ + //format:on + def addAlias(alias: TableAlias): ElasticResult[Boolean] = { + val index = alias.table + val aliasName = alias.alias // Validation... validateIndexName(index) match { case Some(error) => @@ -92,7 +127,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case None => // OK } - validateAliasName(alias) match { + validateAliasName(aliasName) match { case Some(error) => return ElasticFailure( error.copy( @@ -104,7 +139,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case None => // OK } - if (index == alias) { + if (index == aliasName) { return ElasticFailure( ElasticError( message = s"Index and alias cannot have the same name: '$index'", @@ -115,7 +150,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => ) } - indexExists(index) match { + indexExists(index, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -129,15 +164,15 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case _ => // OK } - logger.debug(s"Adding alias '$alias' to index '$index'") + logger.debug(s"Adding alias '$aliasName' to index '$index'") - executeAddAlias(index, alias) match { + executeAddAlias(alias) match { case success @ ElasticSuccess(_) => - logger.info(s"βœ… Alias '$alias' successfully added to index '$index'") + logger.info(s"βœ… Alias '$aliasName' successfully added to index '$index'") success case failure @ ElasticFailure(error) => - logger.error(s"❌ Failed to add alias '$alias' to index '$index': ${error.message}") + logger.error(s"❌ Failed to add alias '$aliasName' to index '$index': ${error.message}") failure } } @@ -249,6 +284,10 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => success case failure @ ElasticFailure(error) => + if (error.statusCode.getOrElse(0) == 404) { + logger.info(s"βœ… Alias '$alias' does not exist") + return ElasticResult.success(false) + } logger.error(s"❌ Failed to check existence of alias '$alias': ${error.message}") failure } @@ -265,14 +304,14 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ //format:on - def getAliases(index: String): ElasticResult[Set[String]] = { + def getAliases(index: String): ElasticResult[Seq[TableAlias]] = { validateIndexName(index) match { case Some(error) => @@ -292,7 +331,7 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => // βœ… Extracting aliases from JSON ElasticResult.fromTry( Try { - new JsonParser().parse(jsonString).getAsJsonObject + mapper.readTree(jsonString) } ) match { case ElasticFailure(error) => @@ -301,32 +340,39 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => case ElasticSuccess(rootObj) => if (!rootObj.has(index)) { logger.warn(s"Index '$index' not found in response") - return ElasticResult.success(Set.empty[String]) + return ElasticResult.success(Seq.empty[TableAlias]) } - val indexObj = rootObj.getAsJsonObject(index) - if (indexObj == null) { - logger.warn(s"Index '$index' is null in response") - return ElasticResult.success(Set.empty[String]) - } + val root = rootObj.get(index) - val aliasesObj = indexObj.getAsJsonObject("aliases") - if (aliasesObj == null || aliasesObj.size() == 0) { + if (!root.has("aliases")) { logger.debug(s"No aliases found for index '$index'") - ElasticResult.success(Set.empty[String]) - } else { - val aliases = aliasesObj.entrySet().asScala.map(_.getKey).toSet - logger.debug( - s"Found ${aliases.size} alias(es) for index '$index': ${aliases.mkString(", ")}" - ) - ElasticResult.success(aliases) + return ElasticResult.success(Seq.empty[TableAlias]) } + + val aliasesNode = root.path("aliases") + val aliases: Map[String, JsonNode] = + if (aliasesNode != null && aliasesNode.isObject) { + aliasesNode + .properties() + .asScala + .map { entry => + val aliasName = entry.getKey + val aliasValue = entry.getValue + aliasName -> aliasValue + } + .toMap + } else { + Map.empty + } + + ElasticSuccess(aliases.toSeq.map(alias => TableAlias(index, alias._1, alias._2))) } } match { case success @ ElasticSuccess(aliases) => if (aliases.nonEmpty) logger.info( - s"βœ… Found ${aliases.size} alias(es) for index '$index': ${aliases.mkString(", ")}" + s"βœ… Found ${aliases.size} alias(es) for index '$index': ${aliases.map(_.alias).sorted.mkString(", ")}" ) else logger.info(s"βœ… No aliases found for index '$index'") @@ -437,11 +483,80 @@ trait AliasApi extends ElasticClientHelpers { _: IndicesApi => } } + //format:off + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * + * @example + * {{{ + * setAliases("my-index", Seq(TableAlias("my-index", "alias1"), TableAlias("my-index", "alias2"))) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + //format:on + def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = { + getAliases(index).flatMap { existingAliases => + val notIndexAliases = aliases.filter(_.table != index) + if (notIndexAliases.nonEmpty) { + return ElasticFailure( + ElasticError( + message = s"All aliases must belong to the index '$index': ${notIndexAliases + .map(_.alias) + .mkString(", ")}", + cause = None, + statusCode = Some(400), + operation = Some("setAliases") + ) + ) + } + + val existingAliasNames = existingAliases.map(_.alias).toSet + val aliasesMap = aliases.map(alias => alias.alias -> alias).toMap + + val toSet = aliasesMap.keys.toSet.diff(existingAliasNames) ++ aliasesMap.keys.toSet.intersect( + existingAliasNames + ) + val toRemove = existingAliasNames.diff(aliasesMap.keys.toSet) + + val setResults = toSet.map(alias => addAlias(aliasesMap(alias))) + val removeResults = toRemove.map(alias => removeAlias(index, alias)) + + val allResults = setResults ++ removeResults + + val failures = allResults.collect { case ElasticFailure(error) => error } + + if (failures.nonEmpty) { + ElasticFailure( + ElasticError( + message = + s"Failed to set aliases for index '$index': ${failures.map(_.message).mkString(", ")}", + cause = None, + statusCode = Some(500), + operation = Some("setAliases") + ) + ) + } else { + ElasticResult.success(true) + } + } + } + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== - private[client] def executeAddAlias(index: String, alias: String): ElasticResult[Boolean] + private[client] def executeAddAlias(alias: TableAlias): ElasticResult[Boolean] private[client] def executeRemoveAlias(index: String, alias: String): ElasticResult[Boolean] diff --git a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala index df9714f4..c396d50a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/BulkApi.scala @@ -23,12 +23,10 @@ import akka.stream.scaladsl.{Balance, Flow, GraphDSL, Merge, Sink, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.file._ import app.softnetwork.elastic.client.result.{ElasticResult, ElasticSuccess} - +import app.softnetwork.elastic.sql.query.{FileFormat, Unknown} +import app.softnetwork.elastic.sql.schema.sqlConfig import org.apache.hadoop.conf.Configuration -import org.json4s.DefaultFormats -import org.json4s.jackson.JsonMethods.parse - import java.time.LocalDate import java.time.format.DateTimeFormatter import scala.concurrent.{ExecutionContext, Future} @@ -83,7 +81,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromFile( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -128,7 +126,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromParquet( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, callbacks: BulkCallbacks = BulkCallbacks.default, bufferSize: Int = 500, hadoopConf: Option[Configuration] = None @@ -153,7 +151,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { def bulkFromJson( filePath: String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, callbacks: BulkCallbacks = BulkCallbacks.default, bufferSize: Int = 500, validateJson: Boolean = true, @@ -219,7 +217,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -324,7 +322,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -407,7 +405,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -602,7 +600,7 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { private def toBulkItem[D]( toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -610,13 +608,13 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { parentIdKey: Option[String], item: D )(implicit bulkOptions: BulkOptions): BulkItem = { - implicit val formats: DefaultFormats = org.json4s.DefaultFormats val document = toDocument(item) - val jsonMap = parse(document, useBigDecimalForDouble = false).extract[Map[String, Any]] + val jsonNode = mapper.readTree(document) + val jsonMap = mapper.convertValue(jsonNode, classOf[Map[String, Any]]) // extract id - val id = idKey.flatMap { i => - jsonMap.get(i).map(_.toString) + val id = idKey.map { keys => + keys.map(i => jsonMap.getOrElse(i, "").toString).mkString(sqlConfig.compositeKeySeparator) } // extract final index name @@ -628,8 +626,9 @@ trait BulkApi extends BulkTypes with ElasticClientHelpers { val index = suffixDateKey .flatMap { s => jsonMap.get(s).map { d => - val strDate = d.toString.substring(0, 10) - val date = LocalDate.parse(strDate, DateTimeFormatter.ofPattern("yyyy-MM-dd")) + val pattern = suffixDatePattern.getOrElse("yyyy-MM-dd") + val strDate = d.toString.substring(0, pattern.length) + val date = LocalDate.parse(strDate, DateTimeFormatter.ofPattern(pattern)) date.format( suffixDatePattern .map(DateTimeFormatter.ofPattern) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 2ccc0500..964a4967 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -17,6 +17,8 @@ package app.softnetwork.elastic.client import app.softnetwork.common.ClientCompanion +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper import com.typesafe.config.{Config, ConfigFactory} import org.json4s.jackson import org.json4s.jackson.Serialization @@ -45,6 +47,9 @@ trait ElasticClientApi with FlushApi with VersionApi with SerializationApi + with PipelineApi + with TemplateApi + with GatewayApi with ClientCompanion { protected def logger: Logger @@ -56,4 +61,6 @@ trait ElasticClientApi trait SerializationApi { implicit val serialization: Serialization.type = jackson.Serialization + val mapper: ObjectMapper = JacksonConfig.objectMapper + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 9168ecc8..a45c174a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -22,13 +22,21 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.{query, schema} +import app.softnetwork.elastic.sql.query.{ + DqlStatement, + SQLAggregation, + SelectStatement, + SingleSearch +} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} import com.typesafe.config.Config import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} import scala.concurrent.{ExecutionContext, Future} -import scala.language.implicitConversions +import scala.language.{dynamics, implicitConversions} import scala.reflect.ClassTag trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { @@ -80,8 +88,23 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * true if the index was created successfully, false otherwise */ - override def createIndex(index: String, settings: String): ElasticResult[Boolean] = - delegate.createIndex(index, settings) + override def createIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = + delegate.createIndex(index, settings, mappings, aliases) + + /** Get an index with the provided name. + * + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + override def getIndex(index: String): ElasticResult[Option[Index]] = + delegate.getIndex(index) /** Delete an index with the provided name. * @@ -128,9 +151,10 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = - delegate.reindex(sourceIndex, targetIndex, refresh) + delegate.reindex(sourceIndex, targetIndex, refresh, pipeline) /** Check if an index exists. * @@ -139,18 +163,96 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * true if the index exists, false otherwise */ - override def indexExists(index: String): ElasticResult[Boolean] = - delegate.indexExists(index) + override def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = + delegate.indexExists(index, pattern) + + /** Truncate an index by deleting all its documents. + * + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + override def truncateIndex(index: String): ElasticResult[Long] = + delegate.truncateIndex(index) + + /** Delete documents by query from an index. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + override def deleteByQuery(index: String, query: String, refresh: Boolean): ElasticResult[Long] = + delegate.deleteByQuery(index, query, refresh) + + override def isIndexClosed(index: String): ElasticResult[Boolean] = + delegate.isIndexClosed(index) + + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + override def updateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + delegate.updateByQuery(index, query, pipelineId, refresh) + + /** Insert documents by query into an index. + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit + system: ActorSystem + ): Future[ElasticResult[DmlResult]] = + delegate.insertByQuery(index, query, refresh) + + /** Load the schema for the provided index. + * + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + override def loadSchema(index: String): ElasticResult[Schema] = + delegate.loadSchema(index) override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = - delegate.executeCreateIndex(index, settings) + delegate.executeCreateIndex(index, settings, None, Nil) override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = delegate.executeDeleteIndex(index) + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + delegate.executeGetIndex(index) + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = delegate.executeCloseIndex(index) @@ -160,13 +262,39 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = - delegate.executeReindex(sourceIndex, targetIndex, refresh) + delegate.executeReindex(sourceIndex, targetIndex, refresh, pipeline) override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = delegate.executeIndexExists(index) + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = + delegate.executeDeleteByQuery(index, query, refresh) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + delegate.executeIsIndexClosed(index) + + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = + delegate.waitForShards(index, status, timeout) + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + delegate.executeUpdateByQuery(index, query, pipelineId, refresh) + // ==================== AliasApi ==================== /** Add an alias to an index. @@ -195,6 +323,31 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def addAlias(index: String, alias: String): ElasticResult[Boolean] = delegate.addAlias(index, alias) + /** Add an alias to an index. + * + * This operation: + * 1. Validates the index and alias names 2. Checks that the index exists 3. Adds the alias + * + * @param alias + * the TableAlias to add + * @return + * ElasticSuccess(true) if added, ElasticFailure otherwise + * @example + * {{{ + * val alias = TableAlias(table = "my-index-2024", alias = "my-index-current") + * addAlias(alias) match { + * case ElasticSuccess(_) => println("Alias added") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + * @note + * An alias can point to multiple indexes (useful for searches) + * @note + * An index can have multiple aliases + */ + override def addAlias(alias: TableAlias): ElasticResult[Boolean] = + delegate.addAlias(alias) + /** Remove an alias from an index. * * @param index @@ -244,13 +397,13 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ - override def getAliases(index: String): ElasticResult[Set[String]] = + override def getAliases(index: String): ElasticResult[Seq[TableAlias]] = delegate.getAliases(index) /** Atomic swap of an alias between two indexes. @@ -285,11 +438,32 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.swapAlias(oldIndex, newIndex, alias) + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * @example + * {{{ + * setAliases("my-index", Set("alias1", "alias2")) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + override def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = + delegate.setAliases(index, aliases) + override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = - delegate.executeAddAlias(index, alias) + delegate.executeAddAlias(alias) override private[client] def executeRemoveAlias( index: String, @@ -436,6 +610,31 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { ): ElasticResult[Boolean] = delegate.updateMapping(index, mapping, settings) + /** Migrate an existing index to a new mapping. + * + * Process: + * 1. Create temporary index with new mapping 2. Reindex data from original to temporary 3. + * Delete original index 4. Recreate original index with new mapping 5. Reindex data from + * temporary to original 6. Delete temporary index + */ + override private[client] def performMigration( + index: String, + tempIndex: String, + mapping: String, + settings: String, + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = + delegate.performMigration(index, tempIndex, mapping, settings, aliases) + + override private[client] def rollbackMigration( + index: String, + tempIndex: String, + originalMapping: String, + originalSettings: String, + originalAliases: Seq[TableAlias] + ): ElasticResult[Boolean] = + delegate.rollbackMigration(index, tempIndex, originalMapping, originalSettings, originalAliases) + override private[client] def executeSetMapping( index: String, mapping: String @@ -882,7 +1081,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a sequence of aggregated results */ - override def aggregate(sqlQuery: SQLQuery)(implicit + override def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[collection.Seq[SingleValueAggregateResult]]] = delegate.aggregate(sqlQuery) @@ -896,7 +1095,8 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * the Elasticsearch response */ - override def search(sql: SQLQuery): ElasticResult[ElasticResponse] = delegate.search(sql) + override def search(statement: DqlStatement): ElasticResult[ElasticResponse] = + delegate.search(statement) /** Search for documents / aggregations matching the Elasticsearch query. * @@ -941,9 +1141,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a Future containing the Elasticsearch response */ - override def searchAsync(sqlQuery: SQLQuery)(implicit + override def searchAsync(statement: DqlStatement)(implicit ec: ExecutionContext - ): Future[ElasticResult[ElasticResponse]] = delegate.searchAsync(sqlQuery) + ): Future[ElasticResult[ElasticResponse]] = delegate.searchAsync(statement) /** Asynchronous search for documents / aggregations matching the Elasticsearch query. * @@ -991,7 +1191,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * the entities matching the query */ override def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = delegate.searchAsUnchecked(sqlQuery) @@ -1047,7 +1247,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * a Future containing the entities */ - override def searchAsyncAsUnchecked[U](sqlQuery: SQLQuery)(implicit + override def searchAsyncAsUnchecked[U](sqlQuery: SelectStatement)(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats @@ -1103,7 +1303,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.multiSearchAsyncAs(elasticQueries, fieldAliases, aggregations) override def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats @@ -1123,8 +1323,8 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { delegate.multisearchWithInnerHits[U, I](elasticQueries, innerField) override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SQLSearchRequest - ): String = + sqlSearch: SingleSearch + )(implicit timestamp: Long): String = delegate.sqlSearchRequestToJsonQuery(sqlSearch) override private[client] def executeSingleSearch( @@ -1151,9 +1351,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SQLQuery, config: ScrollConfig)(implicit + override def scroll(statement: DqlStatement, config: ScrollConfig)(implicit system: ActorSystem - ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(sql, config) + ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = delegate.scroll(statement, config) /** Scroll and convert results into typed entities from an SQL query. * @@ -1175,7 +1375,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { * @return * - Source of tuples (T, ScrollMetrics) */ - override def scrollAsUnchecked[T](sql: SQLQuery, config: ScrollConfig)(implicit + override def scrollAsUnchecked[T](sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem, m: Manifest[T], formats: Formats @@ -1249,7 +1449,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -1311,7 +1511,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -1341,7 +1541,7 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String], - idKey: Option[String], + idKey: Option[Set[String]], suffixDateKey: Option[String], suffixDatePattern: Option[String], update: Option[Boolean], @@ -1397,4 +1597,202 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { /** Conversion BulkActionType -> BulkItem */ override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = delegate.actionToBulkItem(action.asInstanceOf) + + // ==================== PipelineApi (delegate) ==================== + + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def pipeline(sql: String): ElasticResult[Boolean] = + delegate.pipeline(sql) + + override private[client] def pipeline( + statement: query.PipelineStatement + ): ElasticResult[Boolean] = + delegate.pipeline(statement) + + override def createPipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = { + delegate.createPipeline(pipelineName, pipelineDefinition) + } + + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def updatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + delegate.updatePipeline(pipelineName, pipelineDefinition) + + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { + delegate.deletePipeline(pipelineName, ifExists = ifExists) + } + + override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { + delegate.getPipeline(pipelineName) + } + + override def loadPipeline(pipelineName: String): ElasticResult[schema.IngestPipeline] = + delegate.loadPipeline(pipelineName) + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreatePipeline(pipelineName, pipelineDefinition) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeletePipeline(pipelineName, ifExists = ifExists) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = + delegate.executeGetPipeline(pipelineName) + + // ==================== TemplateApi (delegate) ==================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + override def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.createTemplate(templateName, templateDefinition) + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + override def deleteTemplate(templateName: String, ifExists: Boolean): ElasticResult[Boolean] = + delegate.deleteTemplate(templateName, ifExists) + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + override def getTemplate(templateName: String): ElasticResult[Option[String]] = + delegate.getTemplate(templateName) + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + override def listTemplates(): ElasticResult[Map[String, String]] = + delegate.listTemplates() + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + override def templateExists(templateName: String): ElasticResult[Boolean] = + delegate.templateExists(templateName) + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreateComposableTemplate(templateName, templateDefinition) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteComposableTemplate(templateName, ifExists) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + delegate.executeGetComposableTemplate(templateName) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + delegate.executeListComposableTemplates() + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + delegate.executeComposableTemplateExists(templateName) + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + delegate.executeCreateLegacyTemplate(templateName, templateDefinition) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + delegate.executeDeleteLegacyTemplate(templateName, ifExists) + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = + delegate.executeGetLegacyTemplate(templateName) + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = + delegate.executeListLegacyTemplates() + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + delegate.executeLegacyTemplateExists(templateName) + + // ==================== Gateway (delegate) ==================== + + override def run(sql: String)(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = + delegate.run(sql) + + override def run(statement: query.Statement)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] = + delegate.run(statement) } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala index 2ce3a1b2..23f82066 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientHelpers.scala @@ -33,12 +33,13 @@ trait ElasticClientHelpers { * - No colon (:) except for system indexes * - Does not start with -, _, + * - Is not . or .. + * - Max length 255 characters * @param index * name of the index to validate * @return * Some(ElasticError) if invalid, None if valid */ - protected def validateIndexName(index: String): Option[ElasticError] = { + protected def validateIndexName(index: String, pattern: Boolean = false): Option[ElasticError] = { if (index == null || index.trim.isEmpty) { return Some( ElasticError( @@ -86,12 +87,11 @@ trait ElasticClientHelpers { ) } - val invalidChars = """[\\/*?"<>| ,#]""".r + val invalidChars = if (pattern) """[\\/?"<>| ,#]""".r else """[\\/*?"<>| ,#]""".r if (invalidChars.findFirstIn(trimmed).isDefined) { return Some( ElasticError( - message = - "Index name contains invalid characters: \\, /, *, ?, \", <, >, |, space, comma, #", + message = "Index name contains invalid characters: /, *, ?, \", <, >, |, space, comma, #", cause = None, statusCode = Some(400), operation = Some("validateIndexName") @@ -176,6 +176,26 @@ trait ElasticClientHelpers { validateJson("validateJsonSettings", settings) } + /** Validate the JSON mappings. + * @param mappings + * mappings in JSON format + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateJsonMappings(mappings: String): Option[ElasticError] = { + validateJson("validateJsonMappings", mappings) + } + + /** Validate the JSON pipeline definition. + * @param pipelineDefinition + * pipeline definition in JSON format + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateJsonPipeline(pipelineDefinition: String): Option[ElasticError] = { + validateJson("validateJsonPipeline", pipelineDefinition) + } + /** Validate the alias name. Aliases follow the same rules as indexes. * @param alias * alias name to validate @@ -188,13 +208,271 @@ trait ElasticClientHelpers { case Some(error) => Some( error.copy( - operation = Some("validateAliasName") + operation = Some("validateAliasName"), + message = error.message.replaceAll("Index", "Alias") + ) + ) + case None => None + } + } + + /** Validate the pipeline name. Pipelines do not follow the same rules as indexes. only + * alphanumeric characters, points (.), underscores (_), hyphens (-) and at-signs (@) are allowed + * @param pipelineName + * pipeline name to validate + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validatePipelineName(pipelineName: String): Option[ElasticError] = { + if (pipelineName == null || pipelineName.trim.isEmpty) { + return Some( + ElasticError( + message = "Pipeline name cannot be empty", + cause = None, + statusCode = Some(400), + operation = Some("validatePipelineName") + ) + ) + } + + val trimmed = pipelineName.trim + + val pattern = "^[a-zA-Z0-9._\\-@]+$".r + + if (!pattern.matches(trimmed)) { + return Some( + ElasticError( + message = + "Pipeline name contains invalid characters: only alphanumeric characters, points (.), underscores (_), hyphens (-) and at-signs (@) are allowed", + cause = None, + statusCode = Some(400), + operation = Some("validatePipelineName") + ) + ) + } + + None + } + + /** Validate the template name. Index templates follow the same rules as indexes: + * - Not empty + * - Lowercase only + * - No characters: \, /, *, ?, ", <, >, |, space, comma, # + * - No colon (:) + * - Does not start with -, _, + + * - Is not . or .. + * - Max length 255 characters + * - Can contain wildcards (* and ?) in index_patterns but not in the template name itself + * + * @param templateName + * name of the template to validate + * @return + * Some(ElasticError) if invalid, None if valid + */ + protected def validateTemplateName(templateName: String): Option[ElasticError] = { + validateIndexName(templateName) match { + case Some(error) => + Some( + error.copy( + operation = Some("validateTemplateName"), + message = error.message.replaceAll("Index", "Template") ) ) case None => None } } + /** Validate composable template JSON definition (ES 7.8+) */ + protected def validateJsonComposableTemplate( + templateDefinition: String + ): Option[result.ElasticError] = { + if (templateDefinition == null || templateDefinition.trim.isEmpty) { + return Some( + result.ElasticError( + message = "Template definition cannot be null or empty", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + try { + import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + import com.fasterxml.jackson.databind.node.ObjectNode + + val mapper = new ObjectMapper() + val rootNode: JsonNode = mapper.readTree(templateDefinition) + + if (!rootNode.isObject) { + return Some( + result.ElasticError( + message = "Template definition must be a JSON object", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + val objectNode = rootNode.asInstanceOf[ObjectNode] + + // Composable templates require 'index_patterns' + if (!objectNode.has("index_patterns")) { + return Some( + result.ElasticError( + message = "Composable template must contain 'index_patterns' field", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + val indexPatternsNode = objectNode.get("index_patterns") + if (!indexPatternsNode.isArray || indexPatternsNode.size() == 0) { + return Some( + result.ElasticError( + message = "'index_patterns' must be a non-empty array", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate") + ) + ) + } + + // Valid fields for composable templates + val validFields = Set( + "index_patterns", + "template", // Contains settings, mappings, aliases + "priority", // Replaces 'order' + "version", + "composed_of", // Component templates + "_meta", + "data_stream", // For data streams + "allow_auto_create" + ) + + import scala.jdk.CollectionConverters._ + val templateFields = objectNode.fieldNames().asScala.toSet + val invalidFields = templateFields -- validFields + + if (invalidFields.nonEmpty) { + logger.warn( + s"⚠️ Composable template contains potentially invalid fields: ${invalidFields.mkString(", ")}" + ) + } + + None + + } catch { + case e: com.fasterxml.jackson.core.JsonParseException => + Some( + result.ElasticError( + message = s"Invalid JSON syntax: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate"), + cause = Some(e) + ) + ) + case e: Exception => + Some( + result.ElasticError( + message = s"Invalid composable template definition: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonComposableTemplate"), + cause = Some(e) + ) + ) + } + } + + /** Validate legacy template JSON definition (ES < 7.8) */ + protected def validateJsonLegacyTemplate( + templateDefinition: String + ): Option[result.ElasticError] = { + if (templateDefinition == null || templateDefinition.trim.isEmpty) { + return Some( + result.ElasticError( + message = "Template definition cannot be null or empty", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + try { + import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} + import com.fasterxml.jackson.databind.node.ObjectNode + + val mapper = new ObjectMapper() + val rootNode: JsonNode = mapper.readTree(templateDefinition) + + if (!rootNode.isObject) { + return Some( + result.ElasticError( + message = "Template definition must be a JSON object", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + val objectNode = rootNode.asInstanceOf[ObjectNode] + + // Legacy templates require 'index_patterns' or 'template' + if (!objectNode.has("index_patterns") && !objectNode.has("template")) { + return Some( + result.ElasticError( + message = "Legacy template must contain 'index_patterns' or 'template' field", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate") + ) + ) + } + + // Valid fields for legacy templates + val validFields = Set( + "index_patterns", + "template", // Old name for index_patterns + "settings", + "mappings", + "aliases", + "order", + "version", + "_meta" + ) + + import scala.jdk.CollectionConverters._ + val templateFields = objectNode.fieldNames().asScala.toSet + val invalidFields = templateFields -- validFields + + if (invalidFields.nonEmpty) { + logger.warn( + s"⚠️ Legacy template contains potentially invalid fields: ${invalidFields.mkString(", ")}" + ) + } + + None + + } catch { + case e: com.fasterxml.jackson.core.JsonParseException => + Some( + result.ElasticError( + message = s"Invalid JSON syntax: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate"), + cause = Some(e) + ) + ) + case e: Exception => + Some( + result.ElasticError( + message = s"Invalid legacy template definition: ${e.getMessage}", + statusCode = Some(400), + operation = Some("validateJsonLegacyTemplate"), + cause = Some(e) + ) + ) + } + } + /** Logger une erreur avec le niveau appropriΓ© selon le status code. */ protected def logError( operation: String, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala index 8ece3037..df7dd169 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticConversion.scala @@ -64,7 +64,10 @@ trait ElasticConversion { fieldAliases: Map[String, String], aggregations: Map[String, ClientAggregation] ): Try[Seq[Map[String, Any]]] = { - val json = mapper.readTree(results) + var json = mapper.readTree(results) + if (json.has("responses")) { + json = json.get("responses") + } // Check if it's a multi-search response (array of responses) if (json.isArray) { parseMultiSearchResponse(json, fieldAliases, aggregations) @@ -177,18 +180,44 @@ trait ElasticConversion { extractAllTopHits(aggs, fieldAliases, aggregations), aggregations ) - hits.map { hit => - val source = extractSource(hit, fieldAliases) + parseSimpleHits(hits, fieldAliases).map { row => + globalMetrics ++ allTopHits ++ row + } + /*hits.map { hit => + var source = extractSource(hit, fieldAliases) + fieldAliases.foreach(entry => { + val key = entry._1 + if (!source.contains(key)) { + findKeyValue(key, source) match { + case Some(value) => source += (entry._2 -> value) + case None => + } + } + }) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) - globalMetrics ++ allTopHits ++ source ++ metadata ++ innerHits - } + val fieldsNode = Option(hit.path("fields")) + .filter(!_.isMissingNode) + val fields = fieldsNode + .map(jsonNodeToMap(_, fieldAliases)) + .getOrElse(Map.empty) + globalMetrics ++ allTopHits ++ source ++ metadata ++ innerHits ++ fields + }*/ case _ => Seq.empty } } + def findKeyValue(path: String, map: Map[String, Any]): Option[Any] = { + val keys = path.split("\\.") + keys.foldLeft(Option(map): Option[Any]) { + case (Some(m: Map[_, _]), key) => + m.asInstanceOf[Map[String, Any]].get(key) + case _ => None + } + } + /** Parse simple hits (without aggregations) */ def parseSimpleHits( @@ -196,10 +225,23 @@ trait ElasticConversion { fieldAliases: Map[String, String] ): Seq[Map[String, Any]] = { hits.map { hit => - val source = extractSource(hit, fieldAliases) + var source = extractSource(hit, fieldAliases) + fieldAliases.foreach(entry => { + if (!source.contains(entry._2)) { + findKeyValue(entry._1, source) match { + case Some(value) => source += (entry._2 -> value) + case None => + } + } + }) val metadata = extractHitMetadata(hit) val innerHits = extractInnerHits(hit, fieldAliases) - source ++ metadata ++ innerHits + val fieldsNode = Option(hit.path("fields")) + .filter(!_.isMissingNode) + val fields = fieldsNode + .map(jsonNodeToMap(_, fieldAliases)) + .getOrElse(Map.empty) + source ++ metadata ++ innerHits ++ fields } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index 49fb2a41..24e9ad72 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -21,6 +21,7 @@ package app.softnetwork.elastic.client object ElasticsearchVersion { /** Parse Elasticsearch version string (e.g., "7.10.2", "8.11.0") + * * @return * (major, minor, patch) */ @@ -58,6 +59,27 @@ object ElasticsearchVersion { } } + /** Check if version is <= target version + */ + def isAtMost( + version: String, + targetMajor: Int, + targetMinor: Int = 0, + targetPatch: Int = 0 + ): Boolean = { + val (major, minor, patch) = parse(version) + + if (major < targetMajor) true + else if (major > targetMajor) false + else { // major == targetMajor + if (minor < targetMinor) true + else if (minor > targetMinor) false + else { // minor == targetMinor + patch <= targetPatch + } + } + } + /** Check if PIT is supported (ES >= 7.10) */ def supportsPit(version: String): Boolean = { @@ -67,6 +89,64 @@ object ElasticsearchVersion { /** Check if version is ES 8+ */ def isEs8OrHigher(version: String): Boolean = { - isAtLeast(version, 8, 0) + isAtLeast(version, 8) + } + + /** Check if version is ES 7+ + */ + def isEs7OrHigher(version: String): Boolean = { + isAtLeast(version, 7) + } + + /** Check if version is ES 6 + */ + def isEs6(version: String): Boolean = { + val (major, _, _) = parse(version) + major == 6 + } + + /** Check if version is ES 7 + */ + def isEs7(version: String): Boolean = { + val (major, _, _) = parse(version) + major == 7 + } + + /** Check if Data Streams are supported (ES >= 7.9) + */ + def supportsDataStreams(version: String): Boolean = { + isAtLeast(version, 7, 9) + } + + /** Check if Composable Templates are supported (ES >= 7.8) + */ + def supportsComposableTemplates(version: String): Boolean = { + isAtLeast(version, 7, 8) + } + + /** Check if Searchable Snapshots are supported (ES >= 7.10)} + */ + def supportsSearchableSnapshots(version: String): Boolean = { + isAtLeast(version, 7, 10) + } + + /** Check if Elasticsearch version requires _doc type wrapper in mappings + * + * ES 6.x requires mappings to be wrapped in a type name (e.g., "_doc") ES 7.x removed mapping + * types + * + * @param version + * the Elasticsearch version (e.g., "6.8.0") + * @return + * true if _doc wrapper is required + */ + def requiresDocTypeWrapper(version: String): Boolean = { + !isAtLeast(version, 6, 8) + } + + /** Check if deletion by query on closed indices is supported (ES >= 7.5) + */ + def supportsDeletionByQueryOnClosedIndices(version: String): Boolean = { + isAtLeast(version, 7, 5) } } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala new file mode 100644 index 00000000..69b8f80c --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -0,0 +1,1284 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ + DdlResult, + DmlResult, + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess, + PipelineResult, + QueryResult, + QueryRows, + QueryStream, + QueryStructured, + SQLResult, + TableResult +} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + AlterTable, + CopyInto, + CreatePipeline, + CreateTable, + DdlStatement, + Delete, + DescribePipeline, + DescribeTable, + DmlStatement, + DqlStatement, + DropTable, + Insert, + MultiSearch, + PipelineStatement, + SelectStatement, + ShowCreatePipeline, + ShowCreateTable, + ShowPipeline, + ShowTable, + SingleSearch, + Statement, + TableStatement, + TruncateTable, + Update +} +import app.softnetwork.elastic.sql.schema.{ + Impossible, + IngestPipeline, + IngestPipelineType, + PipelineDiff, + Safe, + Table, + TableDiff, + UnsafeReindex +} +import app.softnetwork.elastic.sql.serialization._ +import org.slf4j.Logger + +import scala.concurrent.{ExecutionContext, Future} + +trait Executor[T <: Statement] { + def execute(statement: T)(implicit + system: ActorSystem + ): Future[ElasticResult[QueryResult]] +} + +class DqlExecutor(api: ScrollApi with SearchApi, logger: Logger) extends Executor[DqlStatement] { + + override def execute( + statement: DqlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + + implicit val ec: ExecutionContext = system.dispatcher + + statement match { + + // ============================ + // SELECT ... (AST SQL) + // ============================ + case select: SelectStatement => + select.statement match { + + case Some(single: SingleSearch) => + execute(single.copy(score = select.score)) + + case Some(multiple: MultiSearch) => + execute(multiple) + + case None => + val error = ElasticError( + message = s"SELECT statement could not be translated into a search query: $statement", + statusCode = Some(400), + operation = Some("dql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + + // ============================ + // SingleSearch β†’ SCROLL + // ============================ + case single: SingleSearch => + single.limit match { + case Some(l) if l.offset.map(_.offset).getOrElse(0) > 0 => + logger.info(s"β–Ά Executing classic search on index ${single.from.tables.mkString(",")}") + api.searchAsync(single) map { + case ElasticSuccess(results) => + logger.info(s"βœ… Search returned ${results.results.size} hits.") + ElasticSuccess(QueryStructured(results)) + case ElasticFailure(err) => + ElasticFailure(err.copy(operation = Some("dql"))) + } + case _ => + logger.info(s"β–Ά Executing scroll search on index ${single.from.tables.mkString(",")}") + Future.successful( + ElasticSuccess(QueryStream(api.scroll(single))) + ) + } + + // ============================ + // MultiSearch β†’ searchAsync + // ============================ + case multiple: MultiSearch => + logger.info(s"β–Ά Executing multi-search on ${multiple.requests.size} indices") + api.searchAsync(multiple).map { + case ElasticSuccess(results) => + logger.info(s"βœ… Multi-search returned ${results.results.size} hits.") + ElasticSuccess(QueryStructured(results)) + + case ElasticFailure(err) => + ElasticFailure(err.copy(operation = Some("dql"))) + } + + // ============================ + // Unsupported DQL + // ============================ + case _ => + val error = ElasticError( + message = s"Unsupported DQL statement: $statement", + statusCode = Some(400), + operation = Some("dql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } +} + +class DmlExecutor(api: IndicesApi, logger: Logger) extends Executor[DmlStatement] { + override def execute( + statement: DmlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + statement match { + case delete: Delete => + api.deleteByQuery(delete.table.name, delete.sql) match { + case ElasticSuccess(count) => + logger.info(s"βœ… Deleted $count documents from ${delete.table.name}.") + Future.successful(ElasticResult.success(DmlResult(deleted = count))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case update: Update => + api.updateByQuery(update.table, update.sql) match { + case ElasticSuccess(count) => + logger.info(s"βœ… Updated $count documents in ${update.table}.") + Future.successful(ElasticResult.success(DmlResult(updated = count))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case insert: Insert => + api.insertByQuery(insert.table, insert.sql).map { + case success @ ElasticSuccess(res) => + logger.info(s"βœ… Inserted ${res.inserted} documents into ${insert.table}.") + success + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } + case copy: CopyInto => + api + .copyInto( + copy.source, + copy.targetTable, + doUpdate = copy.onConflict.exists(_.doUpdate), + fileFormat = copy.fileFormat + ) + .map { + case success @ ElasticSuccess(res) => + logger.info( + s"βœ… Copied ${res.inserted} documents into ${copy.targetTable} from ${copy.source}." + ) + success + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } + case _ => + // unsupported DML statement + val error = + ElasticError( + message = s"Unsupported DML statement: $statement", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } +} + +trait DdlExecutor[T <: DdlStatement] extends Executor[T] + +class PipelineExecutor(api: PipelineApi, logger: Logger) extends DdlExecutor[PipelineStatement] { + override def execute( + statement: PipelineStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + statement match { + case show: ShowPipeline => + // handle SHOW PIPELINE statement + api.loadPipeline(show.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"βœ… Retrieved pipeline ${show.name}.") + Future.successful(ElasticResult.success(PipelineResult(pipeline))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case showCreate: ShowCreatePipeline => + // handle SHOW CREATE PIPELINE statement + api.loadPipeline(showCreate.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"βœ… Retrieved pipeline ${showCreate.name}.") + Future.successful( + ElasticResult.success(SQLResult(pipeline.sql)) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case describe: DescribePipeline => + // handle DESCRIBE PIPELINE statement + api.loadPipeline(describe.name) match { + case ElasticSuccess(pipeline) => + logger.info(s"βœ… Retrieved pipeline ${describe.name}.") + Future.successful( + ElasticResult.success(QueryRows(pipeline.processors.map(_.properties))) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + case _ => + // handle PIPELINE statement + api.pipeline(statement) match { + case ElasticSuccess(result) => + logger.info(s"βœ… Executed pipeline statement: $statement.") + Future.successful(ElasticResult.success(DdlResult(result))) + case ElasticFailure(elasticError) => + logger.error( + s"❌ Error executing pipeline statement: $statement. ${elasticError.message}" + ) + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("pipeline")) + ) + ) + } + } + } +} + +class TableExecutor( + api: IndicesApi with PipelineApi with TemplateApi with SettingsApi with MappingApi with AliasApi, + logger: Logger +) extends DdlExecutor[TableStatement] { + override def execute( + statement: TableStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // handle TABLE statement + statement match { + // handle SHOW TABLE statement + case show: ShowTable => + api.loadSchema(show.table) match { + case ElasticSuccess(schema) => + logger.info(s"βœ… Retrieved schema for index ${show.table}.") + Future.successful(ElasticResult.success(TableResult(schema))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case showCreate: ShowCreateTable => + api.loadSchema(showCreate.table) match { + case ElasticSuccess(schema) => + logger.info(s"βœ… Retrieved schema for index ${showCreate.table}.") + Future.successful( + ElasticResult.success(SQLResult(schema.sql)) + ) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + // handle DESCRIBE TABLE statement + case describe: DescribeTable => + api.loadSchema(describe.table) match { + case ElasticSuccess(schema) => + logger.info(s"βœ… Retrieved schema for index ${describe.table}.") + Future.successful(ElasticResult.success(QueryRows(schema.columns.flatMap(_.asMap)))) + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + // handle CREATE TABLE statement + case create: CreateTable => + val ifNotExists = create.ifNotExists + val orReplace = create.orReplace + // will create template if partitioned + val partitioned = create.partitioned + val indexName = create.table + val single: Option[SingleSearch] = create.ddl match { + case Left(dql) => + dql match { + case s: SingleSearch => Some(s) + case _ => + val error = + ElasticError( + message = s"Only single search DQL supported in CREATE TABLE.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case Right(_) => None + } + + // check if index exists + api.indexExists(indexName, pattern = false) match { + // 1) Index exists + IF NOT EXISTS β†’ skip creation, DdlResult(false) + case ElasticSuccess(true) if ifNotExists => + // skip creation + logger.info(s"⚠️ Index $indexName already exists, skipping creation.") + Future.successful(ElasticSuccess(DdlResult(false))) + + // 2) Index exists + no OR REPLACE specified β†’ error + case ElasticSuccess(true) if !orReplace => + val error = + ElasticError( + message = s"Index $indexName already exists.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + + // 3) Index exists + OR REPLACE β†’ remplacement + case ElasticSuccess(true) => + // proceed with replacement + logger.info(s"♻️ Replacing index $indexName.") + replaceExistingIndex(indexName, create, partitioned, single) + + // 4) Index not exists β†’ creation + case ElasticSuccess(false) => + // proceed with creation + logger.info(s"βœ… Creating index $indexName.") + createNonExistentIndex(indexName, create, partitioned, single) + + // 5) Error on indexExists + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + + // handle ALTER TABLE statement + case alter: AlterTable => + val ifExists = alter.ifExists + val indexName = alter.table + + // check if index exists + api.indexExists(indexName, pattern = false) match { + // 1) Index does not exist + IF EXISTS β†’ skip alteration, DdlResult(false) + case ElasticSuccess(false) if ifExists => + // skip alteration + logger.info(s"⚠️ Index $indexName does not exist, skipping alteration.") + Future.successful(ElasticSuccess(DdlResult(false))) + + // 2) Index does not exists + no IF EXISTS β†’ error + case ElasticSuccess(false) => + val error = + ElasticError( + message = s"Index $indexName does not exist.", + statusCode = Some(404), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + + // 3) Index exists β†’ alteration + case ElasticSuccess(true) => + // proceed with alteration + logger.info(s"♻️ Alter index $indexName.") + alterExistingIndex(indexName, alter) + + // 5) Error on indexExists + case ElasticFailure(elasticError) => + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + + // handle TRUNCATE TABLE statement + case truncate: TruncateTable => + // handle TRUNCATE TABLE statement + val indexName = truncate.table + api.truncateIndex(indexName) match { + case ElasticSuccess(count) => + // index truncated successfully + logger.info(s"βœ… Index $indexName truncated successfully ($count documents deleted).") + Future.successful(ElasticResult.success(DdlResult(true))) + case ElasticFailure(elasticError) => + // error truncating index + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("truncate")) + ) + ) + } + + // handle DROP TABLE statement + case drop: DropTable => + // handle DROP TABLE statement + val indexName = drop.table + // check if index exists - pattern indices not supported for drop + api.indexExists(indexName, pattern = false) match { + case ElasticSuccess(false) if drop.ifExists => + // index does not exist and IF EXISTS specified, skip deletion + logger.info(s"⚠️ Index $indexName does not exist, skipping deletion.") + Future.successful(ElasticResult.success(DdlResult(false))) + case ElasticSuccess(false) if !drop.ifExists => + // index does not exist and no IF EXISTS specified, return error + val error = + ElasticError( + message = s"Index $indexName does not exist.", + statusCode = Some(404), + operation = Some("drop") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + case ElasticSuccess(true) => + // proceed with deletion + api.deleteIndex(indexName) match { + case ElasticSuccess(true) => + // index deleted successfully + logger.info(s"βœ… Index $indexName deleted successfully.") + Future.successful(ElasticResult.success(DdlResult(true))) + case ElasticSuccess(false) => + // index deletion failed + logger.warn(s"⚠️ Index $indexName could not be deleted.") + Future.successful(ElasticResult.success(DdlResult(false))) + case ElasticFailure(elasticError) => + // error deleting index + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("drop")) + ) + ) + } + case ElasticFailure(elasticError) => + // error checking index existence + Future.successful(ElasticFailure(elasticError.copy(operation = Some("drop")))) + } + + // handle unsupported table ddl statement + case _ => + // unsupported table ddl statement + val error = + ElasticError( + message = s"Unsupported table DDL statement: $statement", + statusCode = Some(400), + operation = Some("table") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + + /* Alter existing index + */ + private def alterExistingIndex( + indexName: String, + alter: AlterTable + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + // index exists and REPLACE specified, proceed with replacement + logger.info(s"♻️ Altering index $indexName.") + // load existing index schema + api.loadSchema(indexName) match { + case ElasticSuccess(schema) => + logger.info( + s"πŸ”„ Merging existing index $indexName DDL with new DDL." + ) + val updatedTable: Table = schema.merge(alter.statements) + + // load default pipeline diff if needed + val defaultPipelineDiff: Option[List[PipelineDiff]] = + loadTablePipelineDiff( + updatedTable, + IngestPipelineType.Default, + updatedTable.defaultPipelineName, + schema.defaultPipelineName + ) match { + case ElasticSuccess(result) => + result + case ElasticFailure(elasticError) => + return Future.successful(ElasticFailure(elasticError)) + } + + // load final pipeline diff if needed + val finalPipelineDiff: Option[List[PipelineDiff]] = + loadTablePipelineDiff( + updatedTable, + IngestPipelineType.Final, + updatedTable.finalPipelineName, + schema.finalPipelineName + ) match { + case ElasticSuccess(result) => + result + case ElasticFailure(elasticError) => + return Future.successful(ElasticFailure(elasticError)) + } + + // compute diff + var diff = schema.diff(updatedTable) + + // handle pipeline diffs + defaultPipelineDiff match { + case Some(pipelineDiffs) => + diff = diff.copy(pipeline = + diff.pipeline.filterNot(_.pipelineType == IngestPipelineType.Default) ++ pipelineDiffs + ) + if (pipelineDiffs.nonEmpty) { + logger.warn( + s"πŸ”„ Default ingesting pipeline for index $indexName has changes: $pipelineDiffs" + ) + logger.warn( + s"""⚠️ Default Pipeline may be used by multiple indexes. + | Modifying it may impact other indexes. + | Consider creating a dedicated pipeline if isolation is required.""".stripMargin + ) + } + case _ => + } + finalPipelineDiff match { + case Some(pipelineDiffs) => + diff = diff.copy(pipeline = + diff.pipeline.filterNot(_.pipelineType == IngestPipelineType.Final) ++ pipelineDiffs + ) + if (pipelineDiffs.nonEmpty) { + logger.warn( + s"πŸ”„ Final ingesting pipeline for index $indexName has changes: $pipelineDiffs" + ) + logger.warn( + s"""⚠️ Final Pipeline may be used by multiple indexes. + | Modifying it may impact other indexes. + | Consider creating a dedicated pipeline if isolation is required.""".stripMargin + ) + } + case _ => + } + + if (diff.isEmpty) { + logger.info( + s"⚠️ No changes detected for index $indexName, skipping update." + ) + Future.successful(ElasticResult.success(DdlResult(false))) + } else { + logger.info(s"πŸ”„ Updating index $indexName with changes: $diff") + diff.safety match { + + // ------------------------------------------------------------ + // SAFE β†’ mise Γ  jour en place + // ------------------------------------------------------------ + case Safe => + logger.info(s"πŸ”§ Applying SAFE ALTER TABLE changes on $indexName.") + applyUpdatesInPlace(indexName, updatedTable, diff) match { + case Right(result) => Future.successful(ElasticSuccess(DdlResult(result))) + case Left(error) => + Future.successful(ElasticFailure(error.copy(operation = Some("schema")))) + } + + // ------------------------------------------------------------ + // UNSAFE β†’ reindex + // ------------------------------------------------------------ + case UnsafeReindex => + logger.warn(s"⚠️ ALTER TABLE requires REINDEX for $indexName.") + migrateToNewSchema( + indexName, + schema, + updatedTable.copy( + settings = schema.settings // remove settings that cannot be copied + - "uuid" + - "creation_date" + - "provided_name" + - "version" + - "default_pipeline" + - "final_pipeline" + ), + diff + ) match { + case Right(result) => Future.successful(ElasticSuccess(DdlResult(result))) + case Left(error) => + Future.successful(ElasticFailure(error.copy(operation = Some("schema")))) + } + + // ------------------------------------------------------------ + // IMPOSSIBLE + // ------------------------------------------------------------ + case Impossible => + val error = ElasticError( + message = s"ALTER TABLE cannot be applied to index $indexName: $diff", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + + case ElasticFailure(elasticError) => + // error retrieving existing index info + Future.successful( + ElasticFailure(elasticError.copy(operation = Some("schema"))) + ) + } + } + + private def replaceExistingIndex( + indexName: String, + create: CreateTable, + partitioned: Boolean, + single: Option[SingleSearch] + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + + logger.info(s"♻️ Replacing existing index $indexName.") + + // 1. Delete existing index + api.deleteIndex(indexName) match { + case ElasticFailure(err) => + logger.error(s"❌ Failed to delete index $indexName: ${err.message}") + return Future.successful(ElasticFailure(err.copy(operation = Some("schema")))) + + case ElasticSuccess(false) => + logger.warn(s"⚠️ Index $indexName could not be deleted.") + return Future.successful(ElasticSuccess(DdlResult(false))) + + case ElasticSuccess(true) => + logger.info(s"πŸ—‘οΈ Index $indexName deleted successfully.") + } + + // 2. Recreate index using the same logic as createNonExistentIndex + createNonExistentIndex(indexName, create, partitioned, single) + } + + /* Create index / template for non-existent index + */ + private def createNonExistentIndex( + indexName: String, + create: CreateTable, + partitioned: Boolean, + single: Option[SingleSearch] + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + // index does not exist, proceed with creation + logger.info(s"βœ… Creating index $indexName.") + var table: Table = + single match { + case Some(single) => + single.from.tables.map(_.name).toSet.headOption match { + case Some(from) => + api.loadSchema(from) match { + case ElasticSuccess(fromSchema) => + // we update the schema based on the DQL select clause + fromSchema + .mergeWithSearch(single) + .copy( + name = indexName, // set index name + settings = fromSchema.settings // remove settings that cannot be copied + - "uuid" + - "creation_date" + - "provided_name" + - "version" + - "default_pipeline" + - "final_pipeline" + ) + case ElasticFailure(elasticError) => + val error = ElasticError( + message = + s"Error retrieving source schema $from for DQL in CREATE TABLE: ${elasticError.message}", + statusCode = elasticError.statusCode, + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case _ => + val error = + ElasticError( + message = s"Source index not specified for DQL in CREATE TABLE.", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Future.successful(ElasticFailure(error)) + } + case _ => + create.schema + } + + // create index pipeline(s) if needed + table.defaultPipeline match { + case pipeline if pipeline.processors.nonEmpty => + createIndexPipeline( + indexName, + CreatePipeline( + pipeline.name, + pipeline.pipelineType, + ifNotExists = true, + orReplace = false, + pipeline.processors + ) + ) match { + case ElasticSuccess(_) => + table = table.setDefaultPipelineName(pipelineName = pipeline.name) + case ElasticFailure(elasticError) => + return Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case _ => + } + + table.finalPipeline match { + case pipeline if pipeline.processors.nonEmpty => + createIndexPipeline( + indexName, + CreatePipeline( + pipeline.name, + pipeline.pipelineType, + ifNotExists = true, + orReplace = false, + pipeline.processors + ) + ) match { + case ElasticSuccess(_) => + table = table.setFinalPipelineName(pipelineName = pipeline.name) + case ElasticFailure(elasticError) => + return Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + case _ => + } + + // create index / template + createIndexOrTemplate(table, partitioned = partitioned) match { + case ElasticSuccess(_) => + // index / template created successfully + single match { + case Some(single) => + // populate index based on DQL + logger.info( + s"🚚 Populating index $indexName based on DQL." + ) + val query = + s"""INSERT INTO ${table.name} AS ${single.sql} ON CONFLICT DO NOTHING""" + val result = api.insertByQuery( + index = indexName, + query = query + ) + result.map { + case ElasticSuccess(res) => + logger.info( + s"βœ… Index $indexName populated successfully (${res.inserted} documents indexed)." + ) + ElasticResult.success(res) + case ElasticFailure(elasticError) => + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + } + case _ => + // no population needed + Future.successful(ElasticResult.success(DdlResult(true))) + } + case ElasticFailure(elasticError) => + // error creating index / template + Future.successful( + ElasticFailure( + elasticError.copy(operation = Some("schema")) + ) + ) + } + } + + /* Create ingesting pipeline for index + */ + private def createIndexPipeline( + indexName: String, + statement: CreatePipeline + ): ElasticResult[Boolean] = { + logger.info( + s"πŸ”§ Creating ${statement.pipelineType.name} ingesting pipeline ${statement.name} for index $indexName." + ) + api.pipeline(statement) match { + case success @ ElasticSuccess(true) => + logger.info(s"βœ… Pipeline ${statement.name} created successfully.") + success + case ElasticSuccess(_) => + // pipeline creation failed + val error = + ElasticError( + message = s"Failed to create pipeline ${statement.name}.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(_) => + failure + } + + } + + /* Create index or template based on partitioning + */ + private def createIndexOrTemplate(table: Table, partitioned: Boolean): ElasticResult[Boolean] = { + val indexName = table.name + if (partitioned) { + // create index template + api.createTemplate(indexName, table.indexTemplate) match { + case success @ ElasticSuccess(true) => + logger.info(s"βœ… Template $indexName created successfully.") + success + case ElasticSuccess(_) => + // template creation failed + val error = + ElasticError( + message = s"Failed to create template $indexName.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(error) => + logger.error(s"❌ ${error.message}") + failure + } + } else { + // create index + api.createIndex( + index = indexName, + settings = table.indexSettings, + mappings = Some(table.indexMappings), + aliases = table.indexAliases + ) match { + case success @ ElasticSuccess(true) => + // index created successfully + logger.info(s"βœ… Index $indexName created successfully.") + success + case ElasticSuccess(_) => + // index creation failed + val error = + ElasticError( + message = s"Failed to create index $indexName.", + statusCode = Some(500), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case failure @ ElasticFailure(_) => + failure + } + } + } + + private def loadTablePipelineDiff( + table: Table, + pipelineType: IngestPipelineType, + updatedPipelineName: Option[String], + existingPipelineName: Option[String] + ): ElasticResult[Option[List[PipelineDiff]]] = { + updatedPipelineName match { + case Some(pipelineName) if pipelineName != existingPipelineName.getOrElse("") => + logger.info( + s"πŸ”„ ${pipelineType.name} ingesting pipeline for index ${table.name} has been updated to $pipelineName." + ) + // load new pipeline + api.getPipeline(pipelineName) match { + case ElasticSuccess(maybePipeline) if maybePipeline.isDefined => + val pipeline = IngestPipeline( + name = pipelineName, + json = maybePipeline.get, + pipelineType = Some(IngestPipelineType.Default) + ) + // compute diff for pipeline update + val pipelineDiff: List[PipelineDiff] = pipeline.diff(table.defaultPipeline) + ElasticSuccess(Some(pipelineDiff)) + case ElasticSuccess(_) => + val error = + ElasticError( + message = + s"${pipelineType.name} ingesting pipeline $pipelineName for index ${table.name} not found.", + statusCode = Some(404), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case ElasticFailure(elasticError) => + val error = + ElasticError( + message = + s"Error retrieving ${pipelineType.name} ingesting pipeline $pipelineName for index ${table.name}: ${elasticError.message}", + statusCode = elasticError.statusCode, + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + } + case _ => ElasticSuccess(None) + } + } + + private def applyUpdatesInPlace( + indexName: String, + updated: Table, + diff: TableDiff + ): Either[ElasticError, Boolean] = { + + val mappingUpdate = + if (diff.mappings.nonEmpty || diff.columns.nonEmpty) + api.setMapping(indexName, updated.indexMappings) + else ElasticSuccess(true) + + val settingsUpdate = + if (diff.settings.nonEmpty) + api.updateSettings(indexName, updated.indexSettings) + else ElasticSuccess(true) + + val aliasUpdate = + if (diff.aliases.nonEmpty) + api.setAliases(indexName, updated.indexAliases) + else ElasticSuccess(true) + + val defaultPipelineCreateOrUpdate = + if (diff.defaultPipeline.nonEmpty) { + updated.defaultPipelineName match { + case Some(pipelineName) => + logger.info( + s"πŸ”§ Updating default ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Default)) + case None => + val pipelineName = updated.defaultPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Default) + ) + } + } else ElasticSuccess(true) + + val finalPipelineCreateOrUpdate = + if (diff.finalPipeline.nonEmpty) { + updated.finalPipelineName match { + case Some(pipelineName) => + logger.info( + s"πŸ”§ Updating final ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Final)) + case None => + val pipelineName = updated.finalPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Final) + ) + } + } else ElasticSuccess(true) + + for { + _ <- mappingUpdate.toEither + _ <- settingsUpdate.toEither + _ <- aliasUpdate.toEither + _ <- defaultPipelineCreateOrUpdate.toEither + last <- finalPipelineCreateOrUpdate.toEither + } yield last + } + + private def migrateToNewSchema( + indexName: String, + oldSchema: Table, + newSchema: Table, + diff: TableDiff + ): Either[ElasticError, Boolean] = { + + val tmpIndex = s"${indexName}_tmp_${System.currentTimeMillis()}" + + // migrate index with updated schema + val migrate: ElasticResult[Boolean] = + api.performMigration( + index = indexName, + tempIndex = tmpIndex, + mapping = newSchema.indexMappings, + settings = newSchema.indexSettings, + aliases = newSchema.indexAliases + ) match { + case ElasticFailure(err) => + logger.error(s"❌ Failed to perform migration for index $indexName: ${err.message}") + api.rollbackMigration( + index = indexName, + tempIndex = tmpIndex, + originalMapping = oldSchema.indexMappings, + originalSettings = oldSchema.indexSettings, + originalAliases = oldSchema.indexAliases + ) match { + case ElasticSuccess(_) => + logger.info(s"βœ… Rollback of migration for index $indexName completed successfully.") + ElasticFailure(err.copy(operation = Some("schema"))) + case ElasticFailure(rollbackErr) => + logger.error( + s"❌ Failed to rollback migration for index $indexName: ${rollbackErr.message}" + ) + ElasticFailure( + ElasticError( + message = + s"Migration failed: ${err.message}. Rollback failed: ${rollbackErr.message}", + statusCode = Some(500), + operation = Some("schema") + ) + ) + } + + case success @ ElasticSuccess(_) => + logger.info( + s"πŸ”„ Migration performed successfully for index $indexName to temporary index $tmpIndex." + ) + success + } + + val defaultPipelineCreateOrUpdate: ElasticResult[Boolean] = + if (diff.defaultPipeline.nonEmpty) { + newSchema.defaultPipelineName match { + case Some(pipelineName) => + logger.info( + s"πŸ”§ Updating default ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Default)) + case None => + val pipelineName = newSchema.defaultPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Default) + ) + } + } else ElasticSuccess(true) + + val finalPipelineCreateOrUpdate: ElasticResult[Boolean] = + if (diff.finalPipeline.nonEmpty) { + newSchema.finalPipelineName match { + case Some(pipelineName) => + logger.info( + s"πŸ”§ Updating final ingesting pipeline for index $indexName to $pipelineName." + ) + api.pipeline(diff.alterPipeline(pipelineName, IngestPipelineType.Final)) + case None => + val pipelineName = newSchema.finalPipeline.name + api.pipeline( + diff.createPipeline(pipelineName, IngestPipelineType.Final) + ) + } + } else ElasticSuccess(true) + + for { + _ <- migrate.toEither + _ <- defaultPipelineCreateOrUpdate.toEither + last <- finalPipelineCreateOrUpdate.toEither + } yield last + } +} + +class DdlRouterExecutor( + pipelineExec: PipelineExecutor, + tableExec: TableExecutor +) extends Executor[DdlStatement] { + + override def execute( + statement: DdlStatement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = statement match { + + case p: PipelineStatement => pipelineExec.execute(p) + case t: TableStatement => tableExec.execute(t) + + case _ => + Future.successful( + ElasticFailure( + ElasticError(s"Unsupported DDL statement: $statement", statusCode = Some(400)) + ) + ) + } +} + +trait GatewayApi extends ElasticClientHelpers { + _: IndicesApi + with PipelineApi + with MappingApi + with SettingsApi + with AliasApi + with TemplateApi + with SearchApi + with ScrollApi + with VersionApi => + + lazy val dqlExecutor = new DqlExecutor( + api = this, + logger = logger + ) + + lazy val dmlExecutor = new DmlExecutor( + api = this, + logger = logger + ) + + lazy val pipelineExecutor = new PipelineExecutor( + api = this, + logger = logger + ) + + lazy val tableExecutor = new TableExecutor( + api = this, + logger = logger + ) + + lazy val ddlExecutor = new DdlRouterExecutor( + pipelineExec = pipelineExecutor, + tableExec = tableExecutor + ) + + // ======================================================================== + // SQL GATEWAY API + // ======================================================================== + + def run(sql: String)(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + val normalizedQuery = + sql + .split("\n") + .map(_.split("--")(0).trim) + .filterNot(w => w.isEmpty || w.startsWith("--")) + .mkString(" ") + normalizedQuery.split(";\\s*$").toList match { + case Nil => + val error = + ElasticError( + message = s"Empty SQL query.", + statusCode = Some(400), + operation = Some("sql") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + case statement :: Nil => + ElasticResult.attempt(Parser(statement)) match { + case ElasticSuccess(parsedStatement) => + parsedStatement match { + case Right(statement) => + run(statement) + case Left(l) => + // parsing error + val error = + ElasticError( + message = s"Error parsing schema DDL statement: ${l.msg}", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + + case ElasticFailure(elasticError) => + // parsing error + Future.successful(ElasticFailure(elasticError.copy(operation = Some("schema")))) + } + + case statements => + implicit val ec: ExecutionContext = system.dispatcher + // run each statement sequentially and return the result of the last one + val last = statements + .foldLeft( + Future.successful[ElasticResult[QueryResult]](ElasticSuccess(QueryResult.empty)) + ) { (acc, statement) => + acc.flatMap { + case ElasticSuccess(_) => + run(statement) + case failure @ ElasticFailure(_) => + return Future.successful(failure) + } + } + last + } + } + + def run( + statement: Statement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + statement match { + + case dql: DqlStatement => + dqlExecutor.execute(dql) + + // handle DML statements + case dml: DmlStatement => + dmlExecutor.execute(dml) + + // handle DDL statements + case ddl: DdlStatement => + ddlExecutor.execute(ddl) + + case _ => + // unsupported SQL statement + val error = + ElasticError( + message = s"Unsupported SQL statement: $statement", + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticFailure(error)) + } + } + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 6bb7e8f1..680f4ee9 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -16,7 +16,30 @@ package app.softnetwork.elastic.client +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk.BulkOptions import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + Delete, + FileFormat, + From, + Insert, + SingleSearch, + Table, + Unknown, + Update +} +import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline, Schema, TableAlias} +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import org.apache.hadoop.conf.Configuration + +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ /** Index management API. * @@ -26,7 +49,8 @@ import app.softnetwork.elastic.client.result._ * - Parameter validation * - Automatic retry for transient errors */ -trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => +trait IndicesApi extends ElasticClientHelpers { + _: RefreshApi with PipelineApi with BulkApi with ScrollApi with VersionApi with TemplateApi => // ======================================================================== // PUBLIC METHODS @@ -84,10 +108,19 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => * - the name of the index to create * @param settings * - the settings to apply to the index (default is defaultSettings) + * @param mappings + * - optional mappings to apply to the index + * @param aliases + * - optional aliases to apply to the index * @return * true if the index was created successfully, false otherwise */ - def createIndex(index: String, settings: String = defaultSettings): ElasticResult[Boolean] = { + def createIndex( + index: String, + settings: String = defaultSettings, + mappings: Option[String] = None, + aliases: Seq[TableAlias] = Nil + ): ElasticResult[Boolean] = { validateIndexName(index) match { case Some(error) => return ElasticFailure( @@ -113,9 +146,46 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => case None => // OK } + mappings.map(validateJsonMappings) match { + case Some(Some(error)) => + return ElasticFailure( + error.copy( + operation = Some("createIndex"), + statusCode = Some(400), + message = s"Invalid mappings: ${error.message}" + ) + ) + case _ => // OK + } + + aliases.flatMap(alias => validateAliasName(alias.alias)) match { + case error :: _ => + return ElasticFailure( + error.copy( + operation = Some("createIndex"), + statusCode = Some(400), + message = s"Invalid alias: ${error.message}" + ) + ) + case Nil => // OK + } + logger.info(s"Creating index '$index' with settings: $settings") - executeCreateIndex(index, settings) match { + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + val updatedMappings = + mappings.map(mapping => MappingConverter.convert(mapping, elasticVersion)) + + executeCreateIndex(index, settings, updatedMappings, aliases) match { case success @ ElasticSuccess(true) => logger.info(s"βœ… Index '$index' created successfully") success @@ -128,7 +198,189 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => } } + /** Load the schema for the provided index. + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + def loadSchema(index: String): ElasticResult[Schema] = { + getIndex(index) match { + case ElasticSuccess(Some(idx)) => + ElasticSuccess(idx.schema) + case ElasticSuccess(None) => + logger.warn(s"Index '$index' not found for schema loading") + getTemplate(index) match { + case ElasticSuccess(Some(template)) => + logger.info(s"βœ… Template '$index' found for schema loading") + var templateNode = mapper.readTree(template).asInstanceOf[ObjectNode] + if (templateNode.has("index_template")) { + // Index template + templateNode = templateNode.get("index_template").asInstanceOf[ObjectNode] + } + if (templateNode.has("template")) { + // Composable template + templateNode = templateNode.get("template").asInstanceOf[ObjectNode] + } + val root = mapper.createObjectNode() + if (templateNode.has("mappings")) { + root.set("mappings", templateNode.get("mappings")) + } + if (templateNode.has("settings")) { + root.set("settings", templateNode.get("settings")) + } + if (templateNode.has("aliases")) { + root.set("aliases", templateNode.get("aliases")) + } + loadIndexAsSchema(index, root.toString) match { + case ElasticSuccess(Some(idx)) => + ElasticSuccess(idx.schema) + case ElasticSuccess(None) => + val error = + ElasticError( + message = s"Failed to load schema from template for index '$index'", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("loadSchema") + ) + logger.error(s"❌ ${error.message}") + ElasticFailure(error) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to load schema from template for index '$index': ${error.message}" + ) + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } + case ElasticSuccess(None) => + logger.warn(s"Template '$index' not found for schema loading") + ElasticFailure( + ElasticError.notFound( + index = index, + operation = "loadSchema" + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to load template for schema of index '$index': ${error.message}" + ) + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } + case ElasticFailure(error) => + logger.error(s"❌ Failed to load schema for index '$index': ${error.message}") + ElasticFailure(error.copy(operation = Some("loadSchema"))) + } + } + + /** Get an index with the provided name. + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + def getIndex(index: String): ElasticResult[Option[Index]] = { + validateIndexName(index) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("getIndex"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${error.message}" + ) + ) + case None => // OK + } + + logger.info(s"Getting index '$index'") + + executeGetIndex(index) match { + case ElasticSuccess(Some(json)) => + logger.info(s"βœ… Index '$index' retrieved successfully") + loadIndexAsSchema(index, json) + case ElasticSuccess(None) => + logger.warn(s"βœ… Index '$index' not found") + ElasticSuccess(None) + case ElasticFailure(error) if error.statusCode.contains(404) => + logger.info(s"βœ… Index '$index' not found") + ElasticSuccess(None) + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to get index '$index': ${error.message}") + failure + } + } + + private def loadIndexAsSchema(index: String, json: String): ElasticResult[Option[Index]] = { + var tempIndex = Index(index, json) + tempIndex.defaultIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has default ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"βœ… Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(defaultPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = s"Default ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + tempIndex.finalIngestPipelineName match { + case Some(pipeline) => + logger.info( + s"Index '$index' has final ingest pipeline '$pipeline'" + ) + getPipeline(pipeline) match { + case ElasticSuccess(Some(json)) => + logger.info( + s"βœ… Ingest pipeline '$pipeline' for index '$index' exists" + ) + tempIndex = tempIndex.copy(finalPipeline = Some(json)) + case ElasticSuccess(None) => + logger.warn( + s"⚠️ Ingest pipeline '$pipeline' for index '$index' does not exist" + ) + return ElasticFailure( + ElasticError( + message = s"Final ingest pipeline '$pipeline' for index '$index' does not exist", + cause = None, + statusCode = Some(404), + index = Some(index), + operation = Some("getIndex") + ) + ) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to get ingest pipeline '$pipeline' for index '$index': ${error.message}" + ) + return ElasticFailure(error.copy(operation = Some("getIndex"))) + } + case None => // No default ingest pipeline + } + ElasticSuccess(Some(tempIndex)) + } + /** Delete an index with the provided name. + * * @param index * - the name of the index to delete * @return @@ -247,7 +499,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean = true + refresh: Boolean = true, + pipeline: Option[String] = None ): ElasticResult[(Boolean, Option[Long])] = { // Validation... validateIndexName(sourceIndex) match { @@ -291,7 +544,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => logger.info(s"Reindexing from '$sourceIndex' to '$targetIndex' (refresh=$refresh)") // Existence checks... - indexExists(sourceIndex) match { + indexExists(sourceIndex, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -306,7 +559,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => case _ => // OK } - indexExists(targetIndex) match { + indexExists(targetIndex, pattern = false) match { case ElasticSuccess(false) => return ElasticFailure( ElasticError( @@ -322,7 +575,7 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => } // βœ… Performing the reindex with extracting the number of documents - executeReindex(sourceIndex, targetIndex, refresh) match { + executeReindex(sourceIndex, targetIndex, refresh, pipeline) match { case ElasticFailure(error) => logger.error(s"Reindex failed for index '$targetIndex': ${error.message}") ElasticFailure(error) @@ -365,8 +618,8 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => * @return * true if the index exists, false otherwise */ - def indexExists(index: String): ElasticResult[Boolean] = { - validateIndexName(index) match { + def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = { + validateIndexName(index, pattern) match { case Some(error) => return ElasticFailure( error.copy( @@ -386,17 +639,1054 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => val existenceStr = if (exists) "exists" else "does not exist" logger.debug(s"βœ… Index '$index' $existenceStr") success + case f: ElasticFailure if f.isNotFound => + logger.debug(s"βœ… Index '$index' does not exist") + ElasticSuccess(false) case failure @ ElasticFailure(error) => logger.error(s"❌ Failed to check existence of index '$index': ${error.message}") failure } } + def isIndexClosed(index: String): ElasticResult[Boolean] = { + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("isIndexClosed"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${err.message}" + ) + ) + // 2. Check index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("isIndexClosed") + ) + ) + case ElasticFailure(err) => Left(err) + } + // 3. Retrieve index status + closed <- executeIsIndexClosed(index).toEither + + } yield closed + + result match { + case Right(closed) => + val statusStr = if (closed) "closed" else "open" + logger.info(s"βœ… Index '$index' is $statusStr") + ElasticSuccess(closed) + case Left(err) => + logger.error(s"❌ Failed to check if index '$index' is closed: ${err.message}") + ElasticFailure(err) + } + } + + /** Truncate an index by deleting all its documents. + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + def truncateIndex(index: String): ElasticResult[Long] = + deleteByQuery(index, """{"query": {"match_all": {}}}""") + + /** Delete documents by query from an index. + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + def deleteByQuery( + index: String, + query: String, + refresh: Boolean = true + ): ElasticResult[Long] = { + + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid index: ${err.message}" + ) + ) + + // 2. Parse query (SQL or JSON) + jsonQuery <- parseQueryForDeletion(index, query) + + // 3. Check index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("deleteByQuery") + ) + ) + case ElasticFailure(err) => Left(err) + } + + // 4. Open index if needed + tuple <- openIfNeeded(index) + (_, restore) = tuple + + // 5. Execute delete-by-query + deleted <- executeDeleteByQuery(index, jsonQuery, refresh).toEither + + // 6. Restore state + _ <- restore().toEither.left.map { restoreErr => + logger.warn(s"❌ Failed to restore index state for '$index': ${restoreErr.message}") + restoreErr + } + + } yield deleted + + result match { + case Right(count) => + logger.info(s"βœ… Deleted $count documents from index '$index'") + ElasticSuccess(count) + case Left(err) => + logger.error(s"❌ Failed to delete by query on index '$index': ${err.message}") + ElasticFailure(err) + } + } + + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + def updateByQuery( + index: String, + query: String, + pipelineId: Option[String] = None, + refresh: Boolean = true + ): ElasticResult[Long] = { + + val result = for { + // 1. Validate index name + _ <- validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("updateByQuery"), + statusCode = Some(400), + index = Some(index) + ) + ) + + // 2. Parse SQL or JSON + parsed <- parseQueryForUpdate(index, query).toEither + + // 3. Extract SQL pipeline (optional) + sqlPipeline = parsed match { + case Left(u: Update) if u.values.nonEmpty => Some(u.customPipeline) + case _ => None + } + + // 4. Extract SQL WHERE β†’ JSON query + jsonQuery = parsed match { + case Left(u: Update) => + u.where match { + case None => + logger.info( + s"SQL update query has no WHERE clause, updating all documents from index '$index'" + ) + """{"query": {"match_all": {}}}""" + + case Some(where) => + implicit val timestamp: Long = System.currentTimeMillis() + val search: String = + SingleSearch( + from = From(tables = Seq(Table(u.table))), + where = Some(where), + updateByQuery = true + ) + logger.info(s"βœ… Converted SQL update query to search for updateByQuery: $search") + search + } + + case _ => query // JSON passthrough + } + + // 5. Load user pipeline if provided + userPipeline <- pipelineId match { + case Some(id) => + getPipeline(id).toEither.flatMap { + case Some(json) => Right(Some(IngestPipeline(id, json))) + case None => + Left( + ElasticError( + message = s"Pipeline '$id' not found", + index = Some(index), + operation = Some("updateByQuery"), + statusCode = Some(404) + ) + ) + } + case None => Right(None) + } + + // 6. Resolve final pipeline (merge if needed) + elasticVersion <- this.version.toEither + + resolved <- resolveFinalPipeline(userPipeline, sqlPipeline, elasticVersion).toEither + (finalPipelineId, mustDelete) = resolved + + // 7. Ensure index exists + _ <- indexExists(index, pattern = false) match { + case ElasticSuccess(true) => Right(()) + case ElasticSuccess(false) => + Left( + ElasticError( + message = s"Index '$index' does not exist", + statusCode = Some(404), + index = Some(index), + operation = Some("updateByQuery") + ) + ) + case ElasticFailure(err) => Left(err) + } + + // 8. Open index if needed + tuple <- openIfNeeded(index) + (_, restore) = tuple + + // 9. Execute update-by-query + updated <- executeUpdateByQuery(index, jsonQuery, finalPipelineId, refresh).toEither + + // 10. Cleanup temporary pipeline + _ <- + if (mustDelete && finalPipelineId.isDefined) + deletePipeline(finalPipelineId.get, ifExists = true).toEither + else Right(()) + + // 11. Restore index state + _ <- restore().toEither + + } yield updated + + result match { + case Right(count) => ElasticSuccess(count) + case Left(err) => ElasticFailure(err) + } + } + + /** Insert documents by query into an index. + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + def insertByQuery( + index: String, + query: String, + refresh: Boolean = true + )(implicit system: ActorSystem): Future[ElasticResult[DmlResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val result = for { + // 1. Validate index + _ <- Future.fromTry( + validateIndexName(index) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index) + ) + ) + .toTry + ) + + // 2. Parse SQL INSERT + parsed <- parseInsertQuery(index, query).toFuture + + // 3. Load index metadata + idx <- Future.fromTry(getIndex(index).toEither.flatMap { + case Some(i) => Right(i.schema) + case None => + Left(ElasticError.notFound(index, "insertByQuery")) + }.toTry) + + // 3.b Compute effective insert columns (handles INSERT ... AS SELECT without column list) + val effectiveInsertCols: Seq[String] = parsed.values match { + case Left(single: SingleSearch) => + if (parsed.cols.isEmpty) + single.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.sourceField) + } + else parsed.cols + + case _ => + parsed.cols + } + + // 3.c Validate ON CONFLICT rules + _ <- Future.fromTry { + val pk = idx.primaryKey + val doUpdate = parsed.doUpdate + val conflictTarget = parsed.conflictTarget + + val validation: Either[ElasticError, Unit] = + if (!doUpdate) { + Right(()) + } else if (pk.nonEmpty) { + // --- Case 1: PK defined in index + val conflictKey = conflictTarget.getOrElse(pk) + + // conflictTarget must match PK exactly + if (conflictKey.toSet != pk.toSet) + Left( + ElasticError( + message = s"Conflict target columns [${conflictKey + .mkString(",")}] must match primary key [${pk.mkString(",")}]", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + // INSERT must include all PK columns + else if (!pk.forall(effectiveInsertCols.contains)) + Left( + ElasticError( + message = + s"INSERT must include all primary key columns [${pk.mkString(",")}] when using ON CONFLICT DO UPDATE", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + else + Right(()) + } else { + // --- Case 2: No PK defined in index + conflictTarget match { + case None => + Left( + ElasticError( + message = + "ON CONFLICT DO UPDATE requires a conflict target when no primary key is defined in the index", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + + case Some(conflictKey) => + if (!conflictKey.forall(effectiveInsertCols.contains)) + Left( + ElasticError( + message = + s"INSERT must include all conflict target columns [${conflictKey.mkString(",")}] when using ON CONFLICT DO UPDATE", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + else + Right(()) + } + } + + validation.toTry + } + + // 3.d Validate SELECT columns for INSERT ... AS SELECT + _ <- Future.fromTry { + parsed.values match { + + // INSERT ... VALUES β†’ rien Γ  valider + case Right(_) => + Right(()).toTry + + // INSERT ... AS SELECT + case Left(single: SingleSearch) => + val selectCols = single.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.sourceField) + } + + // VΓ©rifier que toutes les colonnes de l'INSERT sont prΓ©sentes dans le SELECT + val missing = effectiveInsertCols.filterNot(selectCols.contains) + + if (missing.nonEmpty) + Left( + ElasticError( + message = + s"INSERT columns [${effectiveInsertCols.mkString(",")}] must all be present in SELECT output columns [${selectCols + .mkString(",")}]. Missing: ${missing.mkString(",")}", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ).toTry + else + Right(()).toTry + + case Left(_) => + Left( + ElasticError( + message = "INSERT AS SELECT requires a SELECT statement", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ).toTry + } + } + + // 4. Derive bulk options + idKey = idx.primaryKey match { + case Nil => None + case pk => Some(pk.toSet) + } + suffixKey = idx.partitionBy.map(_.column) + suffixPattern = idx.partitionBy.flatMap(_.dateFormats.headOption) + + // 5. Build source of documents + source <- Future.fromTry((parsed.values match { + + // INSERT … VALUES + case Right(_) => + parsed.toJson match { + case Some(jsonNode) => + val arrayNode = jsonNode.asInstanceOf[ArrayNode] + val docs: Seq[JsonNode] = arrayNode.elements().asScala.toSeq + Right(Source.fromIterator(() => docs.map(_.toString).toIterator)) + case None => + Left( + ElasticError( + message = "Invalid INSERT ... VALUES clause", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + } + + // INSERT … AS SELECT + case Left(single: SingleSearch) => + Right( + scroll(single).map { case (row, _) => + val jsonNode: JsonNode = row - "_id" - "_index" - "_score" - "_sort" + jsonNode.toString + } + ) + + case Left(_) => + Left( + ElasticError( + message = "INSERT AS SELECT requires a SELECT statement", + operation = Some("insertByQuery"), + index = Some(index), + statusCode = Some(400) + ) + ) + }).toTry) + + // 6. Bulk insert + bulkResult <- bulkWithResult[String]( + items = source, + toDocument = identity, + indexKey = Some(index), + idKey = idKey, + suffixDateKey = suffixKey, + suffixDatePattern = suffixPattern, + update = Some(parsed.doUpdate) + )(BulkOptions(defaultIndex = index, disableRefresh = !refresh), system) + + } yield bulkResult + + result + .map(r => ElasticSuccess(DmlResult(inserted = r.successCount, rejected = r.failedCount))) + .recover { + case e: ElasticError => + ElasticFailure( + e.copy( + operation = Some("insertByQuery"), + index = Some(index) + ) + ) + case e => + ElasticFailure( + ElasticError( + message = e.getMessage, + operation = Some("insertByQuery"), + index = Some(index) + ) + ) + } + } + + /** Copy documents from files into an index. + * + * @param source + * - the source file path (can be local or remote) + * @param target + * - the target index name + * @param doUpdate + * - true to update existing documents, false to insert only + * @param fileFormat + * - optional file format (if not provided, will be inferred from file extension) + * @return + * the number of documents inserted/updated + */ + def copyInto( + source: String, + target: String, + doUpdate: Boolean, + fileFormat: Option[FileFormat] = None, + hadoopConf: Option[Configuration] = None + )(implicit + system: ActorSystem + ): Future[ElasticResult[DmlResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + + val result = for { + // 1. Validate target index + _ <- Future.fromTry( + validateIndexName(target) + .toLeft(()) + .left + .map(err => + err.copy( + operation = Some("copyInto"), + statusCode = Some(400), + index = Some(target) + ) + ) + .toTry + ) + + // 2 Load target metadata + idx <- Future.fromTry(getIndex(target).toEither.flatMap { + case Some(i) => Right(i.schema) + case None => + Left(ElasticError.notFound(target, "copyInto")) + }.toTry) + + // 3. Derive bulk options + idKey = idx.primaryKey match { + case Nil => None + case pk => Some(pk.toSet) + } + suffixKey = idx.partitionBy.map(_.column) + suffixPattern = idx.partitionBy.flatMap(_.dateFormats.headOption) + + // 4. Bulk file copy + bulkResult <- bulkFromFile( + filePath = source, + format = fileFormat.getOrElse(Unknown), + indexKey = Some(target), + idKey = idKey, + suffixDateKey = suffixKey, + suffixDatePattern = suffixPattern, + update = Some(doUpdate), + hadoopConf = hadoopConf + )(BulkOptions(defaultIndex = target), system) + + } yield bulkResult + + result + .map(r => ElasticSuccess(DmlResult(inserted = r.successCount, rejected = r.failedCount))) + .recover { + case e: ElasticError => + ElasticFailure( + e.copy( + operation = Some("copyInto"), + index = Some(target) + ) + ) + case e => + ElasticFailure( + ElasticError( + message = e.getMessage, + operation = Some("copyInto"), + index = Some(target) + ) + ) + } + } + + /** Parse a query for update, determining if it's SQL or JSON. + * + * @param index + * - the name of the index to update + * @param query + * - the query (SQL or JSON) + * @return + * either the parsed SQL Update statement or the validated JSON query + */ + private def parseQueryForUpdate( + index: String, + query: String + ): ElasticResult[Either[Update, String]] = { + val trimmed = query.trim.toUpperCase + val isSql = trimmed.startsWith("UPDATE") + if (isSql) { + logger.info(s"Processing SQL query for updateByQuery on index '$index': $query") + + Parser(query) match { + case Left(err) => + ElasticFailure( + ElasticError( + operation = Some("updateByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL query: ${err.msg}" + ) + ) + + case Right(statement) => + statement match { + + case updateStmt: Update => + if (updateStmt.table != index) + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = + s"SQL query index '${updateStmt.table}' does not match provided index '$index'" + ) + ) + else + ElasticSuccess(Left(updateStmt)) + + case search: SingleSearch => + val tables = search.from.tables + if (tables.size != 1 || tables.head.name != index) + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = + s"SQL query index '${tables.map(_.name).mkString(",")}' does not match provided index '$index'" + ) + ) + else { + implicit val timestamp: Long = System.currentTimeMillis() + val query: String = search.copy(deleteByQuery = false) + logger.info(s"βœ… Converted SQL search query to JSON for updateByQuery: $query") + ElasticSuccess(Right(query)) + } + + case _ => + ElasticFailure( + sqlErrorFor( + operation = "updateByQuery", + index = index, + message = s"Invalid SQL query for updateByQuery" + ) + ) + } + } + } else { + logger.info(s"Processing JSON query for updateByQuery on index '$index': $query") + ElasticSuccess(Right(query)) + } + } + + /** Parse a query for deletion, determining if it's SQL or JSON. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query (SQL or JSON) + * @return + * the validated JSON query + */ + private def parseQueryForDeletion(index: String, query: String): Either[ElasticError, String] = { + val trimmed = query.trim.toUpperCase + val isSql = trimmed.startsWith("SELECT") || + trimmed.startsWith("DELETE") || + trimmed.startsWith("WITH") + if (isSql) parseSqlQueryForDeletion(index, query) + else parseJsonQueryForDeletion(index, query) + } + + /** Validate a JSON query for deletion. + * + * @param index + * - the name of the index to delete from + * @param query + * - the JSON query + * @return + * the validated JSON query + */ + private def parseJsonQueryForDeletion( + index: String, + query: String + ): Either[ElasticError, String] = { + validateJson("deleteByQuery", query) match { + case None => + logger.info(s"Processing JSON query for deleteByQuery on index '$index': $query") + Right(query) + + case Some(err) => + Left( + err.copy( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid JSON query: ${err.message}" + ) + ) + } + } + + /** Parse an SQL query for deletion and convert it to Elasticsearch JSON. + * + * @param index + * - the name of the index to delete from + * @param query + * - the SQL query + * @return + * the Elasticsearch JSON query + */ + private def parseSqlQueryForDeletion( + index: String, + query: String + ): Either[ElasticError, String] = { + logger.info(s"Processing SQL query for deleteByQuery on index '$index': $query") + + Parser(query) match { + case Left(err) => + Left( + ElasticError( + operation = Some("deleteByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL query: ${err.msg}" + ) + ) + + case Right(statement) => + statement match { + + case deleteStmt: Delete => + if (deleteStmt.table.name != index) + Left( + sqlErrorFor( + operation = "deleteByQuery", + index = index, + message = + s"SQL query index '${deleteStmt.table.name}' does not match provided index '$index'" + ) + ) + else + deleteStmt.where match { + case None => + logger.info( + s"SQL delete query has no WHERE clause, deleting all documents from index '$index'" + ) + Right("""{"query": {"match_all": {}}}""") + + case Some(where) => + implicit val timestamp: Long = System.currentTimeMillis() + val search: String = + SingleSearch( + from = From(tables = Seq(deleteStmt.table)), + where = Some(where), + deleteByQuery = true + ) + logger.info(s"βœ… Converted SQL delete query to search for deleteByQuery: $search") + Right(search) + } + + case search: SingleSearch => + val tables = search.from.tables + if (tables.size != 1 || tables.head.name != index) + Left( + sqlErrorFor( + operation = "deleteByQuery", + index = index, + message = + s"SQL query index '${tables.map(_.name).mkString(",")}' does not match provided index '$index'" + ) + ) + else { + implicit val timestamp: Long = System.currentTimeMillis() + val query: String = search.copy(deleteByQuery = true) + logger.info(s"βœ… Converted SQL search query to search for deleteByQuery: $query") + Right(query) + } + + case _ => + Left( + sqlErrorFor( + operation = "deleteByQuery", + index = index, + message = s"Invalid SQL query for deleteByQuery" + ) + ) + } + } + } + + /** Open an index if it is closed, returning a function to restore its state. + * + * @param index + * - the name of the index to open if needed + * @return + * a tuple containing a boolean indicating if the index was opened and a function to restore + * its state + */ + private def openIfNeeded( + index: String + ): Either[ElasticError, (Boolean, () => ElasticResult[Boolean])] = { + for { + // Detect initial state + isClosed <- isIndexClosed(index).toEither + + // Open only if needed + _ <- if (isClosed) openIndex(index).toEither else Right(()) + _ <- if (isClosed) waitForShards(index).toEither else Right(()) + + } yield { + val restore = () => + if (isClosed) closeIndex(index) + else ElasticSuccess(true) + + (isClosed, restore) + } + } + + /** Resolve the final ingest pipeline to use for updateByQuery, merging user and SQL pipelines if + * needed. + * + * @param user + * - optional user-provided ingest pipeline + * @param sql + * - optional SQL-derived ingest pipeline + * @param elasticVersion + * - Elasticsearch version + * @return + * a tuple containing the final pipeline id to use (if any) and a boolean indicating if the + * pipeline is temporary and must be deleted after use + */ + private def resolveFinalPipeline( + user: Option[IngestPipeline], + sql: Option[IngestPipeline], + elasticVersion: String + ): ElasticResult[(Option[String], Boolean)] = { + + (user, sql) match { + + // No pipeline + case (None, None) => + ElasticSuccess((None, false)) + + // User pipeline only + case (Some(u), None) => + ElasticSuccess((Some(u.name), false)) + + // Only SQL pipeline β†’ temporary pipeline + case (None, Some(sqlPipe)) => + val tmpId = s"_tmp_update_${System.nanoTime()}" + val json = + if (ElasticsearchVersion.isEs6(elasticVersion)) { + sqlPipe + .copy( + processors = sqlPipe.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = + processor.properties.filterNot(_._1 == "description").filterNot(_._1 == "if") + ) + } + ) + .json + } else { + sqlPipe + .copy( + processors = sqlPipe.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "if") + ) + } + ) + .json + } + logger.info(s"Creating temporary pipeline for updateByQuery: $json") + createPipeline(tmpId, json) match { + case ElasticSuccess(_) => ElasticSuccess((Some(tmpId), true)) + case ElasticFailure(e) => ElasticFailure(e) + } + + // Merge user + SQL pipeline β†’ temporary pipeline + case (Some(u), Some(sqlPipe)) => + val merged = u.merge(sqlPipe) + val tmpId = s"_tmp_update_${System.nanoTime()}" + val json = + if (ElasticsearchVersion.isEs6(elasticVersion)) { + merged + .copy( + processors = merged.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = + processor.properties.filterNot(_._1 == "description").filterNot(_._1 == "if") + ) + } + ) + .json + } else { + merged + .copy( + processors = merged.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties.filterNot(_._1 == "if") + ) + } + ) + .json + } + logger.info(s"Creating merged temporary pipeline for updateByQuery: $json") + createPipeline(tmpId, json) match { + case ElasticSuccess(_) => ElasticSuccess((Some(tmpId), true)) + case ElasticFailure(e) => ElasticFailure(e) + } + } + } + + /** Parse an SQL INSERT query. + * + * @param index + * - the name of the index to insert into + * @param query + * - the SQL INSERT query + * @return + * the parsed Insert statement + */ + private def parseInsertQuery( + index: String, + query: String + ): ElasticResult[Insert] = { + + Parser(query) match { + case Left(err) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"Invalid SQL: ${err.msg}" + ) + ) + + case Right(insert: Insert) => + if (insert.table != index) + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = s"SQL table '${insert.table}' does not match index '$index'" + ) + ) + else + insert.validate() match { + case Left(msg) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = msg + ) + ) + case Right(_) => + ElasticSuccess(insert) + } + + case Right(_) => + ElasticFailure( + ElasticError( + operation = Some("insertByQuery"), + statusCode = Some(400), + index = Some(index), + message = "Only INSERT statements are allowed" + ) + ) + } + } + + private def sqlErrorFor(operation: String, index: String, message: String): ElasticError = + ElasticError( + operation = Some(operation), + statusCode = Some(400), + index = Some(index), + message = message + ) + + // ================================================================================ + // IMPLICIT CONVERSIONS + // ================================================================================ + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== - private[client] def executeCreateIndex(index: String, settings: String): ElasticResult[Boolean] + private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] + + private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] @@ -407,8 +1697,33 @@ trait IndicesApi extends ElasticClientHelpers { _: RefreshApi => private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] private[client] def executeIndexExists(index: String): ElasticResult[Boolean] + + private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] + + private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] + + private[client] def waitForShards( + index: String, + status: String = "yellow", + timeout: Int = 30 + ): ElasticResult[Unit] = { + // Default implementation does nothing + ElasticSuccess(()) + } + + private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala index d274bac1..088d0bb0 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingApi.scala @@ -22,13 +22,16 @@ import app.softnetwork.elastic.client.result.{ ElasticResult, ElasticSuccess } +import app.softnetwork.elastic.sql.schema.TableAlias +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import java.util.UUID /** Mapping management API. */ -trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi with RefreshApi => +trait MappingApi extends ElasticClientHelpers { + _: SettingsApi with IndicesApi with RefreshApi with VersionApi with AliasApi => // ======================================================================== // PUBLIC METHODS @@ -69,9 +72,51 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w case None => // continue } - logger.debug(s"Setting mapping for index '$index': $mapping") + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + val updatedMapping: String = + if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion)) { + val root = mapper.readTree(mapping).asInstanceOf[ObjectNode] + + if (root.has("properties") || root.has("_meta")) { + logger.info(s"Wrapping mappings with '_doc' type for ES version $elasticVersion") + + val doc = mapper.createObjectNode() + + // Move properties + if (root.has("properties")) { + doc.set("properties", root.get("properties")) + root.remove("properties") + } + + // Move _meta + if (root.has("_meta")) { + doc.set("_meta", root.get("_meta")) + root.remove("_meta") + } + + // Wrap into _doc + root.set("_doc", doc) - executeSetMapping(index, mapping) match { + root.toString + } else { + mapping + } + } else { + mapping + } + + logger.debug(s"Setting mapping for index '$index': $updatedMapping") + + executeSetMapping(index, updatedMapping) match { case success @ ElasticSuccess(true) => logger.info(s"βœ… Mapping for index '$index' updated successfully") success @@ -155,10 +200,22 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w mapping: String, settings: String = defaultSettings ): ElasticResult[Boolean] = { - indexExists(index).flatMap { + indexExists(index, pattern = false).flatMap { case false => // Scenario 1: Index doesn't exist - createIndexWithMapping(index, mapping, settings) + createIndex(index, settings, Some(mapping), Nil).flatMap { + case true => + logger.info(s"βœ… Index '$index' created with mapping successfully") + ElasticResult.success(true) + case false => + ElasticResult.failure( + ElasticError( + message = s"Failed to create index '$index' with mapping", + index = Some(index), + operation = Some("updateMapping") + ) + ) + } case true => // Check if mapping needs update @@ -177,27 +234,6 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w } } - /** Create a new index with the given mapping. - */ - private def createIndexWithMapping( - index: String, - mapping: String, - settings: String - ): ElasticResult[Boolean] = { - logger.info(s"Creating new index '$index' with mapping") - - for { - _ <- createIndex(index, settings) - .filter(_ == true, s"Failed to create index '$index'") - .logSuccess(logger, _ => s"βœ… Index '$index' created successfully") - - _ <- setMapping(index, mapping) - .filter(_ == true, s"Failed to set mapping for index '$index'") - .logSuccess(logger, _ => s"βœ… Mapping for index '$index' set successfully") - - } yield true - } - private def migrateMappingWithRollback( index: String, newMapping: String, @@ -210,17 +246,19 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w val backupResult = for { originalMapping <- getMapping(index) originalSettings <- loadSettings(index) - } yield (originalMapping, originalSettings) + originalAliases <- getAliases(index) + } yield (originalMapping, originalSettings, originalAliases) backupResult match { - case ElasticSuccess((origMapping, origSettings)) => + case ElasticSuccess((origMapping, origSettings, origAliases)) => logger.info(s"βœ… Backed up original mapping and settings for '$index'") val migrationResult = performMigration( index, tempIndex, newMapping, - settings + settings, + Nil ) migrationResult match { @@ -255,23 +293,21 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w * Delete original index 4. Recreate original index with new mapping 5. Reindex data from * temporary to original 6. Delete temporary index */ - private def performMigration( + private[client] def performMigration( index: String, tempIndex: String, mapping: String, - settings: String + settings: String, + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { logger.info(s"Starting migration: $index -> $tempIndex") for { // Create temp index - _ <- createIndex(tempIndex, settings) + _ <- createIndex(tempIndex, settings, Some(mapping), aliases) .filter(_ == true, s"❌ Failed to create temp index '$tempIndex'") - _ <- setMapping(tempIndex, mapping) - .filter(_ == true, s"❌ Failed to set mapping on temp index") - // Reindex to temp _ <- reindex(index, tempIndex, refresh = true) .filter(_._1 == true, s"❌ Failed to reindex to temp") @@ -280,13 +316,10 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w _ <- deleteIndex(index) .filter(_ == true, s"❌ Failed to delete original index") - // Recreate original with new mapping - _ <- createIndex(index, settings) + // Recreate original with new settings, mapping and aliases + _ <- createIndex(index, settings, Some(mapping), Nil) .filter(_ == true, s"❌ Failed to recreate original index") - _ <- setMapping(index, mapping) - .filter(_ == true, s"❌ Failed to set new mapping") - // Reindex back from temp _ <- reindex(tempIndex, index, refresh = true) .filter(_._1 == true, s"❌ Failed to reindex from temp") @@ -303,32 +336,30 @@ trait MappingApi extends ElasticClientHelpers { _: SettingsApi with IndicesApi w } } - private def rollbackMigration( + private[client] def rollbackMigration( index: String, tempIndex: String, originalMapping: String, - originalSettings: String + originalSettings: String, + originalAliases: Seq[TableAlias] = Nil ): ElasticResult[Boolean] = { logger.warn(s"Rolling back migration for '$index'") for { // Check if temp index exists and has data - tempExists <- indexExists(tempIndex) + tempExists <- indexExists(tempIndex, pattern = false) // Delete current (potentially corrupted) index if it exists - _ <- indexExists(index).flatMap { + _ <- indexExists(index, pattern = false).flatMap { case true => deleteIndex(index) case false => ElasticResult.success(true) } // Recreate with original settings and mapping - _ <- createIndex(index, originalSettings) + _ <- createIndex(index, originalSettings, Some(originalMapping), originalAliases) .filter(_ == true, s"❌ Rollback: Failed to recreate index") - _ <- setMapping(index, originalMapping) - .filter(_ == true, s"❌ Rollback: Failed to restore mapping") - // If temp exists, reindex from it _ <- if (tempExists) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala index badb9962..d18fe4c2 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala @@ -26,8 +26,8 @@ object MappingComparator extends StrictLogging { private def parseJsonToMap(jsonString: String): Map[String, JsonElement] = { Try( - new JsonParser() - .parse(jsonString) + JsonParser + .parseString(jsonString) .getAsJsonObject .get("properties") .getAsJsonObject diff --git a/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala new file mode 100644 index 00000000..de224a34 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/MappingConverter.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.{Logger, LoggerFactory} + +object MappingConverter { + + private val logger: Logger = LoggerFactory.getLogger(getClass) + + private lazy val mapper: ObjectMapper = JacksonConfig.objectMapper + + def convert(mapping: String, elasticVersion: String): String = { + val root = mapper.readTree(mapping).asInstanceOf[ObjectNode] + mapper.writeValueAsString(convert(root, elasticVersion)) + } + + def convert(mapping: ObjectNode, elasticVersion: String): ObjectNode = { + if (ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mapping.has("_doc")) { + wrapDocType(mapping) + } else if ( + !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mapping.has("_doc") + ) { + unwrapDocType(mapping) + } else mapping + } + + private def wrapDocType(mappingsNode: ObjectNode): ObjectNode = { + if (!mappingsNode.has("_doc")) { + logger.info(s"Wrapping mappings from '_doc' type") + val root = mapper.createObjectNode() + // Wrap into _doc + root.set("_doc", mappingsNode) + root + } else { + mappingsNode + } + } + + private def unwrapDocType(mappingsNode: ObjectNode): ObjectNode = { + val root = mapper.createObjectNode() + + if (mappingsNode.has("_doc")) { + logger.info(s"Unwrapping mappings from '_doc' type") + val doc = mappingsNode.get("_doc").asInstanceOf[ObjectNode] + doc + .properties() + .forEach(entry => { + root.set(entry.getKey, entry.getValue) + }) + root + } else { + mappingsNode + } + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala new file mode 100644 index 00000000..d74cf353 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/NopeClientApi.scala @@ -0,0 +1,379 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Flow, Source} +import app.softnetwork.elastic.client.bulk.{ + BulkElasticAction, + BulkItem, + BulkOptions, + FailedDocument, + SuccessfulDocument +} +import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.query.SQLAggregation +import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.schema.TableAlias + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions + +trait NopeClientApi extends ElasticClientApi { + + override private[client] def executeAddAlias( + alias: TableAlias + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeRemoveAlias( + index: String, + alias: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetAliases(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeSwapAlias( + oldIndex: String, + newIndex: String, + alias: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + /** Check if client is initialized and connected + */ + override def isInitialized: Boolean = true + + /** Test connection + * + * @return + * true if connection is successful + */ + override def testConnection(): Boolean = true + + override def close(): Unit = {} + + override private[client] def executeCount(query: ElasticQuery): ElasticResult[Option[Double]] = + ElasticResult.success(None) + + override private[client] def executeCountAsync( + query: ElasticQuery + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeDelete( + index: String, + id: String, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeFlush( + index: String, + force: Boolean, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeGet( + index: String, + id: String + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeGetAsync(index: String, id: String)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeIndex( + index: String, + id: String, + source: String, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeIndexAsync( + index: String, + id: String, + source: String, + wait: Boolean + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeReindex( + sourceIndex: String, + targetIndex: String, + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = ElasticResult.success((false, None)) + + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeSetMapping( + index: String, + mapping: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeGetMapping(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = + ElasticResult.success(false) + + /** Classic scroll (works for both hits and aggregations) + */ + override private[client] def scrollClassic( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + aggregations: Map[String, SQLAggregation], + config: ScrollConfig + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + /** Search After (only for hits, more efficient) + */ + override private[client] def searchAfter( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + config: ScrollConfig, + hasSorts: Boolean + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + override private[client] def pitSearchAfter( + elasticQuery: ElasticQuery, + fieldAliases: Map[String, String], + config: ScrollConfig, + hasSorts: Boolean + )(implicit system: ActorSystem): Source[Map[String, Any], NotUsed] = Source.empty + + override private[client] def executeSingleSearch( + elasticQuery: ElasticQuery + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeMultiSearch( + elasticQueries: ElasticQueries + ): ElasticResult[Option[String]] = ElasticResult.success(None) + + override private[client] def executeSingleSearchAsync(elasticQuery: ElasticQuery)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + override private[client] def executeMultiSearchAsync(elasticQueries: ElasticQueries)(implicit + ec: ExecutionContext + ): Future[ElasticResult[Option[String]]] = Future { + ElasticResult.success(None) + } + + /** Implicit conversion of an SQL query to Elasticsearch JSON. Used for query serialization. + * + * @param sqlSearch + * the SQL search request to convert + * @return + * JSON string representation of the query + */ + override private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: query.SingleSearch + )(implicit timestamp: Long): String = "{\"query\": {\"match_all\": {}}}" + + override private[client] def executeUpdateSettings( + index: String, + settings: String + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = + ElasticResult.success("{}") + + override private[client] def executeUpdate( + index: String, + id: String, + source: String, + upsert: Boolean, + wait: Boolean + ): ElasticResult[Boolean] = ElasticResult.success(false) + + override private[client] def executeUpdateAsync( + index: String, + id: String, + source: String, + upsert: Boolean, + wait: Boolean + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = Future { + ElasticResult.success(false) + } + + override private[client] def executeVersion(): ElasticResult[String] = + ElasticResult.success("0.0.0") + + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = + ElasticResult.success(None) + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = + ElasticResult.success(Map.empty) + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + ElasticResult.success(false) + + override private[client] def executeDeleteByQuery( + index: String, + query: String, + refresh: Boolean + ): ElasticResult[Long] = ElasticSuccess(0L) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + ElasticSuccess(false) + + override private[client] def executeUpdateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + ElasticSuccess(0L) + + override type BulkActionType = this.type + + override type BulkResultType = this.type + + override private[client] def toBulkAction(bulkItem: BulkItem): BulkActionType = + throw new UnsupportedOperationException + + override private[client] implicit def toBulkElasticAction(a: BulkActionType): BulkElasticAction = + throw new UnsupportedOperationException + + /** Basic flow for executing a bulk action. This method must be implemented by concrete classes + * depending on the Elasticsearch version and client used. + * + * @param bulkOptions + * configuration options + * @return + * Flow transforming bulk actions into results + */ + override private[client] def bulkFlow(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[BulkActionType], BulkResultType, NotUsed] = + throw new UnsupportedOperationException + + /** Convert a BulkResultType into individual results. This method must extract the successes and + * failures from the ES response. + * + * @param result + * raw result from the bulk + * @return + * sequence of Right(id) for success or Left(failed) for failure + */ + override private[client] def extractBulkResults( + result: BulkResultType, + originalBatch: Seq[BulkItem] + ): Seq[Either[FailedDocument, SuccessfulDocument]] = + throw new UnsupportedOperationException + + /** Conversion BulkActionType -> BulkItem */ + override private[client] def actionToBulkItem(action: BulkActionType): BulkItem = + throw new UnsupportedOperationException +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala new file mode 100644 index 00000000..67b253f5 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/PipelineApi.scala @@ -0,0 +1,363 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + AlterPipeline, + CreatePipeline, + DropPipeline, + PipelineStatement +} +import app.softnetwork.elastic.sql.schema.{GenericProcessor, IngestPipeline} + +trait PipelineApi extends ElasticClientHelpers { _: VersionApi => + + // ======================================================================== + // PIPELINE API + // ======================================================================== + + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ + def pipeline(sql: String): ElasticResult[Boolean] = { + ElasticResult.attempt(Parser(sql)) match { + case ElasticSuccess(parsedStatement) => + parsedStatement match { + + case Right(statement) => + statement match { + case ddl: PipelineStatement => + pipeline(ddl) + case _ => + val error = + ElasticError( + message = s"Unsupported pipeline DDL statement: $statement", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + case Left(l) => + val error = + ElasticError( + message = s"Error parsing pipeline DDL statement: ${l.msg}", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + case ElasticFailure(elasticError) => + ElasticResult.failure(elasticError.copy(operation = Some("pipeline"))) + } + } + + private[client] def pipeline(statement: PipelineStatement): ElasticResult[Boolean] = { + statement match { + case ddl: CreatePipeline => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = ddl.ddlPipeline.copy( + processors = ddl.ddlPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "description") + .filterNot(_._1 == "separator") + .filterNot(_._1 == "ignore_empty_value") + ) + } + ) + createPipeline(ddl.name, pipeline.json) + } else if (ElasticsearchVersion.isEs7(elasticVersion)) { + val pipeline = ddl.ddlPipeline.copy( + processors = ddl.ddlPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "separator") + ) + } + ) + createPipeline(ddl.name, pipeline.json) + } else + createPipeline(ddl.name, ddl.ddlPipeline.json) + case ddl: DropPipeline => + deletePipeline(ddl.name, ifExists = ddl.ifExists) + case ddl: AlterPipeline => + getPipeline(ddl.name) match { + case ElasticSuccess(Some(existing)) => + val existingPipeline = IngestPipeline(name = ddl.name, json = existing) + val updatingPipeline = existingPipeline.merge(ddl.statements) + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve Elasticsearch version: ${error.message}" + ) + return ElasticResult.failure(error) + } + } + if (ElasticsearchVersion.isEs6(elasticVersion)) { + val pipeline = updatingPipeline.copy( + processors = updatingPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "description") + .filterNot(_._1 == "separator") + .filterNot(_._1 == "ignore_empty_value") + ) + } + ) + updatePipeline(ddl.name, pipeline.json) + } else if (ElasticsearchVersion.isEs7(elasticVersion)) { + val pipeline = updatingPipeline.copy( + processors = updatingPipeline.processors.map { processor => + GenericProcessor( + processorType = processor.processorType, + properties = processor.properties + .filterNot(_._1 == "separator") + ) + } + ) + updatePipeline(ddl.name, pipeline.json) + } else + updatePipeline(ddl.name, updatingPipeline.json) + case ElasticSuccess(None) if !ddl.ifExists => + val error = + ElasticError( + message = s"Pipeline with name '${ddl.name}' not found", + statusCode = Some(404), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + case ElasticSuccess(None) if ddl.ifExists => + logger.info( + s"ℹ️ Pipeline with name '${ddl.name}' not found, skipping update as 'ifExists' is true" + ) + ElasticSuccess(false) + case failure @ ElasticFailure(error) => + logger.error( + s"❌ Failed to retrieve pipeline with name '${ddl.name}': ${error.message}" + ) + failure + } + case _ => + val error = + ElasticError( + message = s"Unsupported pipeline DDL statement: $statement", + statusCode = Some(400), + operation = Some("pipeline") + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + } + } + + /** Create a new ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + def createPipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { + validatePipelineName(pipelineName) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("createPipeline"), + statusCode = Some(400), + message = s"Invalid pipeline: ${error.message}" + ) + ) + case None => // OK + } + + validateJsonPipeline(pipelineDefinition) match { + case Some(error) => + return ElasticFailure( + error.copy( + operation = Some("createPipeline"), + statusCode = Some(400), + message = s"Invalid pipeline: ${error.message}" + ) + ) + case None => // OK + } + + executeCreatePipeline(pipelineName, pipelineDefinition) match { + case success @ ElasticSuccess(created) => + if (created) { + logger.info(s"βœ… Successfully created pipeline '$pipelineName'") + } else { + logger.warn(s"⚠️ Pipeline '$pipelineName' not created (it may already exist)") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to create pipeline '$pipelineName': ${error.message}") + failure + } + } + + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + def updatePipeline(pipelineName: String, pipelineDefinition: String): ElasticResult[Boolean] = { + // In Elasticsearch, creating a pipeline with an existing name updates it + createPipeline(pipelineName, pipelineDefinition) match { + case success @ ElasticSuccess(_) => success + case failure @ ElasticFailure(_) => failure + } + } + + /** Delete an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param ifExists + * flag indicating whether to ignore if the pipeline does not exist + * @return + * ElasticResult[Boolean] indicating success or failure + */ + def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = { + if (ifExists) { + getPipeline(pipelineName) match { + case ElasticSuccess(Some(_)) => // Pipeline exists, proceed to delete + case ElasticSuccess(None) => + logger.info( + s"ℹ️ Pipeline '$pipelineName' does not exist, skipping deletion as 'ifExists' is true" + ) + return ElasticSuccess(false) // Indicate that nothing was deleted + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to check existence of pipeline '$pipelineName': ${error.message}") + return failure + } + } + executeDeletePipeline(pipelineName, ifExists = ifExists) match { + case success @ ElasticSuccess(deleted) => + if (deleted) { + logger.info(s"βœ… Successfully deleted pipeline '$pipelineName'") + } else { + logger.warn(s"⚠️ Pipeline '$pipelineName' not deleted (it may not exist)") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to delete pipeline '$pipelineName': ${error.message}") + failure + } + } + + /** Retrieve an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @return + * ElasticResult[Option[String]\] containing the pipeline definition in JSON format if found + */ + def getPipeline(pipelineName: String): ElasticResult[Option[String]] = { + executeGetPipeline(pipelineName) match { + case success @ ElasticSuccess(maybePipeline) => + maybePipeline match { + case Some(_) => + logger.info(s"βœ… Successfully retrieved pipeline '$pipelineName'") + case None => + logger.warn(s"⚠️ Pipeline '$pipelineName' not found") + } + success + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve pipeline '$pipelineName': ${error.message}") + failure + } + } + + def loadPipeline(pipelineName: String): ElasticResult[IngestPipeline] = { + getPipeline(pipelineName) match { + case ElasticSuccess(Some(pipelineJson)) => + ElasticResult.attempt { + IngestPipeline(name = pipelineName, json = pipelineJson) + } match { + case success @ ElasticSuccess(_) => + success + case ElasticFailure(error) => + logger.error(s"❌ Failed to parse pipeline '$pipelineName': ${error.message}") + ElasticResult.failure( + error.copy(operation = Some("loadPipeline")) + ) + } + case ElasticSuccess(None) => + val error = + ElasticError.notFound( + resource = "Pipeline", + name = pipelineName, + operation = "loadPipeline" + ) + logger.error(s"❌ ${error.message}") + ElasticResult.failure(error) + case failure @ ElasticFailure(error) => + logger.error(s"❌ Failed to load pipeline '$pipelineName': ${error.message}") + failure + } + } + + // ======================================================================== + // METHODS TO IMPLEMENT + // ======================================================================== + + private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetPipeline(pipelineName: String): ElasticResult[Option[String]] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index 6d738565..74b3d44f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -29,7 +29,13 @@ import app.softnetwork.elastic.client.scroll.{ UseSearchAfter } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{ + DqlStatement, + MultiSearch, + SQLAggregation, + SelectStatement, + SingleSearch +} import org.json4s.{Formats, JNothing} import org.json4s.jackson.JsonMethods.parse @@ -117,33 +123,54 @@ trait ScrollApi extends ElasticClientHelpers { /** Create a scrolling source with automatic strategy selection */ def scroll( - sql: SQLQuery, + statement: DqlStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { - sql.request match { - case Some(Left(single)) => - if (single.windowFunctions.nonEmpty) - return scrollWithWindowEnrichment(sql, single, config) + implicit def timestamp: Long = System.currentTimeMillis() + statement match { + // Select statement + case select: SelectStatement => + select.statement match { + case Some(single: SingleSearch) => + scroll(single.copy(score = select.score), config) + + case Some(multiple: MultiSearch) => + scroll(multiple, config) + + case None => + Source.failed( + new IllegalArgumentException("SQL query does not contain a valid search request") + ) + } + + // Single search + case single: SingleSearch => + if ( + single.windowFunctions.exists(_.isWindowing) && (!single.select.fields.forall( + _.isAggregation + ) || single.scriptFields.nonEmpty) + ) + return scrollWithWindowEnrichment(single, config) - val sqlRequest = single.copy(score = sql.score) val elasticQuery = - ElasticQuery(sqlRequest, collection.immutable.Seq(sqlRequest.sources: _*)) + ElasticQuery(single, collection.immutable.Seq(single.sources: _*)) scrollWithMetrics( elasticQuery, - sqlRequest.fieldAliases, - sqlRequest.sqlAggregations, + single.fieldAliases, + single.sqlAggregations, config, single.sorts.nonEmpty ) - case Some(Right(_)) => + // Multi search + case _: MultiSearch => Source.failed( new UnsupportedOperationException("Scrolling is not supported for multi-search queries") ) - case None => + case _ => Source.failed( - new IllegalArgumentException("SQL query does not contain a valid search request") + new IllegalArgumentException("Scrolling is only supported for SELECT statements") ) } } @@ -224,7 +251,7 @@ trait ScrollApi extends ElasticClientHelpers { * - Source of tuples (T, ScrollMetrics) */ def scrollAsUnchecked[T]( - sql: SQLQuery, + sql: SelectStatement, config: ScrollConfig = ScrollConfig() )(implicit system: ActorSystem, @@ -376,10 +403,12 @@ trait ScrollApi extends ElasticClientHelpers { /** Scroll with window function enrichment */ private def scrollWithWindowEnrichment( - sql: SQLQuery, - request: SQLSearchRequest, + request: SingleSearch, config: ScrollConfig - )(implicit system: ActorSystem): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { + )(implicit + system: ActorSystem, + timestamp: Long + ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { implicit val ec: ExecutionContext = system.dispatcher @@ -390,7 +419,7 @@ trait ScrollApi extends ElasticClientHelpers { Future(executeWindowAggregations(request)) // Create base query without window functions - val baseQuery = createBaseQuery(sql, request) + val baseQuery = createBaseQuery(request) // Stream and enrich Source @@ -400,7 +429,8 @@ trait ScrollApi extends ElasticClientHelpers { scrollWithMetrics( ElasticQuery( baseQuery, - collection.immutable.Seq(baseQuery.sources: _*) + collection.immutable.Seq(baseQuery.sources: _*), + sql = Some(baseQuery.sql) ), baseQuery.fieldAliases, baseQuery.sqlAggregations, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 947ae75f..7977c791 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -23,7 +23,13 @@ import app.softnetwork.elastic.client.result.{ ElasticSuccess } import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{ + DqlStatement, + MultiSearch, + SQLAggregation, + SelectStatement, + SingleSearch +} import com.google.gson.{Gson, JsonElement, JsonObject, JsonParser} import org.json4s.Formats @@ -55,25 +61,50 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Search for documents / aggregations matching the SQL query. * - * @param sql + * @param statement * the SQL query to execute * @return * the Elasticsearch response */ - def search(sql: SQLQuery): ElasticResult[ElasticResponse] = { - sql.request match { - case Some(Left(single)) => + def search(statement: DqlStatement): ElasticResult[ElasticResponse] = { + implicit def timestamp: Long = System.currentTimeMillis() + val query = statement.sql + statement match { + case select: SelectStatement => + select.statement match { + case Some(statement: SingleSearch) => + search(statement.copy(score = select.score)) + case Some(statement: MultiSearch) => + search(statement) + case None => + logger.error( + s"❌ Failed to execute search for query \n${statement.sql}" + ) + ElasticResult.failure( + ElasticError( + message = s"SQL query does not contain a valid search request\n$query", + operation = Some("search") + ) + ) + } + case single: SingleSearch => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*), - sql = Some(sql.query) + sql = Some(query) + ) + if ( + single.windowFunctions.exists( + _.isWindowing + ) && single.groupBy.isEmpty && (!single.select.fields.forall( + _.isAggregation + ) || single.scriptFields.nonEmpty) ) - if (single.windowFunctions.exists(_.isWindowing) && single.groupBy.isEmpty) - searchWithWindowEnrichment(sql, single) + searchWithWindowEnrichment(single) else singleSearch(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(Right(multiple)) => + case multiple: MultiSearch => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -81,17 +112,17 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { collection.immutable.Seq(query.sources: _*) ) }.toList, - sql = Some(sql.query) + sql = Some(query) ) multiSearch(elasticQueries, multiple.fieldAliases, multiple.sqlAggregations) - case None => + case _ => logger.error( - s"❌ Failed to execute search for query \n${sql.query}" + s"❌ Failed to execute search for query \n${statement.sql}" ) ElasticResult.failure( ElasticError( - message = s"SQL query does not contain a valid search request\n${sql.query}", + message = s"SQL query does not contain a valid search request\n$query", operation = Some("search") ) ) @@ -295,19 +326,40 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * a Future containing the Elasticsearch response */ def searchAsync( - sqlQuery: SQLQuery + statement: DqlStatement )(implicit ec: ExecutionContext ): Future[ElasticResult[ElasticResponse]] = { - sqlQuery.request match { - case Some(Left(single)) => + implicit def timestamp: Long = System.currentTimeMillis() + statement match { + case select: SelectStatement => + select.statement match { + case Some(statement: SingleSearch) => + searchAsync(statement.copy(score = select.score)) + case Some(statement: MultiSearch) => + searchAsync(statement) + case None => + logger.error( + s"❌ Failed to execute asynchronous search for query '${statement.sql}'" + ) + Future.successful( + ElasticResult.failure( + ElasticError( + message = s"SQL query does not contain a valid search request: ${statement.sql}", + operation = Some("searchAsync") + ) + ) + ) + } + + case single: SingleSearch => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*) ) singleSearchAsync(elasticQuery, single.fieldAliases, single.sqlAggregations) - case Some(Right(multiple)) => + case multiple: MultiSearch => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -318,14 +370,15 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { ) multiSearchAsync(elasticQueries, multiple.fieldAliases, multiple.sqlAggregations) - case None => + case _ => + val query = statement.sql logger.error( - s"❌ Failed to execute asynchronous search for query '${sqlQuery.query}'" + s"❌ Failed to execute asynchronous search for query '$query'" ) Future.successful( ElasticResult.failure( ElasticError( - message = s"SQL query does not contain a valid search request: ${sqlQuery.query}", + message = s"SQL query does not contain a valid search request: $query", operation = Some("searchAsync") ) ) @@ -529,7 +582,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * the entities matching the query */ def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = { for { response <- search(sqlQuery) @@ -626,7 +679,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * a Future containing the entities */ def searchAsyncAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], ec: ExecutionContext, @@ -739,20 +792,21 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * tuples (main entity, inner hits) */ def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats ): ElasticResult[Seq[(U, Seq[I])]] = { - sql.request match { - case Some(Left(single)) => + implicit def timestamp: Long = System.currentTimeMillis() + sql.statement match { + case Some(single: SingleSearch) => val elasticQuery = ElasticQuery( single, collection.immutable.Seq(single.sources: _*) ) singleSearchWithInnerHits[U, I](elasticQuery, innerField) - case Some(Right(multiple)) => + case Some(multiple: MultiSearch) => val elasticQueries = ElasticQueries( multiple.requests.map { query => ElasticQuery( @@ -981,7 +1035,9 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * @return * JSON string representation of the query */ - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String + private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String private def parseInnerHits[M: Manifest: ClassTag, I: Manifest: ClassTag]( searchResult: JsonObject, @@ -1095,9 +1151,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * functions) 3. Enrich results with window values */ private def searchWithWindowEnrichment( - sql: SQLQuery, - request: SQLSearchRequest - ): ElasticResult[ElasticResponse] = { + request: SingleSearch + )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { logger.info(s"πŸͺŸ Detected ${request.windowFunctions.size} window functions") @@ -1106,7 +1161,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { windowCache <- executeWindowAggregations(request) // Step 2: Execute base query (without window functions) - baseResponse <- executeBaseQuery(sql, request) + baseResponse <- executeBaseQuery(request) // Step 3: Enrich results enrichedResponse <- enrichResponseWithWindowValues(baseResponse, windowCache, request) @@ -1122,8 +1177,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { * window values */ protected def executeWindowAggregations( - request: SQLSearchRequest - ): ElasticResult[WindowCache] = { + request: SingleSearch + )(implicit timestamp: Long): ElasticResult[WindowCache] = { // Build aggregation request val aggRequest = buildWindowAggregationRequest(request) @@ -1157,8 +1212,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Build aggregation request for window functions */ private def buildWindowAggregationRequest( - request: SQLSearchRequest - ): SQLSearchRequest = { + request: SingleSearch + ): SingleSearch = { // Create modified request with: // - Only window buckets in GROUP BY @@ -1167,7 +1222,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { // - Same WHERE clause (to match base query filtering) request .copy( - select = request.select.copy(fields = request.windowFields), + select = request.select.copy(fields = request.windowFields.map(_.update(request))), groupBy = None, //request.groupBy.map(_.copy(buckets = request.windowBuckets)), orderBy = None, // Not needed for aggregations limit = None // Need all buckets @@ -1180,7 +1235,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { */ private def parseWindowAggregationsToCache( response: ElasticResponse, - request: SQLSearchRequest + request: SingleSearch ): ElasticResult[WindowCache] = { logger.info( @@ -1209,11 +1264,10 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Execute base query without window functions */ private def executeBaseQuery( - sql: SQLQuery, - request: SQLSearchRequest - ): ElasticResult[ElasticResponse] = { + request: SingleSearch + )(implicit timestamp: Long): ElasticResult[ElasticResponse] = { - val baseQuery = createBaseQuery(sql, request) + val baseQuery = createBaseQuery(request) logger.info(s"πŸ” Executing base query without window functions ${baseQuery.sql}") @@ -1231,9 +1285,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { /** Create base query by removing window functions from SELECT */ protected def createBaseQuery( - sql: SQLQuery, - request: SQLSearchRequest - ): SQLSearchRequest = { + request: SingleSearch + ): SingleSearch = { // Remove window function fields from SELECT val baseFields = request.select.fields.filterNot(_.identifier.hasWindow) @@ -1243,7 +1296,6 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { .copy( select = request.select.copy(fields = baseFields) ) - .copy(score = sql.score) .update() baseRequest @@ -1253,7 +1305,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { */ private def extractPartitionKey( row: Map[String, Any], - request: SQLSearchRequest + request: SingleSearch ): PartitionKey = { // Get all partition fields from window functions @@ -1294,7 +1346,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { private def enrichResponseWithWindowValues( response: ElasticResponse, cache: WindowCache, - request: SQLSearchRequest + request: SingleSearch ): ElasticResult[ElasticResponse] = { val baseRows = response.results @@ -1311,7 +1363,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { protected def enrichDocumentWithWindowValues( doc: Map[String, Any], cache: WindowCache, - request: SQLSearchRequest + request: SingleSearch ): Map[String, Any] = { if (request.windowFunctions.isEmpty) { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala index ddefc61b..cd76abeb 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala @@ -129,7 +129,7 @@ trait SettingsApi { _: IndicesApi => def getRefreshInterval(index: String): ElasticResult[String] = { loadSettings(index).flatMap { settingsJson => ElasticResult.attempt( - new JsonParser().parse(settingsJson).getAsJsonObject + JsonParser.parseString(settingsJson).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON settings for index '$index': ${error.message}") @@ -192,7 +192,7 @@ trait SettingsApi { _: IndicesApi => executeLoadSettings(index).flatMap { jsonString => // βœ… Extracting settings from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON settings for index '$index': ${error.message}") diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala new file mode 100644 index 00000000..331c416b --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateApi.scala @@ -0,0 +1,283 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} + +import scala.util.{Failure, Success} + +/** API for managing Elasticsearch index templates. + * + * Automatically handles format conversion: + * - Legacy format β†’ Composable format (ES 7.8+) + * - Composable format β†’ Legacy format (ES < 7.8) + */ +trait TemplateApi extends ElasticClientHelpers { _: VersionApi => + + // ======================================================================== + // TEMPLATE API + // ======================================================================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + // Get Elasticsearch version + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + // Normalize template format based on ES version + val normalizedTemplate = TemplateConverter.normalizeTemplate( + templateDefinition, + elasticVersion + ) match { + case Success(json) => json + case Failure(ex) => + logger.error(s"❌ Failed to normalize template format: ${ex.getMessage}", ex) + return ElasticFailure( + ElasticError( + message = s"Invalid template format: ${ex.getMessage}", + statusCode = Some(400), + operation = Some("createTemplate"), + cause = Some(ex) + ) + ) + } + + // Determine which API to use + val isLegacyFormat = TemplateConverter.isLegacyFormat(templateDefinition) + val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(elasticVersion) + + if (supportsComposable) { + logger.info( + s"βœ… Using composable template API (ES $elasticVersion)" + + (if (isLegacyFormat) " [converted from legacy format]" else "") + ) + + validateJsonComposableTemplate(normalizedTemplate) match { + case Some(error) => ElasticFailure(error) + case None => executeCreateComposableTemplate(templateName, normalizedTemplate) + } + } else { + logger.info(s"⚠️ Using legacy template API (ES $elasticVersion < 7.8)") + + validateJsonLegacyTemplate(normalizedTemplate) match { + case Some(error) => ElasticFailure(error) + case None => executeCreateLegacyTemplate(templateName, normalizedTemplate) + } + } + } + } + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + def deleteTemplate( + templateName: String, + ifExists: Boolean = false + ): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + logger.debug(s"Using composable template API for deletion (ES $elasticVersion)") + executeDeleteComposableTemplate(templateName, ifExists) + } else { + logger.debug(s"Using legacy template API for deletion (ES $elasticVersion < 7.8)") + executeDeleteLegacyTemplate(templateName, ifExists) + } + } + } + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + def getTemplate(templateName: String): ElasticResult[Option[String]] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + (if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeGetComposableTemplate(templateName) + } else { + executeGetLegacyTemplate(templateName) + }) match { + case success @ ElasticSuccess(_) => success + case failure @ ElasticFailure(error) => + error.statusCode match { + case Some(404) => + logger.warn(s"⚠️ Template $templateName not found") + return ElasticSuccess(None) + case _ => + } + failure + } + } + } + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + def listTemplates(): ElasticResult[Map[String, String]] = { + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeListComposableTemplates() + } else { + executeListLegacyTemplates() + } + } + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + def templateExists(templateName: String): ElasticResult[Boolean] = { + validateTemplateName(templateName) match { + case Some(error) => ElasticFailure(error) + case None => + val elasticVersion = { + this.version match { + case ElasticSuccess(v) => v + case ElasticFailure(error) => + logger.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + return ElasticFailure(error) + } + } + + if (ElasticsearchVersion.supportsComposableTemplates(elasticVersion)) { + executeComposableTemplateExists(templateName) + } else { + executeLegacyTemplateExists(templateName) + } + } + } + + // ==================== ABSTRACT METHODS - COMPOSABLE (ES 7.8+) ==================== + + private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] + + private[client] def executeListComposableTemplates(): ElasticResult[Map[String, String]] + + private[client] def executeComposableTemplateExists(templateName: String): ElasticResult[Boolean] + + // ==================== ABSTRACT METHODS - LEGACY (ES < 7.8) ==================== + + private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] + + private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] + + private[client] def executeGetLegacyTemplate(templateName: String): ElasticResult[Option[String]] + + private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] + + private[client] def executeLegacyTemplateExists(templateName: String): ElasticResult[Boolean] + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala new file mode 100644 index 00000000..6a5ba13a --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/TemplateConverter.scala @@ -0,0 +1,318 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.{Logger, LoggerFactory} + +import scala.util.{Success, Try} + +object TemplateConverter { + + private val logger: Logger = LoggerFactory.getLogger(getClass) + + private lazy val mapper: ObjectMapper = JacksonConfig.objectMapper + + /** Normalize template format based on Elasticsearch version + * + * @param templateJson + * the template JSON (legacy or composable format) + * @param elasticVersion + * the Elasticsearch version (e.g., "7.10.0") + * @return + * normalized template JSON + */ + def normalizeTemplate(templateJson: String, elasticVersion: String): Try[String] = { + val supportsComposable = ElasticsearchVersion.supportsComposableTemplates(elasticVersion) + val isLegacy = isLegacyFormat(templateJson) + + (supportsComposable, isLegacy) match { + case (true, true) => + // ES 7.8+ with legacy format β†’ convert to composable + logger.debug("Converting legacy template to composable format") + convertLegacyToComposable(templateJson) + + case (false, _) => + // ES < 7.8 with composable format β†’ convert to legacy + // ES < 6.8 with legacy format β†’ ensure _doc wrapper in mappings + logger.debug( + "Converting composable template to legacy format for ES < 7.8 / Ensuring _doc wrapper in legacy template mappings for ES < 6.8" + ) + convertComposableToLegacy(templateJson, elasticVersion) + + case _ => + // Already in correct format + logger.debug("Template already in correct format") + Success(templateJson) + } + } + + /** Check if template is in legacy format + * + * Legacy format has 'order' field, composable has 'priority' + * + * @param templateJson + * the template JSON + * @return + * true if legacy format + */ + def isLegacyFormat(templateJson: String): Boolean = { + Try { + val root = mapper.readTree(templateJson) + + // Legacy format: has settings/mappings/aliases at root + val hasRootSettings = root.has("settings") + val hasRootMappings = root.has("mappings") + val hasRootAliases = root.has("aliases") + + // Composable format: has 'template' key containing settings/mappings/aliases + val hasTemplateKey = root.has("template") + + // Legacy if has root-level settings/mappings/aliases AND no 'template' key + (hasRootSettings || hasRootMappings || hasRootAliases) && !hasTemplateKey + + }.getOrElse(false) + } + + /** Convert legacy template format to composable template format + * + * Legacy format: + * {{{ + * { + * "index_patterns": ["logs-*"], + * "order": 1, + * "settings": {...}, + * "mappings": {...}, + * "aliases": {...} + * } + * }}} + * + * Composable format: + * {{{ + * { + * "index_patterns": ["logs-*"], + * "priority": 1, + * "template": { + * "settings": {...}, + * "mappings": {...}, + * "aliases": {...} + * } + * } + * }}} + * + * @param legacyJson + * the legacy template JSON + * @return + * composable template JSON + */ + def convertLegacyToComposable(legacyJson: String): Try[String] = { + Try { + val root = mapper.readTree(legacyJson).asInstanceOf[ObjectNode] + val composable = mapper.createObjectNode() + + // 1. Copy index_patterns (required) + if (root.has("index_patterns")) { + composable.set("index_patterns", root.get("index_patterns")) + } else if (root.has("template")) { + // Old legacy format used 'template' instead of 'index_patterns' + val patterns = mapper.createArrayNode() + patterns.add(root.get("template").asText()) + composable.set("index_patterns", patterns) + } + + // 2. Convert 'order' to 'priority' + if (root.has("order")) { + composable.put("priority", root.get("order").asInt()) + } else { + composable.put("priority", 1) // Default priority + } + + // 3. Copy version if present + if (root.has("version")) { + composable.set("version", root.get("version")) + } + + // 4. Wrap settings, mappings, aliases in 'template' object + val templateNode = mapper.createObjectNode() + + if (root.has("settings")) { + templateNode.set("settings", root.get("settings")) + } + + if (root.has("mappings")) { + // Remove _doc wrapper if present (legacy ES 6.x format) + val mappingsNode = root.get("mappings") + if (mappingsNode.isObject && mappingsNode.has("_doc")) { + templateNode.set("mappings", mappingsNode.get("_doc")) + } else { + templateNode.set("mappings", mappingsNode) + } + } + + if (root.has("aliases")) { + templateNode.set("aliases", root.get("aliases")) + } + + composable.set("template", templateNode) + + // 5. Copy _meta if present (valid in composable format) + if (root.has("_meta")) { + composable.set("_meta", root.get("_meta")) + } + + mapper.writeValueAsString(composable) + } + } + + /** Convert composable template format to legacy template format + * + * Used for backward compatibility when ES version < 7.8 + * + * @param composableJson + * the composable template JSON + * @param elasticVersion + * the Elasticsearch version (e.g., "6.8.0") + * @return + * legacy template JSON + */ + def convertComposableToLegacy( + composableJson: String, + elasticVersion: String + ): Try[String] = { + Try { + val root = mapper.readTree(composableJson).asInstanceOf[ObjectNode] + val legacy = mapper.createObjectNode() + + // 1. Copy index_patterns (required) + if (root.has("index_patterns")) { + legacy.set("index_patterns", root.get("index_patterns")) + } + + // 2. Convert 'priority' to 'order' + if (root.has("priority")) { + legacy.put("order", root.get("priority").asInt()) + } else if (root.has("order")) { + // In case 'order' is already present + legacy.put("order", root.get("order").asInt()) + } else { + legacy.put("order", 1) // Default order + } + + // 3. Copy version if present + if (root.has("version")) { + legacy.set("version", root.get("version")) + } + + // 4. ❌ DO NOT copy _meta (not supported in legacy format) + // Legacy templates don't support _meta field + + // 5. Flatten 'template' object to root level + if (root.has("template")) { + val templateNode = root.get("template") + + if (templateNode.has("settings")) { + legacy.set("settings", templateNode.get("settings")) + } + + if (templateNode.has("mappings")) { + val mappingsNode = templateNode.get("mappings").asInstanceOf[ObjectNode] + legacy.set("mappings", MappingConverter.convert(mappingsNode, elasticVersion)) + } + + if (templateNode.has("aliases")) { + legacy.set("aliases", templateNode.get("aliases")) + } + } + + if (root.has("settings")) { + legacy.set("settings", root.get("settings")) + } + + if (root.has("mappings")) { + val mappingsNode = root.get("mappings") + + // Check if we need to wrap in _doc for ES 6.x + if ( + ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && !mappingsNode.has("_doc") + ) { + val wrappedMappings = mapper.createObjectNode() + wrappedMappings.set("_doc", mappingsNode) + legacy.set("mappings", wrappedMappings) + logger.debug(s"Wrapped mappings in '_doc' for ES $elasticVersion") + } else if ( + !ElasticsearchVersion.requiresDocTypeWrapper(elasticVersion) && mappingsNode.has("_doc") + ) { + legacy.set("mappings", mappingsNode.get("_doc")) + } else { + legacy.set("mappings", mappingsNode) + } + } + + if (root.has("aliases")) { + legacy.set("aliases", root.get("aliases")) + } + + // Note: 'composed_of' and 'data_stream' are not supported in legacy format + // They will be silently ignored + if (root.has("composed_of")) { + logger.warn( + "Composable templates are not supported in legacy template format and will be ignored" + ) + } + if (root.has("data_stream")) { + logger.warn("Data streams are not supported in legacy template format and will be ignored") + } + + mapper.writeValueAsString(legacy) + } + } + + /** Validate that a template has required fields + * + * @param templateJson + * the template JSON + * @return + * None if valid, Some(error message) if invalid + */ + def validateTemplate(templateJson: String): Option[String] = { + Try { + val root = mapper.readTree(templateJson) + + // Check for index_patterns (required in both formats) + if (!root.has("index_patterns")) { + return Some("Missing required field: index_patterns") + } + + val indexPatterns = root.get("index_patterns") + if (!indexPatterns.isArray || indexPatterns.size() == 0) { + return Some("index_patterns must be a non-empty array") + } + + // Check for either priority (composable) or order (legacy) + if (!root.has("priority") && !root.has("order")) { + logger.warn("Template missing both 'priority' and 'order', will use default") + } + + None + }.recover { case e: Exception => + Some(s"Invalid JSON: ${e.getMessage}") + }.get + } + +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala index d7aef09e..2758373e 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/file/package.scala @@ -18,17 +18,11 @@ package app.softnetwork.elastic.client import akka.NotUsed import akka.stream.scaladsl.Source -import com.fasterxml.jackson.annotation.JsonInclude +import app.softnetwork.elastic.sql.query.{Delta, FileFormat, Json, JsonArray, Parquet, Unknown} +import app.softnetwork.elastic.sql.serialization.JacksonConfig import com.fasterxml.jackson.core.{JsonFactory, JsonParser, JsonToken} import com.fasterxml.jackson.databind.node.ObjectNode -import com.fasterxml.jackson.databind.{ - DeserializationFeature, - JsonNode, - ObjectMapper, - SerializationFeature -} -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} import org.apache.avro.generic.GenericRecord import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} @@ -51,30 +45,6 @@ package object file { private val logger: Logger = LoggerFactory.getLogger("FileSource") - sealed trait FileFormat { - def name: String - } - - case object Parquet extends FileFormat { - override def name: String = "Parquet" - } - - case object Json extends FileFormat { - override def name: String = "JSON" - } - - case object JsonArray extends FileFormat { - override def name: String = "JSON Array" - } - - case object Delta extends FileFormat { - override def name: String = "Delta Lake" - } - - case object Unknown extends FileFormat { - override def name: String = "Unknown" - } - /** Hadoop configuration with optimizations for local file system */ def hadoopConfiguration: Configuration = { val conf = new Configuration() @@ -86,33 +56,6 @@ package object file { conf } - /** Jackson ObjectMapper configuration */ - object JacksonConfig { - lazy val objectMapper: ObjectMapper = { - val mapper = new ObjectMapper() - - // Scala module for native support of Scala types - mapper.registerModule(DefaultScalaModule) - - // Java Time module for java.time.Instant, LocalDateTime, etc. - mapper.registerModule(new JavaTimeModule()) - - // Setup for performance - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) - mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) - - // Ignores null values in serialization - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) - - // Optimizations - mapper.configure(SerializationFeature.INDENT_OUTPUT, false) // No pretty print - mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, false) - - mapper - } - } - /** Base trait for file sources */ sealed trait FileSource { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 8a16dabd..7b78425f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -30,7 +30,10 @@ import app.softnetwork.elastic.client.{ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result._ import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLQuery} +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.{query, schema} +import app.softnetwork.elastic.sql.query.{DqlStatement, SQLAggregation, SelectStatement} +import app.softnetwork.elastic.sql.schema.{Schema, TableAlias} import org.json4s.Formats import scala.concurrent.{ExecutionContext, Future} @@ -87,12 +90,29 @@ class MetricsElasticClient( // ==================== IndicesApi ==================== - override def createIndex(index: String, settings: String): ElasticResult[Boolean] = { + override def createIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = { measureResult("createIndex", Some(index)) { - delegate.createIndex(index, settings) + delegate.createIndex(index, settings, mappings, aliases) } } + /** Get an index with the provided name. + * + * @param index + * - the name of the index to get + * @return + * the index if it exists, None otherwise + */ + override def getIndex(index: String): ElasticResult[Option[Index]] = + measureResult("getIndex", Some(index)) { + delegate.getIndex(index) + } + override def deleteIndex(index: String): ElasticResult[Boolean] = { measureResult("deleteIndex", Some(index)) { delegate.deleteIndex(index) @@ -114,19 +134,107 @@ class MetricsElasticClient( override def reindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { measureResult("reindex", Some(s"$sourceIndex->$targetIndex")) { - delegate.reindex(sourceIndex, targetIndex, refresh) + delegate.reindex(sourceIndex, targetIndex, refresh, pipeline) } } - override def indexExists(index: String): ElasticResult[Boolean] = { + override def indexExists(index: String, pattern: Boolean): ElasticResult[Boolean] = { measureResult("indexExists", Some(index)) { - delegate.indexExists(index) + delegate.indexExists(index, pattern) + } + } + + /** Truncate an index by deleting all its documents. + * + * @param index + * - the name of the index to truncate + * @return + * the number of documents deleted + */ + override def truncateIndex(index: String): ElasticResult[Long] = + measureResult("truncate", Some(index)) { + delegate.truncateIndex(index) + } + + /** Delete documents by query from an index. + * + * @param index + * - the name of the index to delete from + * @param query + * - the query to delete documents by (can be JSON or SQL) + * @param refresh + * - true to refresh the index after deletion, false otherwise + * @return + * the number of documents deleted + */ + override def deleteByQuery(index: String, query: String, refresh: Boolean): ElasticResult[Long] = + measureResult("deleteByQuery", Some(index)) { + delegate.deleteByQuery(index, query, refresh) + } + + override def isIndexClosed(index: String): ElasticResult[Boolean] = + measureResult("isIndexClosed", Some(index)) { + delegate.isIndexClosed(index) + } + + /** Update documents by query from an index. + * + * @param index + * - the name of the index to update + * @param query + * - the query to update documents by (can be JSON or SQL) + * @param pipelineId + * - optional ingest pipeline id to use for the update + * @param refresh + * - true to refresh the index after update, false otherwise + * @return + * the number of documents updated + */ + override def updateByQuery( + index: String, + query: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = + measureResult("updateByQuery", Some(index)) { + delegate.updateByQuery(index, query, pipelineId, refresh) } + + /** Insert documents by query into an index. + * + * @param index + * - the name of the index to insert into + * @param query + * - the query to insert documents from (can be SQL INSERT ... VALUES or INSERT ... AS SELECT) + * @param refresh + * - true to refresh the index after insertion, false otherwise + * @return + * the number of documents inserted + */ + override def insertByQuery(index: String, query: String, refresh: Boolean)(implicit + system: ActorSystem + ): Future[ElasticResult[DmlResult]] = { + measureAsync("insertByQuery", Some(index)) { + delegate.insertByQuery(index, query, refresh) + }(system.dispatcher) } + /** Load the schema for the provided index. + * + * @param index + * - the name of the index to load the schema for + * @return + * the schema if the index exists, an error otherwise + */ + override def loadSchema(index: String): ElasticResult[Schema] = + measureResult("loadSchema", Some(index.indices.mkString(","))) { + delegate.loadSchema(index) + } + // ==================== AliasApi ==================== override def addAlias(index: String, alias: String): ElasticResult[Boolean] = { @@ -171,13 +279,13 @@ class MetricsElasticClient( * @example * {{{ * getAliases("my-index") match { - * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.mkString(", ")}") + * case ElasticSuccess(aliases) => println(s"Aliases: ${aliases.map(_.alias).mkString(", ")}") * case ElasticFailure(error) => println(s"Error: ${error.message}") * } * * }}} */ - override def getAliases(index: String): ElasticResult[Set[String]] = + override def getAliases(index: String): ElasticResult[Seq[TableAlias]] = measureResult("getAliases", Some(index)) { delegate.getAliases(index) } @@ -216,6 +324,30 @@ class MetricsElasticClient( delegate.swapAlias(oldIndex, newIndex, alias) } + /** Set the exact set of aliases for an index. + * + * This method ensures that the specified index has exactly the provided set of aliases. It adds + * any missing aliases and removes any extra aliases that are not in the provided set. + * + * @param index + * the name of the index + * @param aliases + * the desired set of aliases for the index + * @return + * ElasticSuccess(true) if the operation was successful, ElasticFailure otherwise + * @example + * {{{ + * setAliases("my-index", Set("alias1", "alias2")) match { + * case ElasticSuccess(_) => println("Aliases set successfully") + * case ElasticFailure(error) => println(s"Error: ${error.message}") + * } + * }}} + */ + override def setAliases(index: String, aliases: Seq[TableAlias]): ElasticResult[Boolean] = + measureResult("setAliases", Some(index)) { + delegate.setAliases(index, aliases) + } + // ==================== SettingsApi ==================== override def updateSettings(index: String, settings: String): ElasticResult[Boolean] = { @@ -619,7 +751,7 @@ class MetricsElasticClient( * @return * a sequence of aggregated results */ - override def aggregate(sqlQuery: SQLQuery)(implicit + override def aggregate(sqlQuery: SelectStatement)(implicit ec: ExecutionContext ): Future[ElasticResult[collection.Seq[SingleValueAggregateResult]]] = measureAsync("aggregate") { @@ -635,9 +767,9 @@ class MetricsElasticClient( * @return * the Elasticsearch response */ - override def search(sql: SQLQuery): ElasticResult[ElasticResponse] = + override def search(statement: DqlStatement): ElasticResult[ElasticResponse] = measureResult("search") { - delegate.search(sql) + delegate.search(statement) } /** Asynchronous search for documents / aggregations matching the SQL query. @@ -648,10 +780,10 @@ class MetricsElasticClient( * a Future containing the Elasticsearch response */ override def searchAsync( - sqlQuery: SQLQuery + statement: DqlStatement )(implicit ec: ExecutionContext): Future[ElasticResult[ElasticResponse]] = measureAsync("searchAsync") { - delegate.searchAsync(sqlQuery) + delegate.searchAsync(statement) } /** Searches and converts results into typed entities from an SQL query. @@ -664,7 +796,7 @@ class MetricsElasticClient( * the entities matching the query */ override def searchAsUnchecked[U]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[U], formats: Formats): ElasticResult[Seq[U]] = measureResult("searchAs") { delegate.searchAsUnchecked[U](sqlQuery) @@ -682,7 +814,7 @@ class MetricsElasticClient( * @return * a Future containing the entities */ - override def searchAsyncAsUnchecked[U](sqlQuery: SQLQuery)(implicit + override def searchAsyncAsUnchecked[U](sqlQuery: SelectStatement)(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats @@ -848,7 +980,7 @@ class MetricsElasticClient( } override def searchWithInnerHits[U: Manifest: ClassTag, I: Manifest: ClassTag]( - sql: SQLQuery, + sql: SelectStatement, innerField: String )(implicit formats: Formats @@ -881,12 +1013,12 @@ class MetricsElasticClient( /** Create a scrolling source with automatic strategy selection */ - override def scroll(sql: SQLQuery, config: ScrollConfig)(implicit + override def scroll(statement: DqlStatement, config: ScrollConfig)(implicit system: ActorSystem ): Source[(Map[String, Any], ScrollMetrics), NotUsed] = { // Note: For streams, we measure at the beginning but not every element val startTime = System.currentTimeMillis() - val source = delegate.scroll(sql, config) + val source = delegate.scroll(statement, config) source.watchTermination() { (_, done) => done.onComplete { result => @@ -922,7 +1054,7 @@ class MetricsElasticClient( * @return * - Source of tuples (T, ScrollMetrics) */ - override def scrollAsUnchecked[T](sql: SQLQuery, config: ScrollConfig)(implicit + override def scrollAsUnchecked[T](sql: SelectStatement, config: ScrollConfig)(implicit system: ActorSystem, m: Manifest[T], formats: Formats @@ -951,7 +1083,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -980,7 +1112,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -1021,7 +1153,7 @@ class MetricsElasticClient( items: Source[D, NotUsed], toDocument: D => String, indexKey: Option[String] = None, - idKey: Option[String] = None, + idKey: Option[Set[String]] = None, suffixDateKey: Option[String] = None, suffixDatePattern: Option[String] = None, update: Option[Boolean] = None, @@ -1071,4 +1203,166 @@ class MetricsElasticClient( override def resetMetrics(): Unit = { metricsCollector.resetMetrics() } + + // ==================== PipelineApi (delegate) ==================== + + /** Execute a pipeline DDL statement + * + * @param sql + * the pipeline DDL statement + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def pipeline(sql: String): ElasticResult[Boolean] = + measureResult("pipeline") { + delegate.pipeline(sql) + } + + override private[client] def pipeline( + statement: query.PipelineStatement + ): ElasticResult[Boolean] = + measureResult("pipeline") { + delegate.pipeline(statement) + } + + override def createPipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + measureResult("createPipeline") { + delegate.createPipeline(pipelineName, pipelineDefinition) + } + + /** Update an existing ingest pipeline + * + * @param pipelineName + * the name of the pipeline + * @param pipelineDefinition + * the new pipeline definition in JSON format + * @return + * ElasticResult[Boolean] indicating success or failure + */ + override def updatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + measureResult("updatePipeline") { + delegate.updatePipeline(pipelineName, pipelineDefinition) + } + + override def deletePipeline(pipelineName: String, ifExists: Boolean): ElasticResult[Boolean] = + measureResult("deletePipeline") { + delegate.deletePipeline(pipelineName, ifExists = ifExists) + } + + override def getPipeline(pipelineName: String): ElasticResult[Option[String]] = + measureResult("getPipeline") { + delegate.getPipeline(pipelineName) + } + + override def loadPipeline(pipelineName: String): ElasticResult[schema.IngestPipeline] = + measureResult("loadPipeline") { + delegate.loadPipeline(pipelineName) + } + + // ==================== TemplateApi (delegate) ==================== + + /** Create or update an index template. + * + * Accepts both legacy and composable template formats. Automatically converts to the appropriate + * format based on ES version. + * + * @param templateName + * the name of the template + * @param templateDefinition + * the JSON definition (legacy or composable format) + * @return + * ElasticResult with true if successful + */ + override def createTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + measureResult("createTemplate") { + delegate.createTemplate(templateName, templateDefinition) + } + + /** Delete an index template. Automatically uses composable (ES 7.8+) or legacy templates based on + * ES version. + * + * @param templateName + * the name of the template to delete + * @param ifExists + * if true, do not fail if template doesn't exist + * @return + * ElasticResult with true if successful + */ + override def deleteTemplate(templateName: String, ifExists: Boolean): ElasticResult[Boolean] = + measureResult("deleteTemplate") { + delegate.deleteTemplate(templateName, ifExists) + } + + /** Get an index template definition. + * + * Returns the template in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @param templateName + * the name of the template + * @return + * ElasticResult with Some(json) if found, None if not found + */ + override def getTemplate(templateName: String): ElasticResult[Option[String]] = + measureResult("getTemplate") { + delegate.getTemplate(templateName) + } + + /** List all index templates. + * + * Returns templates in the format used by the current ES version: + * - Composable format for ES 7.8+ + * - Legacy format for ES < 7.8 + * + * @return + * ElasticResult with Map of template name -> JSON definition + */ + override def listTemplates(): ElasticResult[Map[String, String]] = + measureResult("listTemplates") { + delegate.listTemplates() + } + + /** Check if an index template exists. Automatically uses composable (ES 7.8+) or legacy templates + * based on ES version. + * + * @param templateName + * the name of the template + * @return + * ElasticResult with true if exists, false otherwise + */ + override def templateExists(templateName: String): ElasticResult[Boolean] = + measureResult("templateExists") { + delegate.templateExists(templateName) + } + + // ==================== Gateway (delegate) ==================== + + override def run( + sql: String + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + measureAsync("run") { + delegate.run(sql) + } + } + + override def run( + statement: query.Statement + )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { + implicit val ec: ExecutionContext = system.dispatcher + measureAsync("run") { + delegate.run(statement) + } + } + } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala index 23861a22..3582acfe 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/result/package.scala @@ -16,6 +16,11 @@ package app.softnetwork.elastic.client +import akka.NotUsed +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.scroll.ScrollMetrics +import app.softnetwork.elastic.sql.schema.{IngestPipeline, Table} + import scala.util.control.NonFatal package object result { @@ -48,6 +53,13 @@ package object result { /** Converts to Either */ def toEither: Either[ElasticError, T] + /** Converts to Future */ + def toFuture(implicit + ec: scala.concurrent.ExecutionContext + ): scala.concurrent.Future[T] = { + scala.concurrent.Future.fromTry(this.toEither.toTry) + } + /** Fold pattern matching */ def fold[U](onFailure: ElasticError => U, onSuccess: T => U): U @@ -116,7 +128,9 @@ package object result { /** Represents a failed operation. */ - case class ElasticFailure(elasticError: ElasticError) extends ElasticResult[Nothing] { + case class ElasticFailure(elasticError: ElasticError) + extends Throwable(elasticError.message, elasticError) + with ElasticResult[Nothing] { override def isSuccess: Boolean = false override def map[U](f: Nothing => U): ElasticResult[U] = this @@ -141,6 +155,8 @@ package object result { override def get: Nothing = throw new NoSuchElementException( s"ElasticFailure.get: ${elasticError.message}" ) + + def isNotFound: Boolean = elasticError.statusCode.contains(404) } /** Represents an Elasticsearch error. @@ -151,7 +167,7 @@ package object result { statusCode: Option[Int] = None, index: Option[String] = None, operation: Option[String] = None - ) { + ) extends Throwable(message, cause.orNull) { /** Complete message with context */ def fullMessage: String = { @@ -174,6 +190,43 @@ package object result { } } + object ElasticError { + + /** Creates an ElasticError from an exception */ + def fromThrowable( + ex: Throwable, + statusCode: Option[Int] = None, + index: Option[String] = None, + operation: Option[String] = None + ): ElasticError = { + ElasticError( + ex.getMessage, + Some(ex), + statusCode, + index, + operation + ) + } + + /** Creates a not found error */ + def notFound(index: String, operation: String): ElasticError = { + ElasticError( + s"Resource not found in index '$index' during operation '$operation'", + statusCode = Some(404), + index = Some(index), + operation = Some(operation) + ) + } + + def notFound(resource: String, name: String, operation: String): ElasticError = { + ElasticError( + s"$resource '$name' not found during operation '$operation'", + statusCode = Some(404), + operation = Some(operation) + ) + } + } + /** Companion object with utility methods. */ object ElasticResult { @@ -330,4 +383,45 @@ package object result { } } } + + sealed trait QueryResult + + case object EmptyResult extends QueryResult + + object QueryResult { + def empty: QueryResult = EmptyResult + } + + // -------------------- + // DQL (SELECT) + // -------------------- + case class QueryRows(rows: Seq[Map[String, Any]]) extends QueryResult + + case class QueryStream( + stream: Source[(Map[String, Any], ScrollMetrics), NotUsed] + ) extends QueryResult + + case class QueryStructured(response: ElasticResponse) extends QueryResult + + // -------------------- + // DML (INSERT / UPDATE / DELETE) + // -------------------- + case class DmlResult( + inserted: Long = 0L, + updated: Long = 0L, + deleted: Long = 0L, + rejected: Long = 0L + ) extends QueryResult + + // -------------------- + // DDL (CREATE / ALTER / DROP / TRUNCATE) + // -------------------- + case class DdlResult(success: Boolean) extends QueryResult + + case class TableResult(table: Table) extends QueryResult + + case class PipelineResult(pipeline: IngestPipeline) extends QueryResult + + case class SQLResult(sql: String) extends QueryResult + } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala index 5ea8bb8f..2cd72c1c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/AliasApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for AliasApi */ @@ -21,7 +22,7 @@ class AliasApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestAliasApi extends AliasApi with IndicesApi with RefreshApi { + class TestAliasApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -33,10 +34,10 @@ class AliasApiSpec ) var executeSwapAliasResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeIndexExistsResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = { executeAddAliasResult } @@ -68,20 +69,10 @@ class AliasApiSpec executeIndexExistsResult } - // Other required methods - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeGetIndexResult + } + } var aliasApi: TestAliasApi = _ @@ -518,10 +509,10 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1", "alias2") + result.get.map(_.alias).toSet shouldBe Set("alias1", "alias2") verify(mockLogger).debug("Getting aliases for index 'my-index'") - verify(mockLogger).debug("Found 2 alias(es) for index 'my-index': alias1, alias2") +// verify(mockLogger).debug("Found 2 alias(es) for index 'my-index': alias1, alias2") verify(mockLogger).info("βœ… Found 2 alias(es) for index 'my-index': alias1, alias2") } @@ -535,9 +526,10 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty - verify(mockLogger).debug("No aliases found for index 'my-index'") + verify(mockLogger).debug("Getting aliases for index 'my-index'") +// verify(mockLogger).debug("No aliases found for index 'my-index'") verify(mockLogger).info("βœ… No aliases found for index 'my-index'") } @@ -551,7 +543,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "return empty set when index not found in response" in { @@ -563,7 +555,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty verify(mockLogger).warn("Index 'my-index' not found in response") } @@ -629,7 +621,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1", "alias2", "alias3") + result.get.map(_.alias).toSet shouldBe Set("alias1", "alias2", "alias3") } "handle single alias" in { @@ -642,7 +634,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("single-alias") + result.get.map(_.alias).toSet shouldBe Set("single-alias") verify(mockLogger).info(contains("Found 1 alias(es)")) } @@ -664,7 +656,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set("alias1") + result.get.map(_.alias).toSet shouldBe Set("alias1") } } @@ -912,7 +904,7 @@ class AliasApiSpec add1.isSuccess shouldBe true add2.isSuccess shouldBe true getResult.isSuccess shouldBe true - getResult.get should contain allOf ("alias1", "alias2") + getResult.get.map(_.alias) should contain allOf ("alias1", "alias2") } "verify alias existence before and after removal" in { @@ -981,7 +973,7 @@ class AliasApiSpec val result = for { _ <- aliasApi.addAlias("my-index", "my-alias") aliases <- aliasApi.getAliases("my-index") - } yield aliases.contains("my-alias") + } yield aliases.map(_.alias).contains("my-alias") // Then result shouldBe ElasticSuccess(true) @@ -1108,7 +1100,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "handle JSON with unexpected structure" in { @@ -1120,7 +1112,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty } "handle multiple consecutive swapAlias calls" in { @@ -1454,7 +1446,8 @@ class AliasApiSpec aliasApi.getAliases("my-index") // Then - verify(mockLogger).debug(contains("Found 2 alias(es)")) + verify(mockLogger).debug("Getting aliases for index 'my-index'") +// verify(mockLogger).debug(contains("Found 2 alias(es)")) verify(mockLogger).info(contains("βœ… Found 2 alias(es)")) } @@ -1626,12 +1619,11 @@ class AliasApiSpec "not call execute methods when validation fails" in { // Given var executeCalled = false - val validatingApi = new AliasApi with IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) @@ -1641,33 +1633,6 @@ class AliasApiSpec ElasticSuccess(true) } - override private[client] def executeRemoveAlias( - index: String, - alias: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = - ??? - override private[client] def executeGetAliases(index: String): ElasticResult[String] = ??? - override private[client] def executeSwapAlias( - oldIndex: String, - newIndex: String, - alias: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1703,7 +1668,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("filtered-alias") + result.get.map(_.alias) should contain("filtered-alias") } "correctly parse aliases with routing" in { @@ -1725,7 +1690,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("routed-alias") + result.get.map(_.alias) should contain("routed-alias") } "correctly parse aliases with search and index routing" in { @@ -1748,7 +1713,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("complex-alias") + result.get.map(_.alias) should contain("complex-alias") } "handle null values in JSON" in { @@ -1761,7 +1726,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("my-alias") + result.get.map(_.alias) should contain("my-alias") } "handle empty aliases object" in { @@ -1774,7 +1739,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get.map(_.alias) shouldBe Seq.empty } "handle malformed JSON gracefully" in { @@ -1808,7 +1773,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("my-alias") + result.get.map(_.alias) should contain("my-alias") } "handle deeply nested JSON structure" in { @@ -1837,7 +1802,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get should contain("alias1") + result.get.map(_.alias) should contain("alias1") } "handle JSON with unicode characters" in { @@ -1957,7 +1922,7 @@ class AliasApiSpec // Then addCurrent.isSuccess shouldBe true addMonth.isSuccess shouldBe true - getAliases.get should contain allOf ("logs-current", "logs-january") + getAliases.get.map(_.alias) should contain allOf ("logs-current", "logs-january") } "support filtered alias for multi-tenancy" in { @@ -2154,7 +2119,7 @@ class AliasApiSpec // Then result.isSuccess shouldBe true - result.get shouldBe Set.empty + result.get shouldBe Seq.empty result.get.isEmpty shouldBe true } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala index cfd8cc50..d665a82c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/IndicesApiSpec.scala @@ -7,6 +7,7 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for IndicesApi */ @@ -21,11 +22,12 @@ class IndicesApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestIndicesApi extends IndicesApi with RefreshApi { + class TestIndicesApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables for each operation var executeCreateIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) var executeDeleteIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeCloseIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeOpenIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) @@ -37,11 +39,17 @@ class IndicesApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCreateIndexResult } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeGetIndexResult + } + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = { executeDeleteIndexResult } @@ -57,7 +65,8 @@ class IndicesApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { executeReindexResult } @@ -69,6 +78,7 @@ class IndicesApiSpec override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = { executeRefreshResult } + } var indicesApi: TestIndicesApi = _ @@ -143,7 +153,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result shouldBe ElasticSuccess(true) @@ -160,7 +170,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", customSettings) + val result = indicesApi.createIndex("my-index", customSettings, None, Nil) // Then result.isSuccess shouldBe true @@ -175,7 +185,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(false) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result shouldBe ElasticSuccess(false) @@ -190,7 +200,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -201,7 +211,7 @@ class IndicesApiSpec "reject invalid index name" in { // When - val result = indicesApi.createIndex("INVALID") + val result = indicesApi.createIndex("INVALID", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -215,7 +225,7 @@ class IndicesApiSpec "reject empty index name" in { // When - val result = indicesApi.createIndex("") + val result = indicesApi.createIndex("", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -228,7 +238,7 @@ class IndicesApiSpec val invalidJson = """{"index": invalid json}""" // When - val result = indicesApi.createIndex("my-index", invalidJson) + val result = indicesApi.createIndex("my-index", invalidJson, None, Nil) // Then result.isFailure shouldBe true @@ -242,7 +252,7 @@ class IndicesApiSpec "reject empty settings" in { // When - val result = indicesApi.createIndex("my-index", "") + val result = indicesApi.createIndex("my-index", "", None, Nil) // Then result.isFailure shouldBe true @@ -252,7 +262,7 @@ class IndicesApiSpec "reject null settings" in { // When - val result = indicesApi.createIndex("my-index", null) + val result = indicesApi.createIndex("my-index", null, None, Nil) // Then result.isFailure shouldBe true @@ -269,7 +279,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -283,7 +293,7 @@ class IndicesApiSpec val invalidJson = """invalid""" // When - Invalid index name should fail first - val result = indicesApi.createIndex("INVALID", invalidJson) + val result = indicesApi.createIndex("INVALID", invalidJson, None, Nil) // Then result.error.get.message should include("Invalid index") @@ -469,7 +479,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result shouldBe ElasticSuccess(true) @@ -483,7 +493,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(false) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result shouldBe ElasticSuccess(false) @@ -497,7 +507,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticFailure(error) // When - val result = indicesApi.indexExists("my-index") + val result = indicesApi.indexExists("my-index", false) // Then result.isFailure shouldBe true @@ -509,7 +519,7 @@ class IndicesApiSpec "reject invalid index name" in { // When - val result = indicesApi.indexExists("INVALID") + val result = indicesApi.indexExists("INVALID", false) // Then result.isFailure shouldBe true @@ -667,7 +677,7 @@ class IndicesApiSpec "fail when target index does not exist" in { // Given var callCount = 0 - val checkingApi = new IndicesApi with RefreshApi { + val checkingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -676,21 +686,6 @@ class IndicesApiSpec else ElasticSuccess(false) // target doesn't exist } - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -718,7 +713,7 @@ class IndicesApiSpec "fail when target existence check fails" in { // Given var callCount = 0 - val checkingApi = new IndicesApi with RefreshApi { + val checkingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -727,21 +722,6 @@ class IndicesApiSpec else ElasticFailure(ElasticError("Connection error")) // target check fails } - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -760,9 +740,10 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index").map { success => - if (success) "Created" else "Not created" - } + val result = + indicesApi.createIndex("my-index", mappings = None, aliases = Nil).map { success => + if (success) "Created" else "Not created" + } // Then result shouldBe ElasticSuccess("Created") @@ -774,9 +755,10 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index").flatMap { _ => - indicesApi.indexExists("my-index") - } + val result = + indicesApi.createIndex("my-index", mappings = None, aliases = Nil).flatMap { _ => + indicesApi.indexExists("my-index", false) + } // Then result shouldBe ElasticSuccess(true) @@ -789,7 +771,7 @@ class IndicesApiSpec // When val result = for { - created <- indicesApi.createIndex("my-index") + created <- indicesApi.createIndex("my-index", mappings = None, aliases = Nil) opened <- indicesApi.openIndex("my-index") } yield created && opened @@ -804,7 +786,7 @@ class IndicesApiSpec // When val result = indicesApi - .createIndex("my-index") + .createIndex("my-index", mappings = None, aliases = Nil) .map(!_) .flatMap(v => ElasticSuccess(s"Result: $v")) @@ -824,7 +806,7 @@ class IndicesApiSpec indicesApi.executeDeleteIndexResult = ElasticSuccess(true) // When - val created = indicesApi.createIndex("test-index") + val created = indicesApi.createIndex("test-index", mappings = None, aliases = Nil) val closed = indicesApi.closeIndex("test-index") val opened = indicesApi.openIndex("test-index") val deleted = indicesApi.deleteIndex("test-index") @@ -846,9 +828,9 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result1 = indicesApi.createIndex("index1") - val result2 = indicesApi.createIndex("index2") - val result3 = indicesApi.createIndex("index3") + val result1 = indicesApi.createIndex("index1", mappings = None, aliases = Nil) + val result2 = indicesApi.createIndex("index2", mappings = None, aliases = Nil) + val result3 = indicesApi.createIndex("index3", mappings = None, aliases = Nil) // Then result1.isSuccess shouldBe true @@ -867,7 +849,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(error) // When - val result = indicesApi.createIndex("my-index") + val result = indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then result.isFailure shouldBe true @@ -923,34 +905,23 @@ class IndicesApiSpec "validate index name before calling execute methods" in { // Given var executeCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) } - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When - validatingApi.createIndex("INVALID") + validatingApi.createIndex("INVALID", mappings = None, aliases = Nil) // Then executeCalled shouldBe false @@ -959,34 +930,23 @@ class IndicesApiSpec "validate settings after index name validation" in { // Given var executeCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCalled = true ElasticSuccess(true) } - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When - validatingApi.createIndex("valid-index", "invalid json") + validatingApi.createIndex("valid-index", "invalid json", None, Nil) // Then executeCalled shouldBe false @@ -995,7 +955,7 @@ class IndicesApiSpec "validate both indices in reindex before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1003,21 +963,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1030,7 +975,7 @@ class IndicesApiSpec "check source and target are different before existence checks" in { // Given var existsCheckCalled = false - val validatingApi = new IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = { @@ -1038,21 +983,6 @@ class IndicesApiSpec ElasticSuccess(true) } - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1070,7 +1000,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - indicesApi.createIndex("my-index") + indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then verify(mockLogger, atLeastOnce).info(any[String]) @@ -1082,7 +1012,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticFailure(ElasticError("Failed")) // When - indicesApi.createIndex("my-index") + indicesApi.createIndex("my-index", mappings = None, aliases = Nil) // Then verify(mockLogger).error(any[String]) @@ -1093,7 +1023,7 @@ class IndicesApiSpec indicesApi.executeIndexExistsResult = ElasticSuccess(true) // When - indicesApi.indexExists("my-index") + indicesApi.indexExists("my-index", false) // Then verify(mockLogger, atLeastOnce).debug(any[String]) @@ -1123,7 +1053,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex(maxName) + val result = indicesApi.createIndex(maxName, mappings = None, aliases = Nil) // Then result.isSuccess shouldBe true @@ -1179,7 +1109,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", complexSettings) + val result = indicesApi.createIndex("my-index", complexSettings, None, Nil) // Then result.isSuccess shouldBe true @@ -1192,7 +1122,7 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - val result = indicesApi.createIndex("my-index", settings) + val result = indicesApi.createIndex("my-index", settings, None, Nil) // Then result.isSuccess shouldBe true @@ -1209,7 +1139,7 @@ class IndicesApiSpec indicesApi.executeDeleteIndexResult = ElasticSuccess(true) // When - val created = indicesApi.createIndex("new-index") + val created = indicesApi.createIndex("new-index", mappings = None, aliases = Nil) val reindexed = indicesApi.reindex("old-index", "new-index") val deleted = indicesApi.deleteIndex("old-index") @@ -1222,13 +1152,13 @@ class IndicesApiSpec "handle multiple operations with mixed results" in { // Given indicesApi.executeCreateIndexResult = ElasticSuccess(true) - val result1 = indicesApi.createIndex("index1") + val result1 = indicesApi.createIndex("index1", mappings = None, aliases = Nil) indicesApi.executeCreateIndexResult = ElasticFailure(ElasticError("Already exists")) - val result2 = indicesApi.createIndex("index2") + val result2 = indicesApi.createIndex("index2", mappings = None, aliases = Nil) indicesApi.executeCreateIndexResult = ElasticSuccess(false) - val result3 = indicesApi.createIndex("index3") + val result3 = indicesApi.createIndex("index3", mappings = None, aliases = Nil) // Then result1.isSuccess shouldBe true @@ -1263,7 +1193,8 @@ class IndicesApiSpec indicesApi.executeCreateIndexResult = ElasticSuccess(true) // When - Simulate concurrent calls - val results = (1 to 5).map(i => indicesApi.createIndex(s"index-$i")) + val results = + (1 to 5).map(i => indicesApi.createIndex(s"index-$i", mappings = None, aliases = Nil)) // Then results.foreach(_.isSuccess shouldBe true) diff --git a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala index e2ce89e7..7bae66b9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/MappingApiSpec.scala @@ -6,6 +6,7 @@ import org.scalatest.BeforeAndAfterEach import org.mockito.{ArgumentMatchersSugar, MockitoSugar} import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.schema.TableAlias /** Unit tests for MappingApi */ @@ -21,13 +22,13 @@ class MappingApiSpec val mockLogger: Logger = mock[Logger] // Valid test data - val validMapping: String = """{"properties":{"name":{"type":"text"}}}""" + val validMapping: String = """{"_doc":{"properties":{"name":{"type":"text"}}}}""" val validSettings: String = """{"my-index":{"settings":{"index":{"number_of_shards":"1"}}}}""" val updatedMapping: String = """{"properties":{"name":{"type":"text"},"age":{"type":"integer"}}}""" // Concrete implementation for testing - class TestMappingApi extends MappingApi with SettingsApi with IndicesApi with RefreshApi { + class TestMappingApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -35,6 +36,7 @@ class MappingApiSpec var executeGetMappingResult: ElasticResult[String] = ElasticSuccess(validMapping) var executeIndexExistsResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeCreateIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) + var executeGetIndexResult: ElasticResult[Option[String]] = ElasticSuccess(None) var executeDeleteIndexResult: ElasticResult[Boolean] = ElasticSuccess(true) var executeReindexFunction : (String, String, Boolean) => ElasticResult[(Boolean, Option[Long])] = @@ -60,11 +62,16 @@ class MappingApiSpec override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeCreateIndexResult } + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = + executeGetIndexResult + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = { executeDeleteIndexResult } @@ -72,7 +79,8 @@ class MappingApiSpec override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = { executeReindexFunction(sourceIndex, targetIndex, refresh) } @@ -95,6 +103,7 @@ class MappingApiSpec index: String, settings: String ): ElasticResult[Boolean] = ElasticSuccess(true) + } var mappingApi: TestMappingApi = _ @@ -120,6 +129,7 @@ class MappingApiSpec result.isSuccess shouldBe true result.get shouldBe true + verify(mockLogger).info("βœ… Elasticsearch version: 0.0.0") verify(mockLogger).debug(s"Setting mapping for index 'my-index': $validMapping") verify(mockLogger).info("βœ… Mapping for index 'my-index' updated successfully") } @@ -544,9 +554,12 @@ class MappingApiSpec result.isSuccess shouldBe true result.get shouldBe true - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("βœ… Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("βœ… Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("βœ… Index 'my-index' created successfully") - verify(mockLogger, atLeastOnce).info("βœ… Mapping for index 'my-index' set successfully") + verify(mockLogger, atLeastOnce).info("βœ… Index 'my-index' created with mapping successfully") } "do nothing when mapping is up to date" in { @@ -606,16 +619,15 @@ class MappingApiSpec "fail when setMapping fails during creation" in { // Given mappingApi.executeIndexExistsResult = ElasticSuccess(false) - mappingApi.executeCreateIndexResult = ElasticSuccess(true) - val error = ElasticError("Mapping invalid", statusCode = Some(400)) - mappingApi.executeSetMappingResult = ElasticFailure(error) // When - val result = mappingApi.updateMapping("my-index", validMapping) + val result = mappingApi.updateMapping("my-index", "invalid mapping") // Then result.isFailure shouldBe true - result.error.get.message should include("Mapping invalid") + result.error.get.message should include( + "Invalid mappings: Invalid JSON: Unrecognized token 'invalid'" + ) } "rollback when migration fails" in { @@ -865,9 +877,12 @@ class MappingApiSpec // Then result.isSuccess shouldBe true - verify(mockLogger).info("Creating new index 'products' with mapping") + verify(mockLogger).debug("Checking if index 'products' exists") + verify(mockLogger).debug("βœ… Index 'products' does not exist") + verify(mockLogger).info(s"Creating index 'products' with settings: $validSettings") + verify(mockLogger).info("βœ… Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("βœ… Index 'products' created successfully") - verify(mockLogger).info("βœ… Mapping for index 'products' set successfully") + verify(mockLogger).info("βœ… Index 'products' created with mapping successfully") } "successfully update existing index with new mapping" in { @@ -1295,14 +1310,13 @@ class MappingApiSpec result.error.get.message shouldBe "Cannot retrieve mapping" } - "handle partial failure in createIndexWithMapping" in { + "handle partial failure in createIndex" in { // Given mappingApi.executeIndexExistsResult = ElasticSuccess(false) - mappingApi.executeCreateIndexResult = ElasticSuccess(true) // setMapping fails val error = ElasticError("Invalid mapping structure", statusCode = Some(400)) - mappingApi.executeSetMappingResult = ElasticFailure(error) + mappingApi.executeCreateIndexResult = ElasticFailure(error) // When val result = mappingApi.updateMapping("my-index", validMapping) @@ -1495,12 +1509,15 @@ class MappingApiSpec mappingApi.executeSetMappingResult = ElasticSuccess(true) // When - mappingApi.updateMapping("my-index", validMapping) + mappingApi.updateMapping("my-index", validMapping, validSettings) // Then - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("βœ… Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("βœ… Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("βœ… Index 'my-index' created successfully") - verify(mockLogger).info("βœ… Mapping for index 'my-index' set successfully") + verify(mockLogger).info("βœ… Index 'my-index' created with mapping successfully") } /*"log backup failure" in { @@ -1789,9 +1806,12 @@ class MappingApiSpec // Then result.isSuccess shouldBe true - verify(mockLogger).info("Creating new index 'my-index' with mapping") + verify(mockLogger).debug("Checking if index 'my-index' exists") + verify(mockLogger).debug("βœ… Index 'my-index' does not exist") + verify(mockLogger).info(s"Creating index 'my-index' with settings: $validSettings") + verify(mockLogger).info("βœ… Elasticsearch version: 0.0.0") verify(mockLogger, atLeastOnce).info("βœ… Index 'my-index' created successfully") - verify(mockLogger).info("βœ… Mapping for index 'my-index' set successfully") + verify(mockLogger).info("βœ… Index 'my-index' created with mapping successfully") } "handle multi-tenant index mapping update" in { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index e6a86dba..d59965f9 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -7,6 +7,8 @@ import org.mockito.MockitoSugar import org.mockito.ArgumentMatchersSugar import org.slf4j.Logger import app.softnetwork.elastic.client.result._ +import app.softnetwork.elastic.sql.query +import app.softnetwork.elastic.sql.schema.TableAlias import com.google.gson.JsonParser /** Unit tests for SettingsApi Coverage target: 80%+ Using mockito-scala 1.17.12 @@ -22,7 +24,7 @@ class SettingsApiSpec val mockLogger: Logger = mock[Logger] // Concrete implementation for testing - class TestSettingsApi extends SettingsApi with IndicesApi with RefreshApi { + class TestSettingsApi extends NopeClientApi { override protected def logger: Logger = mockLogger // Control variables @@ -52,19 +54,6 @@ class SettingsApiSpec executeOpenIndexResult } - // Other required methods - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } var settingsApi: TestSettingsApi = _ @@ -489,7 +478,7 @@ class SettingsApiSpec // Then result.isSuccess shouldBe true - val parsedResult = new JsonParser().parse(result.get).getAsJsonObject + val parsedResult = JsonParser.parseString(result.get).getAsJsonObject parsedResult.has("number_of_shards") shouldBe true parsedResult.get("number_of_shards").getAsString shouldBe "3" } @@ -709,7 +698,7 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -732,22 +721,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -764,7 +737,7 @@ class SettingsApiSpec var updateCalled = false var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -784,22 +757,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -814,7 +771,7 @@ class SettingsApiSpec // Given var openCalled = false - val workflowApi = new SettingsApi with IndicesApi with RefreshApi { + val workflowApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -833,22 +790,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1385,7 +1326,7 @@ class SettingsApiSpec "validate index name before calling executeCloseIndex" in { // Given var closeCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1393,27 +1334,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1426,7 +1346,7 @@ class SettingsApiSpec "validate settings after index name" in { // Given var closeCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = { @@ -1434,27 +1354,6 @@ class SettingsApiSpec ElasticSuccess(true) } - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeLoadSettings(index: String): ElasticResult[String] = - ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When @@ -1467,7 +1366,7 @@ class SettingsApiSpec "validate index name before calling executeLoadSettings" in { // Given var loadCalled = false - val validatingApi = new SettingsApi with IndicesApi with RefreshApi { + val validatingApi = new NopeClientApi { override protected def logger: Logger = mockLogger override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { @@ -1475,27 +1374,6 @@ class SettingsApiSpec ElasticSuccess("{}") } - override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeUpdateSettings( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = ??? - override private[client] def executeCreateIndex( - index: String, - settings: String - ): ElasticResult[Boolean] = ??? - override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeReindex( - sourceIndex: String, - targetIndex: String, - refresh: Boolean - ): ElasticResult[(Boolean, Option[Long])] = ??? - override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = - ??? - override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = ??? } // When diff --git a/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala new file mode 100644 index 00000000..f86f9223 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/TemplateConverterSpec.scala @@ -0,0 +1,79 @@ +package app.softnetwork.elastic.client + +import com.fasterxml.jackson.databind.ObjectMapper +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TemplateConverterSpec extends AnyFlatSpec with Matchers { + + "isLegacyFormat" should "detect legacy format" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + TemplateConverter.isLegacyFormat(legacy) shouldBe true + } + + it should "detect composable format" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + TemplateConverter.isLegacyFormat(composable) shouldBe false + } + + "convertLegacyToComposable" should "convert legacy to composable" in { + val legacy = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": {"number_of_shards": 1}, + | "mappings": {"properties": {"timestamp": {"type": "date"}}} + |} + |""".stripMargin + + val result = TemplateConverter.convertLegacyToComposable(legacy) + result.isSuccess shouldBe true + + val composable = new ObjectMapper().readTree(result.get) + composable.has("priority") shouldBe true + composable.get("priority").asInt() shouldBe 1 + composable.has("template") shouldBe true + composable.get("template").has("settings") shouldBe true + composable.get("template").has("mappings") shouldBe true + } + + "normalizeTemplate" should "convert legacy to composable for ES 7.8+" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + val result = TemplateConverter.normalizeTemplate(legacy, "7.10.0") + + result.isSuccess shouldBe true + val normalized = new ObjectMapper().readTree(result.get) + normalized.has("priority") shouldBe true + normalized.has("template") shouldBe true + } + + it should "convert composable to legacy for ES < 7.8" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + val result = TemplateConverter.normalizeTemplate(composable, "7.5.0") + + result.isSuccess shouldBe true + val normalized = new ObjectMapper().readTree(result.get) + normalized.has("order") shouldBe true + normalized.has("settings") shouldBe true + normalized.has("template") shouldBe false + } + + it should "keep legacy format for ES < 7.8" in { + val legacy = """{"index_patterns":["logs-*"],"order":1,"settings":{}}""" + val result = TemplateConverter.normalizeTemplate(legacy, "7.5.0") + + result.isSuccess shouldBe true + result.get should include("order") + result.get should not include "priority" + } + + it should "keep composable format for ES 7.8+" in { + val composable = """{"index_patterns":["logs-*"],"priority":1,"template":{"settings":{}}}""" + val result = TemplateConverter.normalizeTemplate(composable, "7.10.0") + + result.isSuccess shouldBe true + result.get should include("priority") + result.get should include("template") + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala index c24cb834..c303d39d 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/file/FileSourceSpec.scala @@ -2,6 +2,8 @@ package app.softnetwork.elastic.client.file import akka.actor.ActorSystem import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.sql.query.{Delta, Json, JsonArray, Parquet, Unknown} +import app.softnetwork.elastic.sql.serialization.JacksonConfig import org.scalatest.BeforeAndAfterAll import org.scalatest.concurrent.ScalaFutures import org.scalatest.matchers.should.Matchers diff --git a/documentation/client/README.md b/documentation/client/README.md index fc37f028..da497288 100644 --- a/documentation/client/README.md +++ b/documentation/client/README.md @@ -18,3 +18,5 @@ Welcome to the Client Engine Documentation. Navigate through the sections below: - [Search Documents](search.md) - [Scroll Search](scroll.md) - [Aggregations](aggregations.md) +- [Template Management](templates.md) +- [SQL Gateway Usage](gateway.md) diff --git a/documentation/client/gateway.md b/documentation/client/gateway.md new file mode 100644 index 00000000..67d19115 --- /dev/null +++ b/documentation/client/gateway.md @@ -0,0 +1,536 @@ +[Back to index](README.md) + +# πŸ“˜ GatewayApi + +The `GatewayApi` provides a unified entry point for executing SQL statements (DQL, DML, DDL, and Pipeline operations) against Elasticsearch. +It exposes a single method: + +```scala +def run(sql: String): Future[ElasticResult[QueryResult]] +``` + +which accepts any SQL statement supported by the SQL Gateway. + +This API is designed to offer a familiar SQL interface while leveraging Elasticsearch’s indexing, search, aggregation, and mapping capabilities under the hood. + +--- + +## Table of Contents + +- [Overview](#overview) +- [Statement Hierarchy](#statement-hierarchy) +- [Supported SQL Statements](#supported-sql-statements) +- [Executing SQL](#executing-sql) + - [Single Statement](#single-statement) + - [Multiple Statements](#multiple-statements) +- [QueryResult Types](#queryresult-types) + - [DQL Results](#dql-results) + - [DML Results](#dml-results) + - [DDL Results](#ddl-results) +- [Examples](#examples) +- [Error Handling](#error-handling) +- [Notes](#notes) + +--- + +## Overview + +`GatewayApi` is a high‑level SQL interface built on top of the Softnetwork Elasticsearch client. +It routes SQL statements to the appropriate executor: + +| SQL Type | Executor | +|----------------------------------------------------------|---------------------| +| DQL (SELECT) | `DqlExecutor` | +| DML (INSERT / UPDATE / DELETE / COPY INTO) | `DmlExecutor` | +| DDL (CREATE / ALTER / DROP / TRUNCATE / SHOW / DESCRIBE) | `DdlRouterExecutor` | + +The API automatically: + +- normalizes SQL (removes comments, trims whitespace) +- splits multiple statements separated by `;` +- parses SQL into AST nodes +- dispatches to the correct executor +- returns a typed `QueryResult` + +--- + +## Statement Hierarchy + +The SQL Gateway classifies SQL statements into three main categories: + +- **DQL** β€” SELECT queries +- **DML** β€” INSERT, UPDATE, DELETE, COPY INTO +- **DDL** β€” schema and pipeline definitions + +DDL statements are further divided into: + +- **Table statements** (`TableStatement`) + CREATE TABLE, ALTER TABLE, DROP TABLE, TRUNCATE TABLE, DESCRIBE TABLE, SHOW TABLE, SHOW CREATE TABLE + +- **Pipeline statements** (`PipelineStatement`) + CREATE PIPELINE, ALTER PIPELINE, DROP PIPELINE, DESCRIBE PIPELINE, SHOW PIPELINE, SHOW CREATE PIPELINE + +Both kinds of statements extend `DdlStatement`. + +--- + +## Supported SQL Statements + +### DQL + +[DQL Statements](../sql/dql_statements.md) supported by the SQL Gateway include: +- `SELECT … FROM …` +- `JOIN UNNEST` +- `GROUP BY`, `HAVING` +- `ORDER BY`, `LIMIT`, `OFFSET` +- Window functions (`OVER`) +- Parent‑level aggregations on nested arrays + +### DML + +[DML Statements](../sql/dml_statements.md) supported by the SQL Gateway include: +- `INSERT INTO … VALUES` +- `INSERT INTO … AS SELECT` +- `UPDATE … SET …` +- `DELETE FROM …` +- `COPY INTO table FROM file` + +### DDL + +[DDL Statements](../sql/ddl_statements.md) supported by the SQL Gateway include: +- `CREATE TABLE` +- `ALTER TABLE` +- `DROP TABLE` +- `TRUNCATE TABLE` +- `DESCRIBE TABLE` +- `SHOW TABLE` +- `SHOW CREATE TABLE` + +- `CREATE PIPELINE` +- `ALTER PIPELINE` +- `DROP PIPELINE` +- `DESCRIBE PIPELINE` +- `SHOW PIPELINE` +- `SHOW CREATE PIPELINE` + +--- + +## Executing SQL + +### Single Statement + +```scala +gateway.run("SELECT * FROM dql_users") +``` + +### Multiple Statements + +Statements separated by `;` are executed **sequentially**: + +```scala +gateway.run(""" + CREATE TABLE users (...); + INSERT INTO users VALUES (...); + SELECT * FROM users; +""") +``` + +The result of the **last** statement is returned. + +--- + +## QueryResult Types + +The Gateway returns one of the following: + +--- + +### DQL Results + +#### `QueryRows` + +Materialized rows: + +```scala +QueryRows(Seq(Map("id" -> 1, "name" -> "Alice"))) +``` + +#### `QueryStream` + +Streaming rows using scroll: + +```scala +QueryStream(Source[(Map[String, Any], ScrollMetrics)]) +``` + +#### `QueryStructured` + +Raw Elasticsearch response: + +```scala +QueryStructured(ElasticResponse) +``` + +--- + +### DML Results + +#### `DmlResult` + +```scala +DmlResult( + inserted = 10, + updated = 0, + deleted = 0, + rejected = 0 +) +``` + +--- + +### DDL Results + +DDL operations return one of the following: + +- `DdlResult(success = true)` + Returned for CREATE / ALTER / DROP / TRUNCATE (tables or pipelines). + +- `TableResult(table: Table` + Returned only by `SHOW TABLE`. + +- `PipelineResult(pipeline: IngestPipeline)` + Returned only by `SHOW PIPELINE`. + +- `QueryRows` + Returned by: + - `DESCRIBE TABLE` + - `DESCRIBE PIPELINE` + +- `SQLResult(sql: String)` + Returned by: + - `SHOW CREATE TABLE` + - `SHOW CREATE PIPELINE` + +--- + +## **Architecture Diagram** + +```text ++-------------------------------------------------------------+ +| GatewayApi | +|-------------------------------------------------------------| +| - run(sql: String): Future[ElasticResult[QueryResult]] | ++-------------------------------+-----------------------------+ + | + v + Normalize / strip comments / split on ';' + | + v + +----------------+ + | Parser | + | (SQL β†’ AST) | + +----------------+ + | + v + +------------------------+ + | Statement | + +------------------------+ + / | \ + / | \ + v v v + +----------------+ +----------------+ +----------------+ + | DqlStatement | | DmlStatement | | DdlStatement | + +----------------+ +----------------+ +----------------+ + | + v + +-------------------------------+ + | DdlRouterExecutor | + +-------------------------------+ + | - execute(ddl: DdlStatement) | + +-------------------------------+ + / \ + / \ + v v + +---------------------------+ +---------------------------+ + | TableStatement | | PipelineStatement | + +---------------------------+ +---------------------------+ + | TableExecutor | | PipelineExecutor | + +---------------------------+ +---------------------------+ + | | + v v + +----------------+ +----------------+ + | Elasticsearch | | Elasticsearch | + +----------------+ +----------------+ + +DQL path: +--------- + +DqlStatement + | + v ++----------------+ +| DqlExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++---------------------------------------------+ +| DQL QueryResult: | +| - QueryRows | +| - QueryStream | +| - QueryStructured | ++---------------------------------------------+ + + +DML path: +--------- + +DmlStatement + | + v ++----------------+ +| DmlExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++---------------------------------------------+ +| DML QueryResult: | +| - DmlResult | ++---------------------------------------------+ + + +DDL (Tables) path: +------------------ + +TableStatement + | + v ++----------------+ +| TableExecutor | ++----------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++------------------------------------------------------+ +| DDL Table QueryResult: | +| - DdlResult (CREATE / ALTER / DROP / TRUNC) | +| - TableResult (SHOW TABLE) | +| - SQLResult (SHOW CREATE TABLE) | +| - QueryRows (DESCRIBE TABLE, etc.) | ++------------------------------------------------------+ + + +DDL (Pipelines) path: +--------------------- + +PipelineStatement + | + v ++--------------------+ +| PipelineExecutor | ++--------------------+ + | + v ++----------------+ +| Elasticsearch | ++----------------+ + | + v ++------------------------------------------------------+ +| DDL Pipeline QueryResult: | +| - DdlResult (CREATE / ALTER / DROP) | +| - PipelineResult (SHOW PIPELINE) | +| - SQLResult (SHOW CREATE PIPELINE) | +| - QueryRows (DESCRIBE PIPELINE, etc.) | ++------------------------------------------------------+ + + +Final API surface: +------------------ + +gateway.run(sql: String) + β†’ Future[ElasticResult[QueryResult]] +``` + +--- + +## Examples + +--- + +### SELECT + +```scala +gateway.run(""" + SELECT id, name, age + FROM dql_users + WHERE age >= 18 + ORDER BY age DESC +""") +``` + +Result: + +```scala +ElasticSuccess(QueryRows(rows)) +``` + +--- + +### INSERT + +```scala +gateway.run(""" + INSERT INTO dml_users (id, name, age) + VALUES (1, 'Alice', 30) +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(inserted = 1)) +``` + +--- + +### UPDATE + +```scala +gateway.run(""" + UPDATE dml_users + SET age = 31 + WHERE id = 1 +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(updated = 1)) +``` + +--- + +### DELETE + +```scala +gateway.run("DELETE FROM dml_users WHERE id = 1") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(deleted = 1)) +``` + +--- + +### CREATE TABLE + +```scala +gateway.run(""" + CREATE TABLE dml_users ( + id INT, + name TEXT, + age INT, + PRIMARY KEY (id) + ) +""") +``` + +--- + +### ALTER TABLE + +```scala +gateway.run(""" + ALTER TABLE dml_users + ADD COLUMN email TEXT +""") +``` + +--- + +### DROP TABLE + +```scala +gateway.run("DROP TABLE dml_users") +``` + +--- + +### TRUNCATE TABLE + +```scala +gateway.run("TRUNCATE TABLE dml_users") +``` + +--- + +### COPY INTO + +```scala +gateway.run(""" + COPY INTO dml_users + FROM 'classpath:/data/users.json' +""") +``` + +Result: + +```scala +ElasticSuccess(DmlResult(inserted = 100)) +``` + +--- + +## Error Handling + +Errors are returned as: + +```scala +ElasticFailure(ElasticError(...)) +``` + +Typical cases: + +- SQL parsing errors +- Unsupported statements +- Mapping conflicts +- Invalid DDL operations +- Elasticsearch indexing errors + +Example: + +```scala +gateway.run("BAD SQL") +β†’ ElasticFailure(ElasticError(message = "Error parsing schema DDL statement: ...")) +``` + +--- + +## Notes + +- SQL comments (`-- ...`) are stripped automatically. +- Empty statements return `EmptyResult`. +- Multiple statements are executed sequentially. +- The last statement’s result is returned. +- Streaming queries (`QueryStream`) require an active `ActorSystem`. + +--- + +[Back to index](README.md) diff --git a/documentation/client/indices.md b/documentation/client/indices.md index cd472f81..db99ff36 100644 --- a/documentation/client/indices.md +++ b/documentation/client/indices.md @@ -71,6 +71,616 @@ val defaultSettings: String = """ --- +## πŸ”§ Index State Detection and Automatic State Restoration + +SoftClient4ES ensures that operations requiring an **open index** (such as `deleteByQuery` and `truncateIndex`) behave safely and consistently, even when the target index is **closed**. + +Elasticsearch does **not** allow `_delete_by_query` on a closed index (including ES 8.x and ES 9.x). +To guarantee correct behavior, SoftClient4ES automatically: + +1. Detects whether the index is open or closed +2. Opens the index if needed +3. Executes the operation +4. Restores the original state (re‑closes the index if it was closed) + +This mechanism is fully transparent to the user. + +--- + +## πŸ”§ Internal Shard Readiness Handling (`waitForShards`) + +Some Elasticsearch operations require the index to be **fully operational**, meaning all primary shards must be allocated and the index must reach at least **yellow** cluster health. +This is especially important for Elasticsearch **6.x**, where reopening a closed index does **not** guarantee immediate shard availability. + +SoftClient4ES includes an internal mechanism called **`waitForShards`**, which ensures that the index is ready before executing operations that depend on shard availability. + +This mechanism is: + +- **fully automatic** +- **transparent to the user** +- **only applied when necessary** +- **client‑specific** (Jest ES6, REST HL ES6) +- a **no‑op** for Elasticsearch 7, 8, and 9 + +--- + +### When is `waitForShards` used? + +`waitForShards` is invoked automatically after reopening an index inside the internal `openIfNeeded` workflow. + +It is used by operations that require: + +- an **open** index +- **allocated** shards +- a **searchable** state + +Specifically: + +- `deleteByQuery` +- `truncateIndex` + +These operations internally perform: + +1. Detect whether the index is closed +2. Open it if needed +3. **Wait for shards to reach the required health status** +4. Execute the operation +5. Restore the original index state (re‑close if necessary) + +This ensures consistent behavior across all Elasticsearch versions. + +--- + +### Why is this needed? + +Elasticsearch 6.x has a known behavior: + +- After reopening a closed index, shards may remain in `INITIALIZING` state for a short period. +- Executing `_delete_by_query` during this window results in: + + ``` + 503 Service Unavailable + search_phase_execution_exception + all shards failed + ``` + +Elasticsearch 7+ no longer exhibits this issue. + +SoftClient4ES abstracts this difference by automatically waiting for shard readiness on ES6. + +--- + +### How does `waitForShards` work? + +Internally, the client performs: + +``` +GET /_cluster/health/?wait_for_status=yellow&timeout=30s +``` + +This ensures: + +- primary shards are allocated +- the index is searchable +- the cluster is ready to process delete‑by‑query operations + +### Client‑specific behavior: + +| Client | ES Version | Behavior | +|---------------------|-------------|-------------------------------------------------------------| +| **Jest** | 6.x | Uses a custom Jest action to call `_cluster/health` | +| **REST HL** | 6.x | Uses the low‑level client to call `_cluster/health` | +| **Java API Client** | 8.x / 9.x | No‑op (Elasticsearch handles shard readiness automatically) | +| **REST HL** | 7.x | No‑op | +| **Jest** | 7.x | No‑op | + +--- + +### Transparency for the user + +`waitForShards` is **not part of the public API**. +It is an internal mechanism that ensures: + +- consistent behavior across ES6, ES7, ES8, ES9 +- predictable delete‑by‑query semantics +- correct handling of closed indices +- no need for users to manually manage shard allocation or cluster health + +Users do **not** need to call or configure anything. + +--- + +### Example (internal workflow) + +When calling: + +```scala +client.deleteByQuery("my_index", """{"query": {"match_all": {}}}""") +``` + +SoftClient4ES internally performs: + +1. Check if `my_index` is closed +2. If closed β†’ open it +3. **Wait for shards to reach yellow** (ES6 only) +4. Execute `_delete_by_query` +5. Restore original state (re‑close if needed) + +This guarantees reliable behavior even on older Elasticsearch clusters. + +--- + +### πŸ”§ updateByQuery + +`updateByQuery` updates documents in an index using either a **JSON query** or a **SQL UPDATE statement**. +It supports ingest pipelines, SQL‑driven SET clauses, and automatic pipeline merging. + +SoftClient4ES ensures consistent behavior across Elasticsearch 6, 7, 8, and 9, including: + +- automatic index opening and state restoration +- shard readiness handling (ES6 only) +- temporary pipeline creation and cleanup +- SQL β†’ JSON query translation +- SQL β†’ ingest pipeline generation + +--- + +#### SQL UPDATE Support + +SoftClient4ES accepts SQL UPDATE statements of the form: + +```sql +UPDATE SET field = value [, field2 = value2 ...] [WHERE ] +``` + +#### βœ”οΈ Supported: + +- simple literal values (`string`, `number`, `boolean`, `date`) +- multiple assignments in the SET clause +- WHERE clause with any supported SQL predicate +- UPDATE without WHERE (updates all documents) +- automatic conversion to: + - a JSON query (`WHERE` β†’ `query`) + - an ingest pipeline (`SET` β†’ processors) + +#### βœ–οΈ Not supported: + +- painless scripts +- complex expressions in SET +- joins or multi‑table updates + +--- + +#### Automatic Pipeline Generation (SQL SET β†’ Ingest Pipeline) + +When using SQL UPDATE, the `SET` clause is automatically converted into an ingest pipeline: + +```json +{ + "processors": [ + { "set": { "field": "name", "value": "Homer" } }, + { "set": { "field": "childrenCount", "value": 3 } } + ] +} +``` + +This pipeline is created **only for the duration of the update**, unless the user explicitly provides a pipeline ID. + +--- + +#### Pipeline Resolution and Merging + +`updateByQuery` supports three pipeline sources: + +| Source | Description | +|-------------------|--------------------------------| +| **User pipeline** | Provided via `pipelineId` | +| **SQL pipeline** | Generated from SQL SET clause | +| **No pipeline** | JSON update without processors | + +#### Pipeline resolution rules: + +| User pipeline | SQL pipeline | Result | +|----------------|---------------|--------------------------------------| +| None | None | No pipeline | +| Some | None | Use user pipeline | +| None | Some | Create temporary pipeline | +| Some | Some | Merge both into a temporary pipeline | + +#### Pipeline merging + +Processors are merged deterministically: + +- processors with the same `(type, field)` β†’ SQL processor overrides user processor +- order is preserved +- merged pipeline is temporary and deleted after execution + +--- + +#### JSON Query Support + +If the query is not SQL, it is treated as a raw JSON `_update_by_query` request: + +```json +{ + "query": { + "term": { "uuid": "A16" } + } +} +``` + +No pipeline is generated unless the user provides one. + +--- + +#### Index State Handling + +`updateByQuery` uses the same robust index‑state workflow as `deleteByQuery`: + +1. Detect whether the index is open or closed +2. Open it if needed +3. **ES6 only:** wait for shards to reach `yellow` +4. Execute update‑by‑query +5. Restore the original state (re‑close if needed) + +This ensures safe, predictable behavior across all Elasticsearch versions. + +--- + +#### Signature + +```scala +def updateByQuery( + index: String, + query: String, + pipelineId: Option[String] = None, + refresh: Boolean = true +): ElasticResult[Long] +``` + +#### Parameters + +| Name | Type | Description | +|--------------|------------------|-------------------------------------------| +| `index` | `String` | Target index | +| `query` | `String` | SQL UPDATE or JSON query | +| `pipelineId` | `Option[String]` | Optional ingest pipeline to apply | +| `refresh` | `Boolean` | Whether to refresh the index after update | + +#### Returns + +- `ElasticSuccess[Long]` β†’ number of updated documents +- `ElasticFailure` β†’ error details + +--- + +#### Examples + +#### SQL UPDATE with WHERE + +```scala +client.updateByQuery( + "person", + """UPDATE person SET name = 'Another Name' WHERE uuid = 'A16'""" +) +``` + +#### SQL UPDATE without WHERE (update all) + +```scala +client.updateByQuery( + "person", + """UPDATE person SET birthDate = '1972-12-26'""" +) +``` + +#### JSON query with user pipeline + +```scala +client.updateByQuery( + "person", + """{"query": {"match_all": {}}}""", + pipelineId = Some("set-birthdate-1972-12-26") +) +``` + +#### SQL UPDATE + user pipeline (merged) + +```scala +client.updateByQuery( + "person", + """UPDATE person SET birthDate = '1972-12-26' WHERE uuid = 'A16'""", + pipelineId = Some("user-update-name") +) +``` + +--- + +#### Behavior Summary + +- SQL UPDATE is fully supported +- SET clause β†’ ingest pipeline +- WHERE clause β†’ JSON query +- Pipelines are merged when needed +- Temporary pipelines are cleaned up automatically +- Index state is preserved +- Works consistently across ES6, ES7, ES8, ES9 + +--- + +### insertByQuery β€” SQL‑Driven Bulk Insert & Upsert for Elasticsearch + +`insertByQuery` executes SQL `INSERT` statements and writes documents into an Elasticsearch index. +It supports: + +- `INSERT … VALUES` +- `INSERT … AS SELECT` +- `ON CONFLICT DO UPDATE` +- Primary keys (simple or composite) +- Partitioning (`partition_by`) +- Column aliasing +- Automatic mapping between SELECT output and INSERT columns +- Strict SQL validation before execution + +This API is designed for ETL pipelines, migrations, and SQL‑driven ingestion workflows. + +--- + +#### Supported SQL Syntax + +**INSERT … VALUES** + +```sql +INSERT INTO index_name (col1, col2) +VALUES (v1, v2) +``` + +**INSERT … AS SELECT** + +```sql +INSERT INTO index_name (col1, col2) +AS SELECT a AS col1, b AS col2 FROM other_index +``` + +**INSERT without column list** + +```sql +INSERT INTO index_name +SELECT a, b, c FROM other_index +``` + +**ON CONFLICT DO UPDATE** + +```sql +INSERT INTO index_name (...) +VALUES (...) +ON CONFLICT DO UPDATE +``` + +or with explicit conflict target: + +```sql +INSERT INTO index_name (...) +VALUES (...) +ON CONFLICT (col1, col2) DO UPDATE +``` + +--- + +#### Primary Key Semantics + +Primary keys are defined in the index mapping under `_meta.primary_key`: + +``` +"_meta": { + "primary_key": ["order_id", "customer_id"] +} +``` + +Rules: + +- PK may be simple or composite. +- PK determines the Elasticsearch `_id`. +- All PK columns must be present in the INSERT. +- For `INSERT … AS SELECT`, the SELECT must produce all PK columns. + +--- + +#### Conflict Handling + +**ON CONFLICT DO UPDATE** + +Triggers an **upsert**. + +Rules when PK exists: + +- If conflictTarget omitted β†’ PK is used. +- If conflictTarget provided β†’ must match PK exactly. +- All PK columns must be present in the INSERT. + +Rules when PK does not exist: + +- conflictTarget is mandatory. +- All conflictTarget columns must be present in the INSERT. + +--- + +#### INSERT … VALUES + +Direct insertion of literal values. + +Validation: + +- Column count must match value count. +- All PK columns must be present if `ON CONFLICT` is used. + +--- + +#### INSERT … AS SELECT + +Inserts documents produced by a SELECT query. + +Behavior: + +- Executes a scroll query. +- Removes Elasticsearch metadata (`_id`, `_index`, `_score`, `_sort`). +- Maps SELECT output to INSERT columns **by name**. +- Supports aliasing. +- Supports INSERT without column list. + +--- + +#### Column Mapping Rules + +**Mapping is always by name, never by position.** + +Example: + +```sql +SELECT foo AS bar +``` + +β†’ INSERT column `bar` receives the value of `foo`. + +--- + +#### Validation Rules + +**General** + +- INSERT column count must match VALUES count. +- conflictTarget βŠ† INSERT columns. +- INSERT must include all PK columns if PK exists. +- If PK does not exist β†’ conflictTarget is mandatory. + +**For DO UPDATE** + +- conflictTarget must match PK exactly when PK exists. + +**For INSERT … AS SELECT** + +- All INSERT columns must be present in SELECT output. +- SELECT metadata fields are ignored. +- Aliases are resolved correctly. + +**For INSERT without column list** + +- INSERT columns = SELECT output columns. + +--- + +#### Composite Primary Keys + +Composite PKs are fully supported: + +``` +"_meta": { + "primary_key": ["order_id", "customer_id"] +} +``` + +The Elasticsearch `_id` is constructed from all PK columns, e.g.: + +``` +O1001|C001 +``` + +This ensures deterministic conflict detection. + +--- + +#### Partitioning + +If the index defines: + +``` +"_meta": { + "partition_by": { + "column": "order_date", + "granularity": "d" + } +} +``` + +Then: + +- Documents are routed to partitioned indices (e.g., `orders-2024-02-01`). +- The partition key is extracted from the INSERT or SELECT row. + +--- + +#### Error Handling + +`insertByQuery` returns: + +- `ElasticSuccess(count)` on success +- `ElasticFailure(error)` on validation or execution error + +Errors include: + +- Missing PK columns +- conflictTarget mismatch +- Missing SELECT columns +- Invalid SQL syntax +- Unsupported INSERT form +- Elasticsearch bulk failures + +`ON CONFLICT DO NOTHING` never raises a conflict error. + +--- + +#### Examples + +**Insert with VALUES** + +```sql +INSERT INTO customers (customer_id, name, email) +VALUES ('C010', 'Bob', 'bob@example.com') +``` + +**Upsert with PK** + +```sql +INSERT INTO products (sku, name, price) +VALUES ('SKU-001', 'Laptop Pro', 1499.99) +ON CONFLICT DO UPDATE +``` + +**Insert‑or‑ignore with DO NOTHING** + +```sql +INSERT INTO products (sku, name, price) +VALUES ('SKU-001', 'Laptop Pro', 1499.99) +ON CONFLICT DO NOTHING +``` + +**Insert from SELECT with alias mapping** + +```sql +INSERT INTO orders (order_id, customer_id, total) +AS SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders +``` + +**Insert from SELECT without column list** + +```sql +INSERT INTO orders +SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders +``` + +**Upsert with composite PK** + +```sql +INSERT INTO orders (order_id, customer_id, total) +AS SELECT id AS order_id, cust AS customer_id, amount AS total +FROM staging_orders_updates +ON CONFLICT (order_id, customer_id) DO UPDATE +``` + +--- + ## Public Methods ### createIndex @@ -82,13 +692,17 @@ Creates a new index with specified settings. ```scala def createIndex( index: String, - settings: String = defaultSettings + settings: String = defaultSettings, + mappings: Option[String] = None, + aliases: Seq[TableAlias] = Seq.empty ): ElasticResult[Boolean] ``` **Parameters:** - `index` - Name of the index to create - `settings` - JSON settings for the index (defaults to `defaultSettings`) +- `mappings` - Optional JSON mappings for the index +- `aliases` - Optional list of aliases to assign to the index **Returns:** - `ElasticSuccess[Boolean]` with `true` if created, `false` otherwise @@ -97,6 +711,8 @@ def createIndex( **Validation:** - Index name format validation - JSON settings syntax validation +- JSON mappings syntax validation if provided +- Alias name format validation **Examples:** @@ -148,6 +764,25 @@ for { --- +### getIndex + +Gets an existing index. + +**Signature:** + +```scala +def getIndex(index: String): ElasticResult[Option[Index]] +``` + +**Parameters:** +- `index` - Name of the index to get + +**Returns:** +- `ElasticSuccess[Option[Index]]` with index configuration if index found, None otherwise +- `ElasticFailure` with error details + +--- + ### deleteIndex Deletes an existing index. @@ -437,6 +1072,175 @@ existenceChecks.foreach { --- +### isIndexClosed + +Checks whether an index is currently **closed**. + +**Signature:** + +```scala +def isIndexClosed(index: String): ElasticResult[Boolean] +``` + +**Parameters:** +- `index` – Name of the index to inspect + +**Returns:** +- `ElasticSuccess(true)` if the index is closed +- `ElasticSuccess(false)` if the index is open +- `ElasticFailure` if the index does not exist or the request fails + +**Behavior:** +- Uses the Elasticsearch `_cat/indices` API internally +- Supported across Jest (ES6), REST HL (ES6/7), Java API Client (ES8/ES9) + +**Examples:** + +```scala +client.isIndexClosed("archive-2023") match { + case ElasticSuccess(true) => println("Index is closed") + case ElasticSuccess(false) => println("Index is open") + case ElasticFailure(err) => println(s"Error: ${err.message}") +} +``` + +--- + +### Automatic Index Opening and State Restoration + +Some operations require the index to be **open**. +SoftClient4ES automatically handles this through an internal helper: + +- Detect initial state (`open` or `closed`) +- Open the index if needed +- Execute the operation +- Restore the original state + +This ensures: + +- **Safety** (no accidental state changes) +- **Idempotence** (index ends in the same state it started) +- **Compatibility** with all Elasticsearch versions + +This logic is used internally by: + +- `deleteByQuery` +- `truncateIndex` + +--- + +### deleteByQuery + +Deletes documents from an index using either a JSON query or a SQL `DELETE`/`SELECT` expression. + +If the index is **closed**, SoftClient4ES will: + +1. Detect that the index is closed +2. Open it temporarily +3. Execute the delete‑by‑query +4. Re‑close it afterward + +**Signature:** + +```scala +def deleteByQuery( + index: String, + query: String, + refresh: Boolean = true +): ElasticResult[Long] +``` + +**Parameters:** +- `index` – Name of the index +- `query` – JSON or SQL delete expression +- `refresh` – Whether to refresh the index after deletion + +**Returns:** +- `ElasticSuccess[Long]` – number of deleted documents +- `ElasticFailure` – error details + +**Behavior:** +- Validates index name +- Parses SQL into JSON when needed +- Ensures index exists +- Automatically opens closed indices +- Restores original state after execution +- Uses `_delete_by_query` internally + +**Examples:** + +```scala +// JSON delete +client.deleteByQuery( + "users", + """{"query": {"term": {"active": false}}}""" +) + +// SQL delete +client.deleteByQuery( + "orders", + "DELETE FROM orders WHERE status = 'cancelled'" +) + +// SQL select (equivalent to delete) +client.deleteByQuery( + "sessions", + "SELECT * FROM sessions WHERE expired = true" +) +``` + +--- + +### truncateIndex + +Deletes **all documents** from an index while preserving its mappings, settings, and aliases. + +This is implemented as: + +```scala +deleteByQuery(index, """{"query": {"match_all": {}}}""") +``` + +If the index is closed, SoftClient4ES will automatically: + +- Open it +- Execute the truncate +- Restore the closed state + +**Signature:** + +```scala +def truncateIndex(index: String): ElasticResult[Long] +``` + +**Parameters:** +- `index` – Name of the index to truncate + +**Returns:** +- `ElasticSuccess[Long]` – number of deleted documents +- `ElasticFailure` – error details + +**Examples:** + +```scala +// Remove all documents +client.truncateIndex("logs-2024") + +// Safe truncate with existence check +for { + exists <- client.indexExists("cache") + deleted <- if (exists) client.truncateIndex("cache") + else ElasticResult.success(0L) +} yield deleted +``` + +**Notes:** +- Index structure is preserved +- Operation is irreversible +- Works even if the index is initially closed + +--- + ## Implementation Requirements The following methods must be implemented by each client-specific trait: diff --git a/documentation/client/templates.md b/documentation/client/templates.md new file mode 100644 index 00000000..55b97726 --- /dev/null +++ b/documentation/client/templates.md @@ -0,0 +1,1601 @@ +# TEMPLATE API + +## Overview + +The **TemplateApi** trait provides comprehensive functionality to manage Elasticsearch index templates. It supports both **legacy templates** (ES 6.x-7.x) and **composable (index) templates** (ES 7.8+), with automatic version detection and conversion capabilities. + +Index templates define settings, mappings, and aliases that are automatically applied to new indices matching specified patterns. + +**Dependencies:** Extends `ElasticClientHelpers` for validation and logging utilities. + +--- + +## Public Methods + +### createTemplate + +Creates or updates an index template with the specified configuration. + +**Signature:** + +```scala +def createTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to create or update +- `templateDefinition` - JSON string containing the template configuration + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template was created/updated successfully +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution (returns 400 error if invalid) +- Automatically detects template type (composable vs legacy) based on JSON structure +- Converts between formats if necessary based on Elasticsearch version +- For ES < 6.8: Uses legacy template format with `_doc` wrapper in mappings +- For ES 7.8+: Supports both legacy and composable templates +- Logs success with βœ… or failure with ❌ + +**Examples:** + +```scala +// Legacy template (ES 6.x-7.x) +val legacyTemplate = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "version": 1, + | "settings": { + | "number_of_shards": 2, + | "number_of_replicas": 1 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": {"type": "date"}, + | "message": {"type": "text"} + | } + | } + | }, + | "aliases": { + | "logs-current": {} + | } + |} + |""".stripMargin + +client.createTemplate("logs-template", legacyTemplate) match { + case ElasticSuccess(true) => println("Template created") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Composable template (ES 7.8+) +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 200, + | "version": 2, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "refresh_interval": "30s" + | }, + | "mappings": { + | "properties": { + | "cpu_usage": {"type": "float"}, + | "timestamp": {"type": "date"} + | } + | }, + | "aliases": { + | "metrics-latest": { + | "is_write_index": true + | } + | } + | }, + | "composed_of": ["component-template-1"], + | "_meta": { + | "description": "Metrics template" + | } + |} + |""".stripMargin + +client.createTemplate("metrics-template", composableTemplate) + +// Update existing template +val updatedTemplate = legacyTemplate.replace("\"order\": 1", "\"order\": 10") +client.createTemplate("logs-template", updatedTemplate) // Overwrites existing + +// Monadic chaining +for { + _ <- client.createTemplate("users-template", userTemplate) + _ <- client.createIndex("users-2024") + result <- client.search("users-2024", query) +} yield result +``` + +**Template Type Detection:** + +The API automatically detects template type based on JSON structure: + +| Field Present | Template Type | Supported Versions | +|-------------------------------|----------------|---------------------| +| `template` (object) | Composable | ES 7.8+ | +| `index_patterns` (root level) | Legacy | ES 6.x-7.x | +| `mappings` (root level) | Legacy | ES 6.x-7.x | + +--- + +### getTemplate + +Retrieves the configuration of one or more index templates. + +**Signature:** + +```scala +def getTemplate(templateName: String): ElasticResult[Option[String]] +``` + +**Parameters:** +- `templateName` - The name of the template to retrieve (supports wildcards) + +**Returns:** +- `ElasticSuccess[Some(String)]` with JSON template configuration if found +- `ElasticSuccess[None]` if template does not exist +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- Returns normalized JSON format regardless of template type +- For composable templates: Returns full template structure +- For legacy templates: Returns converted format with all fields +- Supports wildcard patterns (e.g., `logs-*`) +- Logs success with βœ… or failure with ❌ + +**Examples:** + +```scala +// Retrieve single template +client.getTemplate("logs-template") match { + case ElasticSuccess(Some(json)) => + println(s"Template found: $json") + case ElasticSuccess(None) => + println("Template not found") + case ElasticFailure(e) => + println(s"Error: ${e.message}") +} + +// Check if template exists +def templateExists(name: String): Boolean = { + client.getTemplate(name) match { + case ElasticSuccess(Some(_)) => true + case _ => false + } +} + +// Retrieve with wildcard +client.getTemplate("logs-*") match { + case ElasticSuccess(Some(json)) => + // Returns all matching templates + println(s"Found templates: $json") + case _ => println("No matching templates") +} + +// Parse template configuration +import com.fasterxml.jackson.databind.ObjectMapper + +client.getTemplate("users-template") match { + case ElasticSuccess(Some(json)) => + val mapper = new ObjectMapper() + val root = mapper.readTree(json) + + val patterns = root.get("index_patterns") + val order = root.get("order").asInt() + + println(s"Patterns: $patterns, Order: $order") + + case _ => println("Template not found") +} + +// Conditional operations +for { + existing <- client.getTemplate("my-template") + result <- existing match { + case Some(_) => + client.deleteTemplate("my-template", ifExists = true) + case None => + ElasticResult.success(true) + } + _ <- client.createTemplate("my-template", newTemplate) +} yield result +``` + +--- + +### deleteTemplate + +Deletes one or more index templates. + +**Signature:** + +```scala +def deleteTemplate( + templateName: String, + ifExists: Boolean = false +): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to delete (supports wildcards) +- `ifExists` - If `true`, returns success even if template doesn't exist (default: `false`) + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template was deleted, `false` if it didn't exist (when `ifExists = true`) +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- When `ifExists = true`: Checks existence before deletion, returns success if not found +- When `ifExists = false`: Returns failure if template doesn't exist +- Supports wildcard patterns for bulk deletion +- Logs success with βœ… or failure with ❌ + +**Examples:** + +```scala +// Delete single template +client.deleteTemplate("logs-template") match { + case ElasticSuccess(true) => println("Template deleted") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Safe deletion (no error if missing) +client.deleteTemplate("old-template", ifExists = true) + +// Delete multiple templates with wildcard +client.deleteTemplate("test-*", ifExists = true) + +// Cleanup after tests +def cleanupTemplates(names: List[String]): Unit = { + names.foreach { name => + client.deleteTemplate(name, ifExists = true) + } +} + +// Conditional deletion +client.getTemplate("temporary-template") match { + case ElasticSuccess(Some(_)) => + client.deleteTemplate("temporary-template") + case _ => + ElasticResult.success(true) +} + +// Monadic cleanup +for { + _ <- client.deleteTemplate("old-users-template", ifExists = true) + _ <- client.createTemplate("users-template", newTemplate) + _ <- client.createIndex("users-2024") +} yield () +``` + +--- + +### templateExists + +Checks if an index template exists. + +**Signature:** + +```scala +def templateExists(templateName: String): ElasticResult[Boolean] +``` + +**Parameters:** +- `templateName` - The name of the template to check + +**Returns:** +- `ElasticSuccess[Boolean]` with `true` if template exists, `false` otherwise +- `ElasticFailure` with error details if operation fails + +**Behavior:** +- Validates template name before execution +- Efficiently checks existence without retrieving full template +- Works with both legacy and composable templates +- Logs debug information + +**Examples:** + +```scala +// Simple existence check +client.templateExists("logs-template") match { + case ElasticSuccess(true) => println("Template exists") + case ElasticSuccess(false) => println("Template not found") + case ElasticFailure(e) => println(s"Error: ${e.message}") +} + +// Conditional creation +def createTemplateIfMissing(name: String, definition: String): ElasticResult[Boolean] = { + client.templateExists(name).flatMap { + case true => + println(s"Template '$name' already exists") + ElasticResult.success(false) + case false => + client.createTemplate(name, definition) + } +} + +// Guard clause pattern +for { + exists <- client.templateExists("users-template") + _ <- if (!exists) { + client.createTemplate("users-template", userTemplate) + } else { + ElasticResult.success(true) + } + result <- client.createIndex("users-2024") +} yield result + +// Bulk existence check +val templates = List("logs", "metrics", "traces") +val existenceMap = templates.map { name => + name -> client.templateExists(name) +}.toMap +``` + +--- + +## Template Types + +### Legacy Templates (ES 6.x - 7.x) + +Legacy templates apply to indices matching specified patterns. + +**Structure:** + +``` +{ + "index_patterns": ["pattern-*"], + "order": 0, + "version": 1, + "settings": { ... }, + "mappings": { ... }, + "aliases": { ... } +} +``` + +**Key Fields:** + +| Field | Type | Description | +|------------------|---------|------------------------------------------------------| +| `index_patterns` | Array | Index patterns to match (e.g., `["logs-*"]`) | +| `order` | Integer | Priority when multiple templates match (higher wins) | +| `version` | Integer | Optional version number for tracking | +| `settings` | Object | Index settings (shards, replicas, etc.) | +| `mappings` | Object | Field mappings and types | +| `aliases` | Object | Index aliases to create | + +**Example:** + +```scala +val legacyTemplate = + """ + |{ + | "index_patterns": ["app-logs-*"], + | "order": 10, + | "version": 3, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": 2, + | "refresh_interval": "5s" + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": { + | "type": "date", + | "format": "strict_date_optional_time||epoch_millis" + | }, + | "level": { + | "type": "keyword" + | }, + | "message": { + | "type": "text", + | "fields": { + | "keyword": { + | "type": "keyword", + | "ignore_above": 256 + | } + | } + | }, + | "user_id": { + | "type": "keyword" + | } + | } + | } + | }, + | "aliases": { + | "app-logs-current": {}, + | "app-logs-errors": { + | "filter": { + | "term": { + | "level": "ERROR" + | } + | } + | } + | } + |} + |""".stripMargin + +client.createTemplate("app-logs-template", legacyTemplate) +``` + +--- + +### Composable Templates (ES 7.8+) + +Composable templates provide more flexibility with component templates and better composition. + +**Structure:** + +``` +{ + "index_patterns": ["pattern-*"], + "priority": 100, + "version": 1, + "template": { + "settings": { ... }, + "mappings": { ... }, + "aliases": { ... } + }, + "composed_of": ["component1", "component2"], + "_meta": { ... } +} +``` + +**Key Fields:** + +| Field | Type | Description | +|------------------|---------|--------------------------------------| +| `index_patterns` | Array | Index patterns to match | +| `priority` | Integer | Template priority (higher wins) | +| `version` | Integer | Optional version number | +| `template` | Object | Contains settings, mappings, aliases | +| `composed_of` | Array | Component templates to include | +| `_meta` | Object | Optional metadata | + +**Example:** + +```scala +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 500, + | "version": 2, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 1, + | "refresh_interval": "30s", + | "codec": "best_compression" + | }, + | "mappings": { + | "properties": { + | "timestamp": { + | "type": "date" + | }, + | "cpu_percent": { + | "type": "float" + | }, + | "memory_used": { + | "type": "long" + | }, + | "host": { + | "type": "keyword" + | } + | } + | }, + | "aliases": { + | "metrics-latest": { + | "is_write_index": true + | }, + | "metrics-high-cpu": { + | "filter": { + | "range": { + | "cpu_percent": { + | "gte": 80 + | } + | } + | } + | } + | } + | }, + | "composed_of": [ + | "metrics-component-settings", + | "metrics-component-mappings" + | ], + | "_meta": { + | "description": "Template for system metrics", + | "managed_by": "monitoring-system", + | "version": "2.0" + | } + |} + |""".stripMargin + +client.createTemplate("metrics-template", composableTemplate) +``` + +--- + +## Template Priority and Matching + +When multiple templates match an index pattern, priority determines which template applies: + +### Legacy Templates (`order`) + +```scala +// Lower order +val template1 = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + +// Higher order wins +val template2 = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 10, + | "settings": { + | "number_of_shards": 3 + | } + |} + |""".stripMargin + +client.createTemplate("logs-default", template1) +client.createTemplate("logs-production", template2) + +// New index "logs-2024" will have 3 shards (template2 wins) +``` + +### Composable Templates (`priority`) + +```scala +// Lower priority +val template1 = + """ + |{ + | "index_patterns": ["app-*"], + | "priority": 100, + | "template": { + | "settings": { + | "number_of_replicas": 1 + | } + | } + |} + |""".stripMargin + +// Higher priority wins +val template2 = + """ + |{ + | "index_patterns": ["app-*"], + | "priority": 200, + | "template": { + | "settings": { + | "number_of_replicas": 2 + | } + | } + |} + |""".stripMargin + +client.createTemplate("app-default", template1) +client.createTemplate("app-production", template2) + +// New index "app-users" will have 2 replicas (template2 wins) +``` + +--- + +## Common Use Cases + +### 1. Time-Series Data + +```scala +val timeSeriesTemplate = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": 1, + | "refresh_interval": "5s", + | "index.lifecycle.name": "logs-policy", + | "index.lifecycle.rollover_alias": "logs-current" + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "@timestamp": { + | "type": "date" + | }, + | "message": { + | "type": "text" + | }, + | "level": { + | "type": "keyword" + | } + | } + | } + | }, + | "aliases": { + | "logs-all": {} + | } + |} + |""".stripMargin + +client.createTemplate("logs-template", timeSeriesTemplate) + +// Indices created with date pattern automatically use template +client.createIndex("logs-2024-01-01") +client.createIndex("logs-2024-01-02") +``` + +### 2. Multi-Environment Templates + +```scala +def createEnvironmentTemplate(env: String, order: Int): String = { + val replicas = env match { + case "production" => 2 + case "staging" => 1 + case _ => 0 + } + + s""" + |{ + | "index_patterns": ["${env}-*"], + | "order": $order, + | "settings": { + | "number_of_shards": 3, + | "number_of_replicas": $replicas + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "environment": { + | "type": "keyword" + | } + | } + | } + | } + |} + |""".stripMargin +} + +client.createTemplate("production-template", createEnvironmentTemplate("production", 100)) +client.createTemplate("staging-template", createEnvironmentTemplate("staging", 50)) +client.createTemplate("development-template", createEnvironmentTemplate("development", 10)) +``` + +### 3. Dynamic Mapping with Templates + +```scala +val dynamicTemplate = + """ + |{ + | "index_patterns": ["dynamic-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | }, + | "mappings": { + | "_doc": { + | "dynamic_templates": [ + | { + | "strings_as_keywords": { + | "match_mapping_type": "string", + | "mapping": { + | "type": "keyword" + | } + | } + | }, + | { + | "longs_as_integers": { + | "match_mapping_type": "long", + | "mapping": { + | "type": "integer" + | } + | } + | } + | ], + | "properties": { + | "timestamp": { + | "type": "date" + | } + | } + | } + | } + |} + |""".stripMargin + +client.createTemplate("dynamic-template", dynamicTemplate) +``` + +### 4. Filtered Aliases + +```scala +val templateWithFilteredAliases = + """ + |{ + | "index_patterns": ["events-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 2 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "event_type": {"type": "keyword"}, + | "severity": {"type": "keyword"}, + | "timestamp": {"type": "date"} + | } + | } + | }, + | "aliases": { + | "events-all": {}, + | "events-errors": { + | "filter": { + | "term": { + | "severity": "error" + | } + | } + | }, + | "events-warnings": { + | "filter": { + | "term": { + | "severity": "warning" + | } + | } + | }, + | "events-critical": { + | "filter": { + | "terms": { + | "severity": ["critical", "fatal"] + | } + | }, + | "routing": "1" + | } + | } + |} + |""".stripMargin + +client.createTemplate("events-template", templateWithFilteredAliases) + +// Search specific severity levels through aliases +client.search("events-errors", query) +client.search("events-critical", query) +``` + +--- + +## Version Compatibility + +### Automatic Format Normalization + +The API automatically normalizes template format based on Elasticsearch version: + +```scala +def normalizeTemplate(templateJson: String, elasticVersion: String): Try[String] +``` + +**Normalization Logic:** + +| ES Version | Input Format | Action Taken | +|------------|-------------|--------------| +| **ES 7.8+** | Legacy | Convert to Composable | +| **ES 7.8+** | Composable | Keep as-is | +| **ES 6.8 - 7.7** | Legacy | Keep as-is | +| **ES 6.8 - 7.7** | Composable | Convert to Legacy | +| **ES < 6.8** | Legacy | Ensure `_doc` wrapper in mappings | +| **ES < 6.8** | Composable | Convert to Legacy + ensure `_doc` wrapper | + +--- + +### Type Name Handling + +#### ES < 6.8: Requires `_doc` Type Wrapper + +For Elasticsearch versions **before 6.8**, mappings **must** be wrapped in a type name (typically `_doc`): + +**Input (Typeless Mappings):** + +```json +{ + "index_patterns": ["logs-*"], + "mappings": { + "properties": { + "message": {"type": "text"} + } + } +} +``` + +**Normalized Output (ES < 6.8):** + +```json +{ + "index_patterns": ["logs-*"], + "mappings": { + "_doc": { + "properties": { + "message": {"type": "text"} + } + } + } +} +``` + +**Code Example:** + +```scala +// ES < 6.8 +val template = + """ + |{ + | "index_patterns": ["logs-*"], + | "order": 1, + | "mappings": { + | "properties": { + | "timestamp": {"type": "date"}, + | "message": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// API automatically wraps mappings in _doc for ES < 6.8 +client.createTemplate("logs-template", template) + +// Resulting template in ES < 6.8: +// { +// "index_patterns": ["logs-*"], +// "order": 1, +// "mappings": { +// "_doc": { +// "properties": { +// "timestamp": {"type": "date"}, +// "message": {"type": "text"} +// } +// } +// } +// } +``` + +--- + +#### ES 6.8 - 7.x: Optional Type Names + +Elasticsearch 6.8+ supports typeless mappings but still accepts `_doc` wrapper for backward compatibility: + +**Both formats work:** + +```scala +// Format 1: With _doc wrapper (backward compatible) +val templateWithType = + """ + |{ + | "index_patterns": ["logs-*"], + | "mappings": { + | "_doc": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + |} + |""".stripMargin + +// Format 2: Typeless (modern) +val templateTypeless = + """ + |{ + | "index_patterns": ["logs-*"], + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// Both work in ES 6.8-7.x +client.createTemplate("template1", templateWithType) +client.createTemplate("template2", templateTypeless) +``` + +--- + +#### ES 7.8+: Typeless Only (Composable Templates) + +Composable templates in ES 7.8+ **do not support type names**: + +```scala +val composableTemplate = + """ + |{ + | "index_patterns": ["metrics-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "properties": { + | "cpu": {"type": "float"} + | } + | } + | } + |} + |""".stripMargin + +// No _doc wrapper needed +client.createTemplate("metrics-template", composableTemplate) +``` + +--- + +### Composable β†’ Legacy Conversion + +When using composable templates on **ES < 7.8**, the API automatically converts to legacy format: + +**Input (Composable Format):** + +```json +{ + "index_patterns": ["app-*"], + "priority": 200, + "version": 2, + "template": { + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "properties": { + "user_id": {"type": "keyword"} + } + }, + "aliases": { + "app-current": {} + } + }, + "composed_of": ["component1"], + "_meta": { + "description": "App template" + } +} +``` + +**Normalized Output (ES 6.8):** + +```json +{ + "index_patterns": ["app-*"], + "order": 200, + "version": 2, + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "_doc": { + "properties": { + "user_id": {"type": "keyword"} + } + } + }, + "aliases": { + "app-current": {} + } +} +``` + +**Conversion Rules:** + +| Composable Field | Legacy Field | Notes | +|-----------------|--------------|-------| +| `priority` | `order` | Direct value mapping | +| `template.settings` | `settings` | Flattened to root level | +| `template.mappings` | `mappings` | Flattened + wrapped in `_doc` for ES < 6.8 | +| `template.aliases` | `aliases` | Flattened to root level | +| `composed_of` | ❌ Ignored | Not supported in legacy format (warning logged) | +| `_meta` | ❌ Ignored | Not supported in legacy format | +| `data_stream` | ❌ Ignored | Not supported in legacy format (warning logged) | + +**Code Example:** + +```scala +// Using composable format on ES 6.8 +val composableTemplate = + """ + |{ + | "index_patterns": ["events-*"], + | "priority": 150, + | "template": { + | "settings": { + | "number_of_shards": 2 + | }, + | "mappings": { + | "properties": { + | "event_type": {"type": "keyword"} + | } + | } + | }, + | "composed_of": ["base-settings"], + | "_meta": { + | "owner": "analytics-team" + | } + |} + |""".stripMargin + +// API automatically converts for ES 6.8: +// - priority β†’ order +// - Flattens template object +// - Wraps mappings in _doc +// - Logs warnings for composed_of and _meta +client.createTemplate("events-template", composableTemplate) + +// Logs: +// [WARN] Composable templates are not supported in legacy template format and will be ignored +// [DEBUG] Wrapped mappings in '_doc' for ES 6.8.0 +``` + +--- + +### Legacy β†’ Composable Conversion + +When using legacy templates on **ES 7.8+**, the API can optionally convert to composable format: + +**Input (Legacy Format):** + +```json +{ + "index_patterns": ["logs-*"], + "order": 10, + "version": 1, + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "_doc": { + "properties": { + "message": {"type": "text"} + } + } + }, + "aliases": { + "logs-all": {} + } +} +``` + +**Normalized Output (ES 7.8+):** + +```json +{ + "index_patterns": ["logs-*"], + "priority": 10, + "version": 1, + "template": { + "settings": { + "number_of_shards": 3 + }, + "mappings": { + "properties": { + "message": {"type": "text"} + } + }, + "aliases": { + "logs-all": {} + } + } +} +``` + +**Conversion Rules:** + +| Legacy Field | Composable Field | Notes | +|-------------|------------------|-------| +| `order` | `priority` | Direct value mapping | +| `settings` | `template.settings` | Nested under `template` | +| `mappings` | `template.mappings` | Nested + `_doc` wrapper removed | +| `aliases` | `template.aliases` | Nested under `template` | + +--- + +### Version Detection Helpers + +The API uses internal helpers to determine version-specific behavior: + +```scala +object ElasticsearchVersion { + + /** Check if ES version supports composable templates (7.8+) + */ + def supportsComposableTemplates(version: String): Boolean = { + // Returns true for ES >= 7.8 + } + + /** Check if ES version requires _doc type wrapper (< 6.8) + */ + def requiresDocTypeWrapper(version: String): Boolean = { + // Returns true for ES < 6.8 + } + + /** Check if ES version requires include_type_name parameter (6.8 - 7.x) + */ + def requiresIncludeTypeName(version: String): Boolean = { + // Returns true for ES 6.8 - 7.x + } +} +``` + +**Usage Example:** + +```scala +def createTemplateForVersion( + name: String, + template: String, + esVersion: String +): ElasticResult[Boolean] = { + + // Normalize based on version + val normalized = TemplateConverter.normalizeTemplate(template, esVersion) match { + case Success(json) => json + case Failure(e) => return ElasticResult.failure(e.getMessage) + } + + // Version-specific handling + if (ElasticsearchVersion.supportsComposableTemplates(esVersion)) { + println("Using composable template API") + executeCreateComposableTemplate(name, normalized) + } else if (ElasticsearchVersion.requiresIncludeTypeName(esVersion)) { + println("Using legacy template API with include_type_name") + executeCreateLegacyTemplateWithTypeName(name, normalized) + } else { + println("Using legacy template API") + executeCreateLegacyTemplate(name, normalized) + } +} +``` + +--- + +### Complete Version Matrix + +| ES Version | Template Type | Mappings Format | `include_type_name` | API Endpoint | +|---------------|------------------------|-------------------------|----------------------|----------------------------------------------------------| +| **6.0 - 6.7** | Legacy | `_doc` wrapper required | ❌ Not supported | `PUT /_template/{name}` | +| **6.8 - 7.7** | Legacy | Typeless or `_doc` | βœ… Optional | `PUT /_template/{name}` | +| **7.8+** | Legacy or Composable | Typeless only | ❌ Deprecated | `PUT /_template/{name}` or `PUT /_index_template/{name}` | +| **8.0+** | Composable (preferred) | Typeless only | ❌ Removed | `PUT /_index_template/{name}` | + +--- + +### Troubleshooting Version Issues + +#### Issue: Template creation fails with "unknown field [_doc]" + +**Cause:** Using `_doc` wrapper on ES 7.8+ composable templates + +**Solution:** + +```scala +// The API automatically unwrap _doc wrapper for ES >= 6.8 +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "_doc": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + | } + |} + |""".stripMargin + +// βœ… Correct for ES 6.8+ +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "priority": 100, + | "template": { + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + | } + |} + |""".stripMargin +``` + +#### Issue: Template creation fails with "missing type name" + +**Cause:** Missing `_doc` wrapper on ES < 6.8 + +**Solution :** + +```scala +// The API automatically adds _doc wrapper for ES < 6.8 +// No manual action needed - just use typeless format +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "mappings": { + | "properties": { + | "field": {"type": "text"} + | } + | } + |} + |""".stripMargin + +// API automatically wraps for ES < 6.8 +client.createTemplate("test-template", template) +``` + +#### Issue: Warning "Composable templates are not supported" + +**Cause:** Using `composed_of` or `data_stream` on ES < 7.8 + +**Solution :** + +```scala +// These fields are automatically ignored with warning +// Remove them for ES < 7.8 to avoid confusion +val template = + """ + |{ + | "index_patterns": ["test-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + +// No composed_of or _meta fields +client.createTemplate("test-template", template) +``` + +--- + +## Error Handling + +### Invalid Template Name + +```scala +client.createTemplate("", templateDefinition) match { + case ElasticFailure(error) => + assert(error.statusCode.contains(400)) + assert(error.operation.contains("createTemplate")) + assert(error.message.contains("Invalid")) +} +``` + +### Invalid JSON + +```scala +val invalidJson = """{"index_patterns": ["test-*"}""" // Missing closing brace + +client.createTemplate("test", invalidJson) match { + case ElasticFailure(error) => + println(s"JSON parsing error: ${error.message}") +} +``` + +### Template Not Found + +```scala +client.getTemplate("non-existent") match { + case ElasticSuccess(None) => + println("Template not found") + case ElasticSuccess(Some(json)) => + println(s"Template: $json") + case ElasticFailure(e) => + println(s"Error: ${e.message}") +} +``` + +### Conflicting Templates + +```scala +// Template with same pattern but different order +val template1 = """{"index_patterns": ["app-*"], "order": 1}""" +val template2 = """{"index_patterns": ["app-*"], "order": 10}""" + +client.createTemplate("template1", template1) +client.createTemplate("template2", template2) + +// template2 will take precedence due to higher order +``` + +--- + +## Performance Considerations + +### Template Application + +⚠️ Templates are applied **only when indices are created**, not to existing indices. + +```scala +// ❌ Bad - template won't affect existing index +client.createIndex("logs-2024") +client.createTemplate("logs-template", template) // Too late! + +// βœ… Good - create template first +client.createTemplate("logs-template", template) +client.createIndex("logs-2024") // Template applied +``` + +### Bulk Template Creation + +```scala +// βœ… Create templates before bulk indexing +val templates = Map( + "logs-template" -> logsTemplate, + "metrics-template" -> metricsTemplate, + "events-template" -> eventsTemplate +) + +templates.foreach { case (name, definition) => + client.createTemplate(name, definition, ifExists = true) +} + +// Now bulk create indices +val indices = List("logs-2024", "metrics-2024", "events-2024") +indices.foreach(client.createIndex) +``` + +### Template Caching + +```scala +// Templates are cached by Elasticsearch +// Frequent updates can impact performance + +// ❌ Bad - updating template repeatedly +(1 to 100).foreach { i => + val updated = template.replace("\"version\": 1", s""""version": $i""") + client.createTemplate("my-template", updated) +} + +// βœ… Good - update once with final configuration +client.createTemplate("my-template", finalTemplate) +``` + +--- + +## Implementation Requirements + +### executeCreateLegacyTemplate + +Must be implemented by each client-specific trait for legacy templates. + +**Signature:** + +```scala +private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**REST High Level Client (ES 6-7):** + +```scala +private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) +} +``` + +### executeCreateComposableTemplate + +Must be implemented for composable templates (ES 7.8+). + +**Signature:** + +```scala +private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] +``` + +**Java Client (ES 8+):** + +```scala +private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String +): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "createTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) +} +``` + +### executeGetLegacyTemplate / executeGetComposableTemplate + +Retrieves legacy / composable template configuration. + +**Signature:** + +```scala +private[client] def executeGetLegacyTemplate( + templateName: String +): ElasticResult[Option[String]] +``` + +### executeDeleteLegacyTemplate / executeDeleteComposableTemplate + +Deletes legacy / composable template. + +**Signature:** + +```scala +private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean +): ElasticResult[Boolean] +``` + +### executeListLegacyTemplates / executeListComposableTemplates + +Retrieves legacy / composable template configurations. + +**Signature:** + +```scala +private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] +``` + +--- + +## Testing Examples + +### Complete Test Suite + +```scala +class TemplateApiSpec extends AnyFlatSpec with Matchers { + + val client: ElasticClient = // Initialize client + + "createTemplate" should "create a legacy template" in { + val template = + """ + |{ + | "index_patterns": ["test-*"], + | "order": 1, + | "settings": { + | "number_of_shards": 1 + | } + |} + |""".stripMargin + + client.createTemplate("test-template", template) shouldBe a[ElasticSuccess[_]] + + client.templateExists("test-template") match { + case ElasticSuccess(true) => succeed + case _ => fail("Template not created") + } + + client.deleteTemplate("test-template", ifExists = true) + } + + it should "update an existing template" in { + val template1 = """{"index_patterns": ["update-*"], "order": 1}""" + val template2 = """{"index_patterns": ["update-*"], "order": 2, "version": 2}""" + + client.createTemplate("update-test", template1) + client.createTemplate("update-test", template2) + + client.getTemplate("update-test") match { + case ElasticSuccess(Some(json)) => + json should include("\"order\":2") + json should include("\"version\":2") + case _ => fail("Failed to get updated template") + } + + client.deleteTemplate("update-test", ifExists = true) + } + + "getTemplate" should "return None for non-existent template" in { + client.getTemplate("non-existent-template") match { + case ElasticSuccess(None) => succeed + case _ => fail("Should return None") + } + } + + "deleteTemplate" should "delete existing template" in { + val template = """{"index_patterns": ["delete-*"], "order": 1}""" + + client.createTemplate("delete-test", template) + client.deleteTemplate("delete-test") shouldBe a[ElasticSuccess[_]] + + client.templateExists("delete-test") match { + case ElasticSuccess(false) => succeed + case _ => fail("Template not deleted") + } + } + + it should "succeed with ifExists=true for non-existent template" in { + client.deleteTemplate("non-existent", ifExists = true) match { + case ElasticSuccess(false) => succeed + case _ => fail("Should succeed with ifExists=true") + } + } + + "templateExists" should "return true for existing template" in { + val template = """{"index_patterns": ["exists-*"], "order": 1}""" + + client.createTemplate("exists-test", template) + + client.templateExists("exists-test") match { + case ElasticSuccess(true) => succeed + case _ => fail("Template should exist") + } + + client.deleteTemplate("exists-test", ifExists = true) + } + + it should "return false for non-existent template" in { + client.templateExists("non-existent") match { + case ElasticSuccess(false) => succeed + case _ => fail("Template should not exist") + } + } +} +``` + +--- + +[Back to index](README.md) | [Next: Component Templates](component-templates.md) \ No newline at end of file diff --git a/documentation/sql/README.md b/documentation/sql/README.md index d383815f..bd2cbdad 100644 --- a/documentation/sql/README.md +++ b/documentation/sql/README.md @@ -2,7 +2,7 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: -- [Query Structure](request_structure.md) +- [Query Structure](dql_statements.md) - [Query Validation](validation.md) - [Operators](operators.md) - [Operator Precedence](operator_precedence.md) @@ -14,3 +14,6 @@ Welcome to the SQL Engine Documentation. Navigate through the sections below: - [Conditional Functions](functions_conditional.md) - [Geo Functions](functions_geo.md) - [Keywords](keywords.md) +- [DDL Support](ddl_statements.md) +- [DML Support](dml_statements.md) +- [DQL Support](dql_statements.md) diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md new file mode 100644 index 00000000..d0ba8481 --- /dev/null +++ b/documentation/sql/ddl_statements.md @@ -0,0 +1,587 @@ +[Back to index](README.md) + +# πŸ“˜ **DDL Statements β€” SQL Gateway for Elasticsearch** + +--- + +## Introduction + +The SQL Gateway provides a full Data Definition Language (DDL) layer on top of Elasticsearch. +It allows defining tables, schemas, pipelines, mappings, and settings using a relational syntax while generating the appropriate Elasticsearch structures: + +- **indices** (for non-partitioned tables) +- **index templates** (for partitioned tables) +- **ingest pipelines** (default or user-defined) +- **mappings** and **settings** +- **metadata** (primary key, generated columns, comments, options) + +The DDL engine is: + +- **version-aware** (ES6 β†’ ES9) +- **client-agnostic** (Jest, RHLC, Java Client) +- **schema-driven** +- **round-trip safe** (DESCRIBE returns normalized SQL) + +--- + +## Table Model + +A SQL table corresponds to: + +| SQL Definition | Elasticsearch Structure | +|-----------------------------------------|----------------------------------------------------| +| `CREATE TABLE` without `PARTITIONED BY` | **Concrete index** | +| `CREATE TABLE` with `PARTITIONED BY` | **Index template** (legacy ES6 or composable ES7+) | + +### Index-backed table (no partitioning) + +```sql +CREATE TABLE users ( + id INT, + name VARCHAR, + PRIMARY KEY (id) +); +``` + +Creates: + +- index `users` +- default pipeline `users_ddl_default_pipeline` +- mapping + settings + +### Template-backed table (with partitioning) + +```sql +CREATE TABLE users ( + id INT, + birthdate DATE, + PRIMARY KEY (id) +) +PARTITIONED BY (birthdate MONTH); +``` + +Creates: + +- template `users` +- default pipeline with `date_index_name` +- indices generated dynamically: + - `users-2025-01` + - `users-2025-02` + +--- + +## Column Types & Mapping + +The SQL Gateway supports the following type system: + +| SQL Type | Elasticsearch Mapping | +|---------------------|--------------------------------------| +| `NULL` | `null` | +| `TINYINT` | `byte` | +| `SMALLINT` | `short` | +| `INT` | `integer` | +| `BIGINT` | `long` | +| `DOUBLE` | `double` | +| `REAL` | `float` | +| `BOOLEAN` | `boolean` | +| `VARCHAR` \| `TEXT` | `text` + optional `keyword` subfield | +| `KEYWORD` | `keyword` | +| `DATE` | `date` | +| `TIMESTAMP` | `date` | +| `STRUCT` | `object` with nested properties | +| `ARRAY` | `nested` | +| `GEO_POINT` | `geo_point` | + +--- + +### 🧩 Nested and Structured Data + +#### FIELDS for Multi-fields + +`FIELDS (...)` can be used to define **multi-fields** for text columns. +This allows indexing the same column in multiple ways (e.g., with different analyzers). + +**Example:** +```sql +CREATE TABLE docs ( + content VARCHAR FIELDS ( + keyword VARCHAR OPTIONS (analyzer = 'keyword'), + english VARCHAR OPTIONS (analyzer = 'english') + ) +) +``` + +- `content` is indexed as text. +- `content.keyword` is a keyword sub-field. +- `content.english` is a text sub-field with the English analyzer. + +--- + +#### FIELDS for STRUCT or NESTED OBJECTS + +`FIELDS (...)` also enables the definition of **STRUCT** types, representing hierarchical data. + +**Example:** +```sql +CREATE TABLE users ( + id INT NOT NULL, + profile STRUCT FIELDS ( + first_name VARCHAR NOT NULL, + last_name VARCHAR NOT NULL, + address STRUCT FIELDS ( + street VARCHAR, + city VARCHAR, + zip VARCHAR + ), + join_date DATE, + seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + ) +) +``` + +- `profile` is a `STRUCT` column containing multiple fields. +- `address` is a nested `STRUCT` inside `profile`. + +--- + +#### FIELDS for ARRAY + +**Example:** +```sql +CREATE TABLE store ( + id INT NOT NULL, + products ARRAY FIELDS ( + name VARCHAR NOT NULL, + description VARCHAR NOT NULL, + price BIGINT NOT NULL + ) +) +``` + +- `products` is an `ARRAY` column. +- Maps naturally to Elasticsearch `nested`. + +--- + +#### Notes + +- On `VARCHAR` β†’ defines **multi-fields** +- On `STRUCT` β†’ defines **object fields** +- On `ARRAY` β†’ defines **nested fields** +- Sub-fields support: + - nested `FIELDS` + - `DEFAULT` + - `NOT NULL` + - `COMMENT` + - `OPTIONS` + - `SCRIPT AS` (except inside ARRAY) +- Multi-level nesting is supported. + +--- + +## Constraints & Column Options + +### Primary Key + +```sql +id INT, +PRIMARY KEY (id) +``` + +Used for: + +- document ID generation +- upsert semantics +- COPY INTO conflict resolution + +--- + +### πŸ”‘ Composite Primary Keys + +SoftClient4ES supports composite primary keys in SQL. + +#### Syntax + +```sql +CREATE TABLE users ( + id INT NOT NULL, + birthdate DATE NOT NULL, + name VARCHAR, + PRIMARY KEY (id, birthdate) +); +``` + +#### Elasticsearch Translation + +```json +{ + "processors": [ + { + "set": { + "field": "_id", + "value": "{{id}}||{{birthdate}}" + } + } + ] +} +``` + +#### Notes + +- Composite PK fields must be immutable. +- Avoid long `_id` values. +- Mapping rules: + - `PRIMARY KEY (id)` β†’ `_id = id` + - `PRIMARY KEY (id, birthdate)` β†’ `_id = "{{id}}-{{birthdate}}"` + +--- + +## Partitioning + +Partitioning routes documents to time-based indices using `date_index_name`. + +### Supported Granularities + +| SQL Granularity | ES `date_rounding` | Example Index Name | +|-----------------|--------------------|---------------------------| +| YEAR | "y" | users-2025 | +| MONTH | "M" | users-2025-12 | +| DAY (default) | "d" | users-2025-12-10 | +| HOUR | "h" | users-2025-12-10-09 | +| MINUTE | "m" | users-2025-12-10-09-46 | +| SECOND | "s" | users-2025-12-10-09-46-30 | + +--- + +## Pipelines in DDL + +## CREATE PIPELINE + +```sql +CREATE OR REPLACE PIPELINE user_pipeline +WITH PROCESSORS ( + SET ( + field = "name", + if = "ctx.name == null", + description = "DEFAULT 'anonymous'", + ignore_failure = true, + value = "anonymous" + ), + SCRIPT ( + description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + lang = "painless", + source = "...", + ignore_failure = true + ), + DATE_INDEX_NAME ( + field = "birthdate", + index_name_prefix = "users-", + date_formats = ["yyyy-MM"], + date_rounding = "M", + separator = "-", + ignore_failure = true + ) +); +``` + +## DROP PIPELINE + +```sql +DROP PIPELINE IF EXISTS user_pipeline; +``` + +## ALTER PIPELINE + +```sql +ALTER PIPELINE IF EXISTS user_pipeline ( + ADD PROCESSOR SET ( + field = "status", + if = "ctx.status == null", + description = "status DEFAULT 'active'", + ignore_failure = true, + value = "active" + ), + DROP PROCESSOR SET (_id) +); +``` + +--- + +## SHOW PIPELINE + +```sql +SHOW PIPELINE pipeline_name; +``` + +**Description** + +- Returns a high‑level view of the pipeline processors + +**Example** + +```sql +SHOW PIPELINE user_pipeline; +``` + +--- + +## DESCRIBE PIPELINE + +```sql +DESCRIBE PIPELINE pipeline_name; +``` + +**Description** + +- Returns the full, normalized definition of the pipeline: + - processors in execution order + - full configuration of each processor (`SET`, `SCRIPT`, `REMOVE`, `RENAME`, `DATE_INDEX_NAME`, etc.) + - flags such as `ignore_failure`, `if`, `description` + +**Example** + +```sql +DESCRIBE PIPELINE user_pipeline; +``` + +--- + +## CREATE TABLE + +### Basic Example + +```sql +CREATE TABLE users ( + id INT, + name VARCHAR DEFAULT 'anonymous', + birthdate DATE, + age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), + PRIMARY KEY (id) +); +``` + +### Partitioned Example + +```sql +CREATE TABLE users ( + id INT, + birthdate DATE, + PRIMARY KEY (id) +) +PARTITIONED BY (birthdate MONTH); +``` + +--- + +## ALTER TABLE + +**Supported statements:** + +- `ADD COLUMN [IF NOT EXISTS] column_definition` +- `DROP COLUMN [IF EXISTS] column_name` +- `RENAME COLUMN old_name TO new_name` +- `ALTER COLUMN column_name SET SCRIPT AS (sql)` +- `ALTER COLUMN column_name DROP SCRIPT` +- `ALTER COLUMN column_name SET|ADD OPTION (key = value)` +- `ALTER COLUMN column_name DROP OPTION key` +- `ALTER COLUMN column_name SET COMMENT 'comment'` +- `ALTER COLUMN column_name DROP COMMENT` +- `ALTER COLUMN column_name SET DEFAULT value` +- `ALTER COLUMN column_name DROP DEFAULT` +- `ALTER COLUMN column_name SET NOT NULL` +- `ALTER COLUMN column_name DROP NOT NULL` +- `ALTER COLUMN column_name SET DATA TYPE new_type` +- `ALTER COLUMN column_name SET|ADD FIELD field_definition` +- `ALTER COLUMN column_name DROP FIELD field_name` +- `ALTER COLUMN column_name SET FIELDS (...)` +- `SET|ADD MAPPING (key = value)` +- `DROP MAPPING key` +- `SET|ADD SETTING (key = value)` +- `DROP SETTING key` + +### Type Changes and Safety + +When applying `ALTER COLUMN column_name SET DATA TYPE new_type`, the SQL Gateway computes a structural diff between the current schema and the target schema. + +Type changes fall into two categories: + +- **Convertible types** (`SQLTypeUtils.canConvert(from, to) = true`) + The change is allowed but requires a full **reindex** of the underlying data. + The Gateway automatically performs the reindex operation and swaps aliases when complete. + These changes are classified as `UnsafeReindex`. + +- **Incompatible types** (`SQLTypeUtils.canConvert(from, to) = false`) + The change is **not allowed** and the `ALTER TABLE` statement fails. + These changes are classified as `Impossible`. + +This is the only case where an `ALTER TABLE` operation can be rejected for safety reasons. +All other ALTER operations (adding/dropping columns, renaming, modifying options, modifying nested fields, etc.) are allowed. + +--- + +## DROP TABLE + +```sql +DROP TABLE IF EXISTS users; +``` + +Deletes: + +- index (non-partitioned) +- template (partitioned) + +--- + +## TRUNCATE TABLE + +```sql +TRUNCATE TABLE users; +``` + +Deletes all documents while keeping: + +- mapping +- settings +- pipeline +- template (if any) + +--- + +## SHOW TABLE + +```sql +SHOW TABLE users; +``` + +Returns: + +- index or template metadata +- primary key +- partitioning +- pipeline +- mapping summary + +--- + +## DESCRIBE TABLE + +```sql +DESCRIBE TABLE users; +``` + +Returns the **normalized SQL schema**, including: + +- columns +- types +- defaults +- scripts +- STRUCT fields +- PK +- options +- comments + +--- + +## CREATE TABLE AS SELECT (CTAS) + +```sql +CREATE TABLE new_users AS +SELECT id, name FROM users; +``` + +The gateway: + +- infers the schema +- generates mappings +- creates index or template +- **populates data using the Bulk API** + +--- + +## πŸ”„ Index Migration Workflow + +### Initial Creation + +```sql +CREATE TABLE users (...); +``` + +Creates: + +- index or template +- default pipeline +- mapping + settings +- metadata (PK, defaults, scripts) + +--- + +### Schema Evolution + +#### Add a column + +```sql +ALTER TABLE users ADD COLUMN last_login TIMESTAMP; +``` + +#### Modify a column + +```sql +ALTER TABLE users ALTER COLUMN name SET OPTIONS (analyzer = 'french'); +``` + +#### Add a STRUCT field + +```sql +ALTER TABLE users ALTER COLUMN profile ADD FIELD followers INT; +``` + +#### Drop a column + +```sql +ALTER TABLE users DROP COLUMN old_field; +``` + +--- + +### Migration Safety + +The Gateway ensures: + +- non-destructive updates +- mapping compatibility checks +- pipeline regeneration when needed +- template updates for partitioned tables +- index updates for non-partitioned tables + +--- + +### Full Replacement (CTAS) + +```sql +CREATE OR REPLACE TABLE users AS +SELECT id, name FROM old_users; +``` + +Steps: + +1. infer schema +2. create new index/template +3. bulk-copy data +4. atomically replace + +--- + +## Version Compatibility + +| Feature | ES6 | ES7 | ES8 | ES9 | +|----------------------|------|------|------|------| +| Legacy templates | βœ” | βœ” | βœ– | βœ– | +| Composable templates | βœ– | βœ” | βœ” | βœ” | +| date_index_name | βœ” | βœ” | βœ” | βœ” | +| Generated scripts | βœ” | βœ” | βœ” | βœ” | +| STRUCT | βœ” | βœ” | βœ” | βœ” | +| ARRAY | βœ” | βœ” | βœ” | βœ” | + +--- + +[Back to index](README.md) diff --git a/documentation/sql/dml_statements.md b/documentation/sql/dml_statements.md new file mode 100644 index 00000000..c2350264 --- /dev/null +++ b/documentation/sql/dml_statements.md @@ -0,0 +1,362 @@ +[Back to index](README.md) + +# πŸ“˜ DML Statements β€” SQL Gateway for Elasticsearch + +--- + +## Introduction + +The SQL Gateway supports the following Data Manipulation Language (DML) operations: + +- **INSERT INTO ... VALUES** +- **INSERT INTO ... AS SELECT ... [ON CONFLICT ...]** +- **UPDATE ... SET ... [WHERE]** +- **DELETE FROM ... [WHERE]** +- **COPY INTO ...** (bulk ingestion) + +The DML engine is: + +- **schema-aware** (PK, defaults, scripts, STRUCT, ARRAY) +- **pipeline-aware** (default pipeline + user pipelines) +- **partition-aware** (date-based routing) +- **primary-key-aware** (upsert semantics) +- **version-aware** (ES6 β†’ ES9) + +All DML operations are translated into Elasticsearch write APIs: + +- `INSERT` β†’ index API (bulk) +- `INSERT ... AS SELECT` β†’ search + bulk index +- `UPDATE` β†’ update-by-query +- `DELETE` β†’ delete-by-query +- `COPY INTO` β†’ bulk from file + +`DELETE` without a `WHERE` clause is allowed and behaves like a **TRUNCATE TABLE**, removing all documents from the index. + +Each DML statement returns: + +```scala + case class DmlResult( + inserted: Long = 0L, + updated: Long = 0L, + deleted: Long = 0L, + rejected: Long = 0L + ) extends QueryResult +``` + +--- + +## INSERT + +### INSERT INTO ... VALUES + +```sql +INSERT INTO table_name (col1, col2, ...) +VALUES (v1, v2, ...), (v3, v4, ...), ...; +``` + +INSERT operations: + +- use the table’s **primary key** to generate `_id` +- pass documents through the **table pipeline** +- support: + - STRUCT + - ARRAY + - DEFAULT values + - SCRIPT AS generated columns + - NOT NULL constraints + +--- + +### Full Example: INSERT INTO with STRUCT + +```sql +CREATE TABLE IF NOT EXISTS dql_users ( + id INT NOT NULL, + name VARCHAR FIELDS( + raw KEYWORD + ) OPTIONS (fielddata = true), + age INT SCRIPT AS (YEAR(CURRENT_DATE) - YEAR(birthdate)), + birthdate DATE, + profile STRUCT FIELDS( + city VARCHAR OPTIONS (fielddata = true), + followers INT + ) +); + +INSERT INTO dql_users (id, name, birthdate, profile) VALUES + (1, 'Alice', '1994-01-01', {city = "Paris", followers = 100}), + (2, 'Bob', '1984-05-10', {city = "Lyon", followers = 50}), + (3, 'Chloe', '1999-07-20', {city = "Paris", followers = 200}), + (4, 'David', '1974-03-15', {city = "Marseille", followers = 10}); +``` + +**Behavior** + +- `profile` β†’ Elasticsearch `object` +- `name.raw` β†’ multi-field +- DEFAULT, SCRIPT, NOT NULL β†’ applied via pipeline +- PK β†’ `_id` generated automatically + +--- + +### Notes: + +- Fields not provided in the `VALUES` clause are simply omitted from the document (Elasticsearch does not store `NULL`). +- Unknown fields are rejected if the index mapping is strict. +- All values are validated against the Elasticsearch mapping (type, format, date patterns, etc.). +- Nested structures (`STRUCT`, `ARRAY`) must match the expected JSON shape. + +--- + +### INSERT INTO ... AS SELECT ... [ON CONFLICT ...] + +```sql +INSERT INTO orders (order_id, customer_id, order_date, total) +AS SELECT + id AS order_id, + cust AS customer_id, + date AS order_date, + amount AS total +FROM staging_orders; +``` + +**Behavior** + +- The SELECT is executed by the Gateway +- Results are inserted using the Bulk API +- Table pipelines are applied +- PK ensures upsert semantics + +--- + +### INSERT INTO ... AS SELECT ... [ON CONFLICT ...] β€” Validation Workflow** + +Before executing an `INSERT ... AS SELECT`, the Gateway performs a full validation pipeline to ensure schema correctness and safe upsert behavior. + +### **1. Index Validation** +Ensures the target index name is valid. +Invalid names return a 400 error. + +### **2. SQL Parsing** +Extracts: +- target columns +- SELECT statement +- ON CONFLICT clause +- DO UPDATE flag +- conflict target columns + +### **3. Load Index Metadata** +Loads the real Elasticsearch schema: +- primary key +- partitioning +- mapping +- settings + +### **3.b Determine Effective Insert Columns** +If the INSERT column list is omitted, the Gateway derives it from the SELECT output. + +### **3.c Validate ON CONFLICT Rules** +- If the index has a primary key: + - conflict target must match the PK exactly + - INSERT must include all PK columns +- If the index has no primary key: + - ON CONFLICT requires an explicit conflict target + - all conflict target columns must be included in INSERT + +### **3.d Validate SELECT Output Columns** +Ensures : +- every INSERT column exists in the SELECT output +- aliases are resolved +- SELECT is valid + +Otherwise, a 400 error is returned. + +### **4. Derive Bulk Options** +Determines: +- `_id` generation (PK or composite PK) +- partitioning suffix +- upsert behavior (`update = DO UPDATE`) + +### **5. Build Document Source** +- For VALUES: convert to JSON array +- For SELECT: scroll the query and convert rows to JSON + +### **6. Bulk Insert** +Uses the Bulk API with: +- PK-based `_id` +- partitioning +- pipeline execution +- upsert if DO UPDATE + +Returns: + +```scala +DmlResult(inserted = N, rejected = M) +``` + +--- + +### Notes + +- All columns listed in the `INSERT` target must be present in the `SELECT` list. +- The order of columns does not matter; matching is done by column name. +- Extra columns in the `SELECT` that are not part of the target table are ignored. +- Nested structures (`STRUCT`, `ARRAY`) are supported as long as the selected expressions produce valid JSON structures for the target fields. +- No type validation is performed at the SQL layer; type errors are reported by Elasticsearch at indexing time. + +--- + +## UPDATE ... SET ... [WHERE] + +### Syntax + +```sql +UPDATE table_name +SET col1 = expr1, col2 = expr2, ... +WHERE condition; +``` + +**Behavior** + +- `UPDATE` is implemented using Elasticsearch `update-by-query`. +- Only top-level scalar fields or entire nested objects can be replaced. +- Updating a specific element inside an `ARRAY` is not supported. +- `SET field = NULL` removes the field from the document (Elasticsearch does not store SQL NULL). +- Expressions such as `SET x = x + 1` are not supported (no script-based incremental updates). +- Fields not present in the mapping cannot be added unless dynamic mapping is enabled. + +--- + +## DELETE FROM ... [WHERE] + +**Syntax** + +```sql +DELETE FROM table_name +WHERE condition; +``` + +**Behavior** + +- `DELETE` is implemented using Elasticsearch `delete-by-query`. +- `WHERE` is optional. + - If provided, only matching documents are deleted. + - If omitted, **all documents in the index are deleted** (equivalent to `TRUNCATE TABLE`). +- Deleting a nested field or an element inside an array is not supported; only whole documents can be removed. + +--- + +## COPY INTO ... + +COPY INTO is a **DML operator** that loads documents from external files into an Elasticsearch index. +It uses **only the Bulk API**, not the SQL engine. + +**Syntax** + +```sql +COPY INTO table_name +FROM 'path/to/file' +[FILE_FORMAT = 'JSON' | 'JSON_ARRAY' | 'PARQUET' | 'DELTA_LAKE'] +[ON CONFLICT (pk_column) DO UPDATE]; +``` + +**Behavior** + +COPY INTO performs: + +1. **Index name validation** +2. **Loading of the real Elasticsearch schema** + - mapping + - primary key + - partitioning +3. **Primary key extraction** + - PK β†’ `_id` + - composite PK β†’ concatenated `_id` +4. **Partitioning extraction** + - suffix index name based on date +5. **Bulk ingestion via `bulkFromFile`** +6. **Pipeline execution** +7. **Return of `DmlResult`** + - `inserted` = successfully indexed docs + - `rejected` = Bulk failures + +There are **no** strategies like `insertAfter`, `updateAfter`, or `deleteAfter`. +COPY INTO **does not** perform SQL-level operations β€” everything is Bulk. + +**Full Example** + +- **Table Definition** + +```sql +CREATE TABLE IF NOT EXISTS copy_into_test ( + uuid KEYWORD NOT NULL, + name VARCHAR, + birthDate DATE, + childrenCount INT, + PRIMARY KEY (uuid) +); +``` + +- **Data File** (`example_data.json`) + +``` +{"uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0} +{"uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0} +{"uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 2} +``` + +- **COPY INTO Statement** + +```sql +COPY INTO copy_into_test +FROM 's3://my-bucket/path/to/example_data.json' +FILE_FORMAT = 'JSON' +ON CONFLICT (uuid) DO UPDATE; +``` + +**Behavior** + +- PK = `uuid` β†’ `_id = uuid` +- ON CONFLICT DO UPDATE β†’ Bulk upsert +- Table pipeline is applied +- Partitioning is applied if defined +- Returns `DmlResult(inserted = N, rejected = 0)` + +--- + +## DML Lifecycle Example + +```sql +INSERT INTO dml_chain (id, value) VALUES + (1, 10), + (2, 20), + (3, 30); + +UPDATE dml_chain +SET value = 50 +WHERE id IN (1, 3); + +DELETE FROM dml_chain +WHERE value > 40; + +SELECT * FROM dml_chain ORDER BY id ASC; +``` + +--- + +## Version Compatibility + +| Feature | ES6 | ES7 | ES8 | ES9 | +|------------------|------|------|------|------| +| INSERT | βœ” | βœ” | βœ” | βœ” | +| INSERT AS SELECT | βœ” | βœ” | βœ” | βœ” | +| UPDATE | βœ” | βœ” | βœ” | βœ” | +| DELETE | βœ” | βœ” | βœ” | βœ” | +| COPY INTO | βœ” | βœ” | βœ” | βœ” | +| JSON_ARRAY | βœ” | βœ” | βœ” | βœ” | +| PARQUET | βœ” | βœ” | βœ” | βœ” | +| DELTA_LAKE | βœ” | βœ” | βœ” | βœ” | + +--- + +[Back to index](README.md) diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md new file mode 100644 index 00000000..71d84a7c --- /dev/null +++ b/documentation/sql/dql_statements.md @@ -0,0 +1,698 @@ +[Back to index](README.md) + +# πŸ“˜ DQL Statements β€” SQL Gateway for Elasticsearch + +## Introduction + +The SQL Gateway provides a Data Query Language (DQL) on top of Elasticsearch, centered around the `SELECT` statement. +It offers a familiar SQL experience while translating queries into Elasticsearch search, aggregations, and scroll APIs. + +DQL supports: + +- `SELECT` with expressions, aliases, nested fields, STRUCT and ARRAY +- `WHERE`, `GROUP BY`, `HAVING`, `ORDER BY`, `LIMIT`, `OFFSET` +- `UNION ALL` +- `JOIN UNNEST` on `ARRAY` +- aggregations, parent-level aggregations on nested arrays +- window functions with `OVER` +- rich function support (numeric, string, date/time, geo, conditional, type conversion) + +--- + +## SELECT + +#### Basic syntax + +```sql +SELECT [DISTINCT] expr1, expr2, ... +FROM table_name [alias] +[WHERE condition] +[GROUP BY expr1, expr2, ...] +[HAVING condition] +[ORDER BY expr1 [ASC|DESC], ...] +[LIMIT n] +[OFFSET m]; +``` + +#### Nested fields and aliases + +```sql +SELECT id, + name AS full_name, + profile.city AS city, + profile.followers AS followers +FROM dql_users +ORDER BY id ASC; +``` + +- `profile` is a `STRUCT` column. +- `profile.city` and `profile.followers` access nested fields. +- Aliases (`AS full_name`, `AS city`) are returned as column names. + +--- + +## WHERE + +The `WHERE` clause supports: + +- comparison operators: `=`, `!=`, `<`, `<=`, `>`, `>=` +- logical operators: `AND`, `OR`, `NOT` +- `IN`, `NOT IN` +- `BETWEEN` +- `IS NULL`, `IS NOT NULL` +- `LIKE`, `RLIKE` (regex) +- conditions on nested fields (`profile.city`, `profile.followers`) + +**Example** + +```sql +SELECT id, name, age +FROM dql_users +WHERE (age > 20 AND profile.followers >= 100) + OR (profile.city = 'Lyon' AND age < 50) +ORDER BY age DESC; +``` + +Another example with multiple operators: + +```sql +SELECT id, + age + 10 AS age_plus_10, + name +FROM dql_users +WHERE age BETWEEN 20 AND 50 + AND name IN ('Alice', 'Bob', 'Chloe') + AND name IS NOT NULL + AND (name LIKE 'A%' OR name RLIKE '.*o.*'); +``` + +--- + +## ORDER BY + +`ORDER BY` sorts the result set by one or more expressions. + +- Supports multiple sort keys +- Supports `ASC` and `DESC` +- Supports expressions and nested fields (e.g., `profile.city`) +- When used inside a window function (`OVER`), `ORDER BY` defines the logical ordering of the window + +**Example** + +```sql +SELECT id, name, age +FROM dql_users +ORDER BY age DESC, name ASC +LIMIT 2 OFFSET 1; +``` + +--- + +## LIMIT / OFFSET + +- `LIMIT n` restricts the number of returned rows. +- `OFFSET m` skips the first `m` rows. +- Translated to Elasticsearch `from` + `size`. + +Example: + +```sql +SELECT id, name, age +FROM dql_users +ORDER BY age DESC +LIMIT 10 OFFSET 20; +``` + +--- + +`UNION ALL` combines the results of multiple `SELECT` queries **without removing duplicates**. + +All SELECT statements in a UNION ALL must be **strictly compatible**: + +- **same number of columns** +- **same column names** (after alias resolution) +- **same or implicitly compatible types** + +If these conditions are not met, the Gateway raises a validation error before executing the query. + +**Example** + +```sql +SELECT id, name FROM dql_users WHERE age > 30 +UNION ALL +SELECT id, name FROM dql_users WHERE age <= 30; +``` + +### Execution model + +The SQL Gateway executes `UNION ALL` using **Elasticsearch Multi‑Search (`_msearch`)**: + +1. Each SELECT query is translated into an independent ES search request. +2. All requests are sent in a single `_msearch` call. +3. The Gateway concatenates the results **in order**, without deduplication. +4. ORDER BY, LIMIT, OFFSET apply **per SELECT**, not globally (unless wrapped in a subquery, which is not supported). + +### Notes + +- `UNION ALL` does **not** sort or deduplicate results. +- Column names in the final output are taken from the **first SELECT**. +- All subsequent SELECTs must produce columns with the **same names**. +- Type mismatches should result in a validation error before execution. (⚠️ not implemented yet) + +--- + +## JOIN UNNEST + +The Gateway supports a specific form of join: `JOIN UNNEST` on `ARRAY` columns. + +### Table definition + +```sql +CREATE TABLE IF NOT EXISTS dql_orders ( + id INT NOT NULL, + customer_id INT, + items ARRAY FIELDS( + product VARCHAR OPTIONS (fielddata = true), + quantity INT, + price DOUBLE + ) OPTIONS (include_in_parent = false) +); +``` + +### Query with JOIN UNNEST and window function + +```sql +SELECT + o.id, + items.product, + items.quantity, + SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price +FROM dql_orders o +JOIN UNNEST(o.items) AS items +WHERE items.quantity >= 1 +ORDER BY o.id ASC; +``` + +`JOIN UNNEST` is intended to behave like a standard SQL UNNEST operation, where each element of an `ARRAY` becomes a logical row that can participate in expressions, filters, and window functions. + +⚠️ **Current status:** +The SQL Gateway already supports reading and filtering nested array elements through `JOIN UNNEST`, and supports parent-level aggregations using window functions. +However, **full row-level expansion (one output row per array element)** is **not implemented yet**. + +This means: +- expressions such as `items.price` and `items.quantity` are fully usable +- window functions over `PARTITION BY parent_id` work +- parent-level aggregations can be computed +- but the final output still returns **one row per parent**, not one row per item + +--- + +## Aggregations + +Supported aggregate functions include: + +- `COUNT(*)`, `COUNT(expr)` +- `SUM(expr)` +- `AVG(expr)` +- `MIN(expr)` +- `MAX(expr)` + +### GROUP BY and HAVING + +```sql +SELECT profile.city AS city, + COUNT(*) AS cnt, + AVG(age) AS avg_age +FROM dql_users +GROUP BY profile.city +HAVING COUNT(*) >= 1 +ORDER BY COUNT(*) DESC; +``` + +- `GROUP BY` supports nested fields (`profile.city`). +- `HAVING` filters groups based on aggregate conditions. +- Translated to Elasticsearch aggregations. + +--- + +## Parent-Level Aggregations on Nested Arrays + +The SQL Gateway supports computing aggregations **over nested arrays** (e.g., `ARRAY`) **without exploding them into multiple rows**. + +This pattern: + +- reads the nested array (`JOIN UNNEST`) +- computes aggregations per parent document (`PARTITION BY parent_id`) +- **returns one row per parent** +- **preserves the original nested array** +- **adds the aggregated value as a top-level field** + +**Example** + +```sql +SELECT + o.id, + o.items, + SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price +FROM dql_orders o +JOIN UNNEST(o.items) AS items +WHERE items.quantity >= 1 +ORDER BY o.id ASC; +``` + +**Result** + +```json +[ + { + "id": 1, + "items": [ + {"product": "A", "quantity": 2, "price": 10.0}, + {"product": "B", "quantity": 1, "price": 20.0} + ], + "total_price": 40.0 + }, + { + "id": 2, + "items": [ + {"product": "C", "quantity": 3, "price": 5.0} + ], + "total_price": 15.0 + } +] +``` + +### Notes + +- This is **not** a standard SQL window function (which would return one row per item). +- This is **not** an Elasticsearch nested aggregation (which would not return the items). +- This is a **hybrid parent-level aggregation**, unique to the SQL Gateway. + +--- + +## Window Functions + +Window functions operate over a logical window of rows defined by `OVER (PARTITION BY ... ORDER BY ...)`. + +Supported window functions include: + +- `SUM(expr) OVER (...)` +- `COUNT(expr) OVER (...)` +- `FIRST_VALUE(expr) OVER (...)` +- `LAST_VALUE(expr) OVER (...)` +- `ARRAY_AGG(expr) OVER (...)` + +#### Basic window example + +```sql +SELECT + product, + customer, + amount, + SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product +FROM dql_sales +ORDER BY product, ts; +``` + +#### FIRST_VALUE / LAST_VALUE / ARRAY_AGG + +```sql +SELECT + product, + customer, + amount, + SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product, + FIRST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS first_amount, + LAST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS last_amount, + ARRAY_AGG(amount) OVER (PARTITION BY product ORDER BY ts ASC LIMIT 10) AS amounts_array +FROM dql_sales +ORDER BY product, ts; +``` + +Notes: + +- `PARTITION BY` defines the grouping key. +- `ORDER BY` inside `OVER` defines the window ordering. +- `LIMIT` inside `ARRAY_AGG` restricts the collected values. +- Frame clauses (`ROWS BETWEEN ...`) are not exposed; the engine uses a default frame per function semantics. + +--- + +## Functions + +The SQL Gateway provides a rich set of SQL functions covering: + +- numeric and trigonometric operations +- string manipulation +- date and time extraction, arithmetic, formatting and parsing +- geospatial functions +- conditional expressions +- type conversion + +All functions operate on Elasticsearch documents and are evaluated by the SQL engine. + +--- + +#### Numeric & Trigonometric + +##### **Arithmetic:** + +| Function | Description | +|---------------|----------------------| +| `ABS(x)` | Absolute value | +| `CEIL(x)` | Round up | +| `FLOOR(x)` | Round down | +| `ROUND(x, n)` | Round to n decimals | +| `SQRT(x)` | Square root | +| `POW(x, y)` | Power | +| `EXP(x)` | Exponential | +| `LOG(x)` | Natural logarithm | +| `LOG10(x)` | Base-10 logarithm | +| `SIGN(x)` | Sign of x (βˆ’1, 0, 1) | + + +##### **Trigonometric:** + +| Function | Description | +|---------------|----------------------------------| +| `SIN(x)` | Sine | +| `COS(x)` | Cosine | +| `TAN(x)` | Tangent | +| `ASIN(x)` | Arc-sine | +| `ACOS(x)` | Arc-cosine | +| `ATAN(x)` | Arc-tangent | +| `ATAN2(y, x)` | Arc-tangent of y/x with quadrant | +| `PI()` | Ο€ constant | +| `RADIANS(x)` | Degrees β†’ radians | +| `DEGREES(x)` | Radians β†’ degrees | + +**Example** + +```sql +SELECT id, + ABS(age) AS abs_age, + SQRT(age) AS sqrt_age, + POW(age, 2) AS pow_age, + LOG(age) AS log_age, + SIN(age) AS sin_age, + ATAN2(age, 10) AS atan2_val +FROM dql_users; +``` + +--- + +#### String + +##### **Manipulation:** + + +| Function | Description | +|------------------------------|-------------------------------| +| `CONCAT(a, b, ...)` | Concatenate strings | +| `SUBSTRING(str, start, len)` | Extract substring | +| `LOWER(str)` | Lowercase | +| `UPPER(str)` | Uppercase | +| `TRIM(str)` | Trim both sides | +| `LTRIM(str)` | Trim left | +| `RTRIM(str)` | Trim right | +| `LENGTH(str)` | String length | +| `REPLACE(str, from, to)` | Replace substring | +| `LEFT(str, n)` | Left n chars | +| `RIGHT(str, n)` | Right n chars | +| `REVERSE(str)` | Reverse string | +| `POSITION(substr IN str)` | 1-based position of substring | + +##### **Pattern matching:** + +| Function | Description | +|------------------------------|-----------------------------------------------------| +| `REGEXP_LIKE(str, pattern)` | True if regex matches | +| `MATCH(str) AGAINST (query)` | Full-text match (backed by ES query_string / match) | + +**Example:** + +```sql +SELECT id, + CONCAT(name.raw, '_suffix') AS name_concat, + SUBSTRING(name.raw, 1, 2) AS name_sub, + LOWER(name.raw) AS name_lower, + LTRIM(name.raw) AS name_ltrim, + POSITION('o' IN name.raw) AS pos_o, + REGEXP_LIKE(name.raw, '.*o.*') AS has_o +FROM dql_users +ORDER BY id ASC; +``` + +--- + +#### Date & Time + +##### **Current :** + +| Function | Description | +|-------------------------------------------|-----------------------------| +| `CURRENT_DATE` | Current date (UTC) | +| `TODAY()` | Alias for CURRENT_DATE | +| `CURRENT_TIMESTAMP` \| `CURRENT_DATETIME` | Current timestamp (UTC) | +| `NOW()` | Alias for CURRENT_TIMESTAMP | +| `CURRENT_TIME` | Current time (UTC) | + +##### **Extraction:** + +| Function | Description | +|-------------------|--------------| +| `YEAR(date)` | Year | +| `MONTH(date)` | Month | +| `DAY(date)` | Day of month | +| `WEEKDAY(date)` | Day of week | +| `YEARDAY(date)` | Day of year | +| `HOUR(ts)` | Hour | +| `MINUTE(ts)` | Minute | +| `SECOND(ts)` | Second | +| `MILLISECOND(ts)` | Millisecond | +| `MICROSECOND(ts)` | Microsecond | +| `NANOSECOND(ts)` | Nanosecond | + +##### **EXTRACT:** + +```sql +EXTRACT(unit FROM date_or_timestamp) +``` + +Supported units include: `YEAR`, `MONTH`, `DAY`, `HOUR`, `MINUTE`, `SECOND`, etc. + +**Example:** + +```sql +SELECT id, + EXTRACT(YEAR FROM birthdate) AS year_b, + EXTRACT(MONTH FROM birthdate) AS month_b +FROM dql_users; +``` + +##### **Arithmetic:** + +| Function | Description | +|-------------------------------------|----------------------------------| +| `DATE_ADD(date, INTERVAL n unit)` | Add interval | +| `DATE_SUB(date, INTERVAL n unit)` | Subtract interval | +| `DATETIME_ADD(ts, INTERVAL n unit)` | Add interval to timestamp | +| `DATETIME_SUB(ts, INTERVAL n unit)` | Subtract interval from timestamp | +| `DATE_DIFF(date1, date2, unit)` | Difference in units | +| `DATE_TRUNC(date, unit)` | Truncate to unit | + +##### **Formatting & parsing:** + +| Function | Description | +|--------------------------------|-----------------------------| +| `DATE_FORMAT(ts, pattern)` | Format date as string | +| `DATE_PARSE(str, pattern)` | Parse string into date | +| `DATETIME_FORMAT(ts, pattern)` | Format timestamp as string | +| `DATETIME_PARSE(str, pattern)` | Parse string into timestamp | + +**Supported MySQL-style Date/Time Patterns** + +##### **Special functions:** + +| Function | Description | +|------------------------|-------------------------------| +| `LAST_DAY(date)` | Last day of month | +| `EPOCHDAY(date)` | Days since epoch (1970-01-01) | +| `OFFSET_SECONDS(date)` | Epoch seconds | + +**Example:** + +```sql +SELECT id, + YEAR(CURRENT_DATE) AS current_year, + MONTH(CURRENT_DATE) AS current_month, + DAY(CURRENT_DATE) AS current_day, + YEAR(birthdate) AS year_b, + DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years, + DATE_TRUNC(birthdate, MONTH) AS trunc_month, + DATETIME_FORMAT(birthdate, '%Y-%m-%d') AS birth_str +FROM dql_users; +``` + +--- + +#### Geospatial + +##### **POINT:** + +```sql +POINT(longitude, latitude) +``` + +##### **ST_DISTANCE:** + +```sql +ST_DISTANCE(location, POINT(2.3522, 48.8566)) +``` + +Example: + +```sql +CREATE TABLE IF NOT EXISTS dql_geo ( + id INT NOT NULL, + location GEO_POINT, + PRIMARY KEY (id) +); + +SELECT id, + ST_DISTANCE(location, POINT(2.3522, 48.8566)) AS dist_paris +FROM dql_geo; +``` + +--- + +#### Conditional + +##### CASE WHEN + +```sql +CASE + WHEN condition THEN value + [WHEN condition2 THEN value2] + [ELSE default] +END +``` + +##### COALESCE + +```sql +COALESCE(a, b, c) +``` + +Returns the first non-null value. + +##### NULLIF + +```sql +NULLIF(a, b) +``` + +Returns NULL if `a = b`, otherwise `a`. + +**Example:** + +```sql +SELECT id, + CASE + WHEN age >= 50 THEN 'senior' + WHEN age >= 30 THEN 'adult' + ELSE 'young' + END AS age_group, + COALESCE(name, 'unknown') AS safe_name +FROM dql_users; +``` + +--- + +#### Type Conversion + +##### CAST + +```sql +CAST(value AS TYPE) +``` + +##### TRY_CAST + +Returns NULL instead of failing on invalid conversion. + +```sql +TRY_CAST('123' AS INT) +``` + +##### SAFE_CAST + +Alias for TRY_CAST. + +##### PostgreSQL-style operator + +```sql +value::TYPE +``` + +**Example:** + +```sql +SELECT id, + age::BIGINT AS age_bigint, + CAST(age AS DOUBLE) AS age_double, + TRY_CAST('123' AS INT) AS try_cast_ok, + SAFE_CAST('abc' AS INT) AS safe_cast_null +FROM dql_users; +``` + +--- + +## Scroll & Pagination + +For large result sets, the Gateway uses Elasticsearch scroll or search-after mechanisms depending on backend capabilities ([Scroll Search](../client/scroll.md)). + +Notes: + +- `LIMIT` and `OFFSET` are applied by the SQL engine after retrieving documents from Elasticsearch +- Deep pagination may require scroll +- When using `search_after`, an explicit `ORDER BY` clause is required for deterministic pagination +- Without `ORDER BY`, result ordering is not guaranteed + +--- + +## Version Compatibility + +| Feature | ES6 | ES7 | ES8 | ES9 | +|--------------------------------|-----|-----|-----|-----| +| Basic SELECT | βœ” | βœ” | βœ” | βœ” | +| Nested fields | βœ” | βœ” | βœ” | βœ” | +| UNION ALL | βœ” | βœ” | βœ” | βœ” | +| JOIN UNNEST | βœ” | βœ” | βœ” | βœ” | +| Aggregations | βœ” | βœ” | βœ” | βœ” | +| Parent-level nested array aggs | βœ” | βœ” | βœ” | βœ” | +| Window functions | βœ” | βœ” | βœ” | βœ” | +| Geospatial functions | βœ” | βœ” | βœ” | βœ” | +| Date/time functions | βœ” | βœ” | βœ” | βœ” | +| String / math functions | βœ” | βœ” | βœ” | βœ” | + +--- + +## Limitations + +Even though the DQL engine is powerful, some SQL features are not (yet) supported: + +- No traditional SQL joins (only `JOIN UNNEST` on `ARRAY`) +- No correlated subqueries +- No arbitrary subqueries in `SELECT` or `WHERE` (except `INSERT ... AS SELECT` in DML) +- No `GROUPING SETS`, `CUBE`, `ROLLUP` +- No `DISTINCT ON` +- No explicit window frame clauses (`ROWS BETWEEN ...`) + +These constraints keep the translation to Elasticsearch efficient and predictable. + +--- + +[Back to index](README.md) diff --git a/documentation/sql/keywords.md b/documentation/sql/keywords.md index 986848de..5cb98962 100644 --- a/documentation/sql/keywords.md +++ b/documentation/sql/keywords.md @@ -6,6 +6,13 @@ A list of reserved words recognized by the parser for this engine. ## Main clauses SELECT +INSERT +UPDATE +DELETE +CREATE +ALTER +DROP +TRUNCATE FROM JOIN UNNEST diff --git a/documentation/sql/request_structure.md b/documentation/sql/request_structure.md deleted file mode 100644 index 44c01070..00000000 --- a/documentation/sql/request_structure.md +++ /dev/null @@ -1,112 +0,0 @@ -[Back to index](README.md) - -# Query Structure - -**Navigation:** [Operators](operators.md) Β· [Functions β€” Aggregate](functions_aggregate.md) Β· [Keywords](keywords.md) - -This page documents the SQL clauses supported by the engine and how they map to Elasticsearch. - ---- - -## SELECT -**Description:** -Projection of fields, expressions and computed values. - -**Behavior:** -- `_source` includes for plain fields. -- Computed expressions are translated into `script_fields` (Painless) when push-down is not otherwise possible. -- Aggregates are translated to ES aggregations and the top-level `size` is often set to `0` for aggregation-only queries. - -**Example:** -```sql -SELECT department, COUNT(*) AS cnt -FROM emp -GROUP BY department; -``` - ---- - -## FROM -**Description:** -Source index (one or more). Translates to the Elasticsearch index parameter. - -**Example:** -```sql -SELECT * FROM employees; -``` - ---- - -## UNNEST -**Description:** -Expand an array / nested field into rows. Mapped to Elasticsearch `nested` and inner hits where necessary. - -**Example:** -```sql -SELECT id, phone -FROM customers -JOIN UNNEST(customers.phones) AS phone; -``` - ---- - -## WHERE -**Description:** -Row-level predicates. Mapped to `bool` queries; complex expressions become `script` queries (Painless). - -**Example:** -```sql -SELECT * FROM emp WHERE salary > 50000 AND department = 'IT'; -``` - ---- - -## GROUP BY -**Description:** -Aggregation buckets. Mapped to `terms`/`date_histogram` and nested sub-aggregations. -Non-aggregated selected fields are disallowed unless included in the `GROUP BY` (standard SQL semantics). - -**Example:** -```sql -SELECT department, AVG(salary) AS avg_salary -FROM emp -GROUP BY department; -``` - ---- - -## HAVING -**Description:** -Filter groups using aggregate expressions. Implemented with pipeline aggregations and `bucket_selector` where possible, or client-side filtering if required. - -**Example:** -```sql -SELECT department, COUNT(*) AS cnt -FROM emp -GROUP BY department -HAVING COUNT(*) > 10; -``` - ---- - -## ORDER BY -**Description:** -Sorting of final rows or ordering used inside window/aggregations (pushed to `sort` or `top_hits`). - -**Example:** -```sql -SELECT name, salary FROM emp ORDER BY salary DESC; -``` - ---- - -## LIMIT / OFFSET -**Description:** -Limit and paging. For pure aggregations, `size` is typically set to 0 and `limit` applies to aggregations or outer rows. - -**Example:** -```sql -SELECT * FROM emp ORDER BY hire_date DESC LIMIT 10 OFFSET 20; -``` - -[Back to index](README.md) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d76b14a5..792d3e0d 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -22,7 +22,6 @@ import app.softnetwork.elastic.sql.query.{ Asc, BucketIncludesExcludes, BucketNode, - BucketTree, Criteria, Desc, Field, @@ -101,7 +100,7 @@ object ElasticAggregation { having: Option[Criteria], bucketsDirection: Map[String, SortOrder], allAggregations: Map[String, SQLAggregation] - ): ElasticAggregation = { + )(implicit timestamp: Long): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -156,14 +155,14 @@ object ElasticAggregation { if (transformFuncs.nonEmpty) { val context = PainlessContext() val scriptSrc = identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => val context = PainlessContext() val scriptSrc = th.identifier.painless(Some(context)) - val script = Script(s"$context$scriptSrc").lang("painless") + val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) case _ => buildField(aggName, sourceField) } @@ -231,7 +230,7 @@ object ElasticAggregation { .filter(_.isScriptField) .groupBy(_.sourceField) .map(_._2.head) - .map(f => f.sourceField -> Script(f.painless(None)).lang("painless")) + .map(f => f.sourceField -> now(Script(f.painless(None)).lang("painless"))) .toMap, size = limit, sorts = th.orderBy @@ -269,14 +268,14 @@ object ElasticAggregation { val painless = script.identifier.painless(None) bucketScriptAggregation( aggName, - Script(s"$painless").lang("painless"), + now(Script(s"$painless").lang("painless")), params.toMap ) case _ => throw new IllegalArgumentException(s"Unsupported aggregation type: $aggType") } - val nestedElement = identifier.nestedElement + val nestedElement = sqlAgg.nestedElement val nestedElements: Seq[NestedElement] = nestedElement.map(n => NestedElements.buildNestedTrees(Seq(n))).getOrElse(Nil) @@ -349,11 +348,7 @@ object ElasticAggregation { having: Option[Criteria], nested: Option[NestedElement], allElasticAggregations: Seq[ElasticAggregation] - ): Seq[Aggregation] = { - val trees = BucketTree(buckets.flatMap(_.headOption)) - println( - s"[DEBUG] buildBuckets called with buckets: \n$trees" - ) + )(implicit timestamp: Long): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = tree.sortBy(_.level).reverse.foldLeft(Seq.empty[NodeAggregation]) { (current, node) => @@ -371,7 +366,7 @@ object ElasticAggregation { if (!bucket.isBucketScript && bucket.shouldBeScripted) { val context = PainlessContext() val painless = bucket.painless(Some(context)) - Some(Script(s"$context$painless").lang("painless")) + Some(now(Script(s"$context$painless").lang("painless"))) } else { None } @@ -520,7 +515,7 @@ object ElasticAggregation { val bucketSelector = bucketSelectorAggregation( "having_filter", - Script(script), + now(Script(script)), extractMetricsPathForBucket( criteria, nested, diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 187bf278..ad5ad8ad 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -45,7 +45,7 @@ case class ElasticBridge(filter: ElasticFilter) { def query( innerHitsNames: Set[String] = Set.empty, currentQuery: Option[ElasticBoolQuery] - ): Query = { + )(implicit timestamp: Long): Query = { filter match { case boolQuery: ElasticBoolQuery => import boolQuery._ @@ -72,13 +72,10 @@ case class ElasticBridge(filter: ElasticFilter) { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.map { source => - (n.path.split('.').toSeq ++ Seq(source)).mkString(".") - }.toArray - ) + inner = inner.docValueFields( + n.sources.map { source => + (n.path.split('.').toSeq ++ Seq(source)).mkString(".") + } ) } inner diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index caa12dd0..f745889a 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -21,7 +21,9 @@ import com.sksamuel.elastic4s.searches.queries.Query case class ElasticCriteria(criteria: Criteria) { - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit + timestamp: Long + ): Query = { val query = criteria.boolQuery.copy(group = group) query .filter(criteria.asFilter(Option(query))) diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index 7f7843b4..99e37c2c 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -47,10 +47,17 @@ import scala.language.implicitConversions package object bridge { + def now(script: Script)(implicit timestamp: Long): Script = { + if (!script.script.contains("params.__now__")) { + return script + } + script.param("__now__", timestamp) + } + implicit def requestToNestedFilterAggregation( - request: SQLSearchRequest, + request: SingleSearch, innerHitsName: String - ): Option[FilterAggregation] = { + )(implicit timestamp: Long): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -125,8 +132,8 @@ package object bridge { } implicit def requestToFilterAggregation( - request: SQLSearchRequest - ): Option[FilterAggregation] = + request: SingleSearch + )(implicit timestamp: Long): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => val boolQuery = Option(ElasticBoolQuery(group = true)) @@ -142,9 +149,9 @@ package object bridge { } implicit def requestToRootAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[AbstractAggregation] = { + )(implicit timestamp: Long): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) val notNestedBuckets = request.bucketTree.filterNot(_.bucket.nested) @@ -192,9 +199,9 @@ package object bridge { } implicit def requestToScopedAggregations( - request: SQLSearchRequest, + request: SingleSearch, aggregations: Seq[ElasticAggregation] - ): Seq[NestedAggregation] = { + )(implicit timestamp: Long): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations .filter(_.nested) @@ -324,7 +331,7 @@ package object bridge { scopedAggregations } - implicit def requestToNestedWithoutCriteriaQuery(request: SQLSearchRequest): Option[Query] = + implicit def requestToNestedWithoutCriteriaQuery(request: SingleSearch): Option[Query] = NestedElements.buildNestedTrees(request.nestedElementsWithoutCriteria) match { case Nil => None case nestedTrees => @@ -336,12 +343,7 @@ package object bridge { case _ => } if (n.sources.nonEmpty) { - inner = inner.fetchSource( - FetchSourceContext( - fetchSource = true, - includes = n.sources.toArray - ) - ) + inner = inner.docValueFields(n.sources) } inner } @@ -404,7 +406,9 @@ package object bridge { } } - implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit + timestamp: Long + ): ElasticSearchRequest = ElasticSearchRequest( request.sql, request.select.fields, @@ -419,7 +423,9 @@ package object bridge { request.orderBy.map(_.sorts).getOrElse(Seq.empty) ).minScore(request.score) - implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + implicit def requestToSearchRequest( + request: SingleSearch + )(implicit timestamp: Long): SearchRequest = { import request._ val aggregations = request.aggregates.map( @@ -461,7 +467,11 @@ package object bridge { case _ => nestedWithoutCriteriaQuery.getOrElse(matchAllQuery()) } - } sourceFiltering (fields, excludes) + } + + if (!request.deleteByQuery && !request.updateByQuery) { + _search = _search sourceFiltering (fields, excludes) + } _search = if (allAggregations.nonEmpty) { _search aggregations { @@ -479,13 +489,15 @@ package object bridge { val script = field.painless(Some(context)) scriptField( field.scriptName, - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") - .params(field.identifier.functions.headOption match { - case Some(f: PainlessParams) => f.params - case _ => Map.empty[String, Any] - }) + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + .params((field.identifier.functions.headOption match { + case Some(f: PainlessParams) => f.params + case _ => Map.empty[String, Any] + })) + ) ) } } @@ -517,9 +529,11 @@ package object bridge { } val scriptSort = ScriptSort( - script = Script(script = script) - .lang("painless") - .scriptType(Source), + script = now( + Script(script = script) + .lang("painless") + .scriptType(Source) + ), scriptSortType = sort.field.out match { case _: SQLTemporal | _: SQLNumeric => ScriptSortType.Number case _ => ScriptSortType.String @@ -550,8 +564,8 @@ package object bridge { } implicit def requestToMultiSearchRequest( - request: SQLMultiSearchRequest - ): MultiSearchRequest = { + request: MultiSearch + )(implicit timestamp: Long): MultiSearchRequest = { MultiSearchRequest( request.requests.map(implicitly[SearchRequest](_)) ) @@ -562,7 +576,7 @@ package object bridge { doubleOp: Double => A ): A = n.toEither.fold(longOp, doubleOp) - implicit def expressionToQuery(expression: GenericExpression): Query = { + implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long): Query = { import expression._ if (isAggregation) return matchAllQuery() @@ -575,7 +589,11 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) return scriptQuery( - Script(script = s"$context$script").lang("painless").scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } // Geo distance special case @@ -793,18 +811,22 @@ package object bridge { val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => val context = PainlessContext() val script = painless(Some(context)) scriptQuery( - Script(script = s"$context$script") - .lang("painless") - .scriptType("source") + now( + Script(script = s"$context$script") + .lang("painless") + .scriptType("source") + ) ) } case _ => matchAllQuery() @@ -860,7 +882,7 @@ package object bridge { implicit def betweenToQuery( between: BetweenExpr - ): Query = { + )(implicit timestamp: Long): Query = { import between._ // Geo distance special case identifier.functions.headOption match { @@ -991,12 +1013,12 @@ package object bridge { @deprecated implicit def sqlQueryToAggregations( - query: SQLQuery - ): Seq[ElasticAggregation] = { + query: SelectStatement + )(implicit timestamp: Long): Seq[ElasticAggregation] = { import query._ - request + statement .map { - case Left(l) => + case l: SingleSearch => val filteredAgg: Option[FilterAggregation] = requestToFilterAggregation(l) l.aggregates .map(ElasticAggregation(_, l.having.flatMap(_.criteria), l.sorts, l.sqlAggregations)) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index 88221237..a612cb81 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -8,16 +8,20 @@ import com.sksamuel.elastic4s.searches.SearchRequest import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLCriteriaSpec extends AnyFlatSpec with Matchers { - import Queries._ + import parser.Queries._ import scala.language.implicitConversions def asQuery(sql: String): String = { import SQLImplicits._ + implicit def timestamp: Long = + ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli val criteria: Option[Criteria] = sql val result = SearchBodyBuilderFn( SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index be4c7494..4265603d 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1,20 +1,24 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.Queries._ -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.parser.Queries._ +import app.softnetwork.elastic.sql.query.{SelectStatement, SingleSearch} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers +import java.time.ZonedDateTime + /** Created by smanciot on 13/04/17. */ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions - implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { - sqlQuery.request match { - case Some(Left(value)) => + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = { + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) case None => throw new IllegalArgumentException( @@ -23,9 +27,9 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } } - "SQLQuery" should "perform native count" in { + "SelectStatement" should "perform native count" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(t.id) c2 from Table t where t.nom = 'Nom'") + SelectStatement("select count(t.id) c2 from Table t where t.nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -61,7 +65,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform count distinct" in { val results: Seq[ElasticAggregation] = - SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") + SelectStatement("select count(distinct t.id) as c2 from Table as t where nom = 'Nom'") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -97,7 +101,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as email from index i join unnest(i.emails) as inner_emails where i.nom = 'Nom'" ) results.size shouldBe 1 @@ -142,7 +146,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with nested criteria" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" ) results.size shouldBe 1 @@ -205,7 +209,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with filter" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\")) having inner_emails.context = \"profile\"" ) results.size shouldBe 1 @@ -279,7 +283,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with \"and not\" operator" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" ) results.size shouldBe 1 @@ -353,7 +357,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested count with date filtering" in { val results: Seq[ElasticAggregation] = - SQLQuery( + SelectStatement( "select count(distinct inner_emails.value) as count_distinct_emails from index join unnest(index.emails) as inner_emails join unnest(index.profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" ) results.size shouldBe 1 @@ -421,7 +425,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform nested select" in { val select: ElasticSearchRequest = - SQLQuery(""" + SelectStatement(""" |SELECT |profileId, |profile_ccm.email as email, @@ -470,17 +474,15 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "profile_ccm", | "from": 0, - | "_source": { - | "includes": [ - | "profiles.email", - | "profiles.city", - | "profiles.firstName", - | "profiles.lastName", - | "profiles.postalCode", - | "profiles.birthYear" - | ] - | }, - | "size": 100 + | "size": 100, + | "docvalue_fields": [ + | "profiles.email", + | "profiles.city", + | "profiles.firstName", + | "profiles.lastName", + | "profiles.postalCode", + | "profiles.birthYear" + | ] | } | } | } @@ -499,7 +501,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "exclude fields from select" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( except ) select.query shouldBe @@ -517,7 +519,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform query with group by and having" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHaving) + SelectStatement(groupByWithHaving) val query = select.query println(query) query shouldBe @@ -577,7 +579,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "perform complex query" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( s"""SELECT | inner_products.category as cat, | min(inner_products.price) as min_price, @@ -839,7 +841,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "add script fields" in { val select: ElasticSearchRequest = - SQLQuery(fieldsWithInterval) + SelectStatement(fieldsWithInterval) val query = select.query println(query) query shouldBe @@ -851,7 +853,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "ct": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); param1" | } | } | }, @@ -879,7 +881,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateTimeAndInterval) + SelectStatement(filterWithDateTimeAndInterval) val query = select.query println(query) query shouldBe @@ -914,7 +916,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with date and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithDateAndInterval) + SelectStatement(filterWithDateAndInterval) val query = select.query println(query) query shouldBe @@ -949,7 +951,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "filter with time and interval" in { val select: ElasticSearchRequest = - SQLQuery(filterWithTimeAndInterval) + SelectStatement(filterWithTimeAndInterval) val query = select.query println(query) query shouldBe @@ -997,7 +999,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle having with date functions" in { val select: ElasticSearchRequest = - SQLQuery("""SELECT userId, MAX(createdAt) as lastSeen + SelectStatement("""SELECT userId, MAX(createdAt) as lastSeen |FROM table |GROUP BY userId |HAVING MAX(createdAt) > now - interval 7 day""".stripMargin) @@ -1028,7 +1030,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1041,11 +1046,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by with having and date time functions" in { val select: ElasticSearchRequest = - SQLQuery(groupByWithHavingAndDateTimeFunctions) + SelectStatement(groupByWithHavingAndDateTimeFunctions) val query = select.query println(query) query shouldBe @@ -1090,7 +1096,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": "lastSeen" | }, | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -1106,78 +1115,83 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle group by index" in { val select: ElasticSearchRequest = - SQLQuery( + SelectStatement( groupByWithHavingAndDateTimeFunctions.replace("GROUP BY Country, City", "GROUP BY 3, 2") ) val query = select.query println(query) query shouldBe """{ - | "query": { - | "match_all": {} - | }, - | "size": 0, - | "_source": false, - | "aggs": { - | "Country": { - | "terms": { - | "field": "Country", - | "exclude": "USA", - | "min_doc_count": 1, - | "order": { - | "_key": "asc" - | } - | }, - | "aggs": { - | "City": { - | "terms": { - | "field": "City", - | "exclude": "Berlin", - | "min_doc_count": 1 - | }, - | "aggs": { - | "cnt": { - | "value_count": { - | "field": "CustomerID" - | } - | }, - | "lastSeen": { - | "max": { - | "field": "createdAt" - | } - | }, - | "having_filter": { - | "bucket_selector": { - | "buckets_path": { - | "cnt": "cnt", - | "lastSeen": "lastSeen" - | }, - | "script": { - | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()" - | } - | } - | } - | } - | } - | } - | } - | } - |}""".stripMargin + | "query": { + | "match_all": {} + | }, + | "size": 0, + | "_source": false, + | "aggs": { + | "Country": { + | "terms": { + | "field": "Country", + | "exclude": "USA", + | "min_doc_count": 1, + | "order": { + | "_key": "asc" + | } + | }, + | "aggs": { + | "City": { + | "terms": { + | "field": "City", + | "exclude": "Berlin", + | "min_doc_count": 1 + | }, + | "aggs": { + | "cnt": { + | "value_count": { + | "field": "CustomerID" + | } + | }, + | "lastSeen": { + | "max": { + | "field": "createdAt" + | } + | }, + | "having_filter": { + | "bucket_selector": { + | "buckets_path": { + | "cnt": "cnt", + | "lastSeen": "lastSeen" + | }, + | "script": { + | "source": "params.cnt > 1 && params.lastSeen > ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS).toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } + | } + | } + | } + | } + | } + | } + | } + | } + |}""".stripMargin .replaceAll("\\s", "") .replaceAll("ChronoUnit", " ChronoUnit") .replaceAll("==", " == ") .replaceAll("!=", " != ") .replaceAll("&&", " && ") .replaceAll(">", " > ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle date_parse function" in { val select: ElasticSearchRequest = - SQLQuery(dateParse) + SelectStatement(dateParse) val query = select.query println(query) query shouldBe @@ -1246,7 +1260,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateFormat) + SelectStatement(dateFormat) val query = select.query println(query) query shouldBe @@ -1266,49 +1280,49 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfYear(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z'))); def param2 = param1 != null ? param1.withMonth((((param1.getMonthValue() - 1) / 3) * 3) + 1).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS) : null; def param3 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param3.format(param2)" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).with(DayOfWeek.SUNDAY).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.HOURS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "m2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.MINUTES)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | }, | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).truncatedTo(ChronoUnit.SECONDS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1345,7 +1359,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_parse function" in { // #25 val select: ElasticSearchRequest = - SQLQuery(dateTimeParse) + SelectStatement(dateTimeParse) val query = select.query println(query) query shouldBe @@ -1415,7 +1429,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_format function" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeFormat) + SelectStatement(dateTimeFormat) val query = select.query println(query) query shouldBe @@ -1435,7 +1449,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "lastSeen": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toInstant().atZone(ZoneId.of('Z')).withDayOfMonth(1).truncatedTo(ChronoUnit.DAYS)); def param2 = DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss XXX\"); (param1 == null) ? null : param2.format(param1)" | } | } | }, @@ -1469,7 +1483,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_diff function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateDiff) + SelectStatement(dateDiff) val query = select.query println(query) query shouldBe @@ -1481,7 +1495,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "diff": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" | } | } | }, @@ -1511,7 +1525,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with date_diff function" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithDateDiff) + SelectStatement(aggregationWithDateDiff) val query = select.query println(query) query shouldBe @@ -1532,7 +1546,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "max": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value); def param2 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param3 = (param2 == null) ? null : ZonedDateTime.parse(param2, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")); (param1 == null || param2 == null) ? null : ChronoUnit.DAYS.between(param1, param3)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = (doc['updatedAt'].size() == 0 ? null : doc['updatedAt'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate()); def param3 = ((param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")) != null ? (param1 == null) ? null : ZonedDateTime.parse(param1, DateTimeFormatter.ofPattern(\"yyyy-MM-dd HH:mm:ss.SSS XXX\")).toLocalDate() : null); (param1 == null || param2 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param3, param2))" | } | } | } @@ -1564,7 +1578,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateAdd) + SelectStatement(dateAdd) val query = select.query println(query) query shouldBe @@ -1615,7 +1629,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle date_sub function as script field" in { // 30 val select: ElasticSearchRequest = - SQLQuery(dateSub) + SelectStatement(dateSub) val query = select.query println(query) query shouldBe @@ -1666,7 +1680,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_add function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeAdd) + SelectStatement(dateTimeAdd) val query = select.query println(query) query shouldBe @@ -1717,7 +1731,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle datetime_sub function as script field" in { val select: ElasticSearchRequest = - SQLQuery(dateTimeSub) + SelectStatement(dateTimeSub) val query = select.query println(query) query shouldBe @@ -1768,7 +1782,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnull) + SelectStatement(isnull) val query = select.query println(query) query shouldBe @@ -1806,7 +1820,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull function as script field" in { val select: ElasticSearchRequest = - SQLQuery(isnotnull) + SelectStatement(isnotnull) val query = select.query println(query) query shouldBe @@ -1848,7 +1862,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_null criteria as must_not exists" in { val select: ElasticSearchRequest = - SQLQuery(isNullCriteria) + SelectStatement(isNullCriteria) val query = select.query println(query) query shouldBe @@ -1880,7 +1894,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle is_notnull criteria as exists" in { val select: ElasticSearchRequest = - SQLQuery(isNotNullCriteria) + SelectStatement(isNotNullCriteria) val query = select.query println(query) query shouldBe @@ -1906,7 +1920,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle coalesce function as script field" in { val select: ElasticSearchRequest = - SQLQuery(coalesce) + SelectStatement(coalesce) val query = select.query println(query) query shouldBe @@ -1918,7 +1932,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).minus(35, ChronoUnit.MINUTES)); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param1 != null ? param1 : param2)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -1953,11 +1970,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle nullif function as script field" in { val select: ElasticSearchRequest = - SQLQuery(nullif) + SelectStatement(nullif) val query = select.query println(query) query shouldBe @@ -1969,7 +1987,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).minus(2, ChronoUnit.DAYS); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); (param3 != null ? param3 : param4)", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2012,11 +2033,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll(":ZonedDateTime", " : ZonedDateTime") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle cast function as script field" in { val select: ElasticSearchRequest = - SQLQuery(conversion) + SelectStatement(conversion) val query = select.query println(query) query shouldBe @@ -2028,19 +2050,28 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); def param2 = LocalDate.parse(\"2025-09-11\", DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")); def param3 = param1 == null || param1.isEqual(param2) ? null : param1; def param4 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(2, ChronoUnit.HOURS); try { (param3 != null ? param3 : param4) } catch (Exception e) { return null; }", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c2": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toInstant().toEpochMilli()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toInstant().toEpochMilli()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c3": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate()" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate()", + | "params": { + | "__now__": 1767139200000 + | } | } | }, | "c4": { @@ -2097,11 +2128,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("try \\{", "try { ") .replaceAll("} catch", " } catch") .replaceAll(";\\(param", "; (param") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case function as script field" in { // 40 val select: ElasticSearchRequest = - SQLQuery(caseWhen) + SelectStatement(caseWhen) val query = select.query println(query) query shouldBe @@ -2113,7 +2145,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).minus(7, ChronoUnit.DAYS); def param3 = param1 == null ? false : (param1.isAfter(param2)); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.plus(2, ChronoUnit.DAYS)); def param5 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param3 ? param1 : param4 != null ? param4 : param5", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2150,11 +2185,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle case with expression function as script field" in { val select: ElasticSearchRequest = - SQLQuery(caseWhenExpr) + SelectStatement(caseWhenExpr) val query = select.query println(query) query shouldBe @@ -2166,7 +2202,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate().plus(2, ChronoUnit.DAYS)); def param4 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param1.isEqual(param2) ? param2 : param1 != null && param1.isEqual(param3) ? param3 : param4", + | "params": { + | "__now__": 1767139200000 + | } | } | } | }, @@ -2205,11 +2244,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("=ZonedDateTime", " = ZonedDateTime") .replaceAll("=p", " = p") .replaceAll(":p", " : p") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle extract function as script field" in { val select: ElasticSearchRequest = - SQLQuery(extract) + SelectStatement(extract) val query = select.query println(query) query shouldBe @@ -2221,91 +2261,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "dom": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "dow": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_WEEK)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_WEEK)); param1" | } | }, | "doy": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -2333,7 +2373,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle arithmetic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(arithmetic.replace("as group1", "")) + SelectStatement(arithmetic.replace("as group1", "")) val query = select.query println(query) query shouldBe @@ -2345,7 +2385,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().get(ChronoField.YEAR); (param1 == null) ? null : (param1 * (param2 - 10)) > 10000", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2418,11 +2461,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("-", " - ") .replaceAll("==", " == ") .replaceAll("\\|\\|", " || ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle mathematic function as script field and condition" in { val select: ElasticSearchRequest = - SQLQuery(mathematical) + SelectStatement(mathematical) val query = select.query println(query) query shouldBe @@ -2434,7 +2478,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1) > 100.0" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1)) > 100.0" | } | } | } @@ -2445,61 +2489,61 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "abs_identifier_plus_1_0_mul_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Math.abs(param1) + 1.0) * ((double) 2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); ((param1 == null) ? null : Double.valueOf(Math.abs(param1)) + 1.0) * ((double) 2)" | } | }, | "ceil_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.ceil(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.ceil(param1))" | } | }, | "floor_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.floor(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.floor(param1))" | } | }, | "sqrt_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sqrt(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sqrt(param1))" | } | }, | "exp_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.exp(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.exp(param1))" | } | }, | "log_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log(param1))" | } | }, | "log10_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.log10(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.log10(param1))" | } | }, | "pow_identifier_3": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.pow(param1, 3)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.pow(param1, 3))" | } | }, | "round_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 0); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "round_identifier_2": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Math.round((param1 * param2) / param2)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); def param2 = Math.pow(10, 2); (param1 == null || param2 == null) ? null : Long.valueOf(Math.round((param1 * param2) / param2))" | } | }, | "sign_identifier": { @@ -2511,43 +2555,43 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "cos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.cos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.cos(param1))" | } | }, | "acos_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.acos(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.acos(param1))" | } | }, | "sin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.sin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.sin(param1))" | } | }, | "asin_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.asin(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.asin(param1))" | } | }, | "tan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.tan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.tan(param1))" | } | }, | "atan_identifier": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan(param1)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan(param1))" | } | }, | "atan2_identifier_3_0": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Math.atan2(param1, 3.0)" + | "source": "def param1 = (doc['identifier'].size() == 0 ? null : doc['identifier'].value); (param1 == null) ? null : Double.valueOf(Math.atan2(param1, 3.0))" | } | } | }, @@ -2592,7 +2636,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle string function as script field and condition" in { // 45 val select: ElasticSearchRequest = - SQLQuery(string) + SelectStatement(string) val query = select.query println(query) query shouldBe @@ -2633,7 +2677,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "sub": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(3, ((String)param1).length()))" | } | }, | "tr": { @@ -2663,13 +2707,13 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "l": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(0, Math.min(5, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(0, (int)Math.min(5, ((String)param1).length()))" | } | }, | "r": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : param1.substring(param1.length() - Math.min(3, param1.length()))" + | "source": "def param1 = (doc['identifier2'].size() == 0 ? null : doc['identifier2'].value); (param1 == null) ? null : ((String)param1).substring(((String)param1).length() - (int)Math.min(3, ((String)param1).length()))" | } | }, | "rep": { @@ -2711,7 +2755,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("def_", "def _") .replaceAll("=_", " = _") .replaceAll(",_", ", _") - .replaceAll(",\\(", ", (") .replaceAll("if\\(", "if (") .replaceAll("=\\(", " = (") .replaceAll(":\\(", " : (") @@ -2748,7 +2791,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle top hits aggregation" in { val select: ElasticSearchRequest = - SQLQuery(topHits) + SelectStatement(topHits) val query = select.query println(query) query shouldBe @@ -2865,7 +2908,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle last day function" in { val select: ElasticSearchRequest = - SQLQuery(lastDay) + SelectStatement(lastDay) val query = select.query println(query) query shouldBe @@ -2877,7 +2920,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.now(ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28" + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')); param1.toLocalDate().withDayOfMonth(param1.toLocalDate().lengthOfMonth()).get(ChronoField.DAY_OF_MONTH) > 28", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -2928,11 +2974,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("&&", " && ") .replaceAll("\\|\\|", " || ") .replaceAll("(\\d)=", "$1 = ") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "handle all extractors" in { val select: ElasticSearchRequest = - SQLQuery(extractors) + SelectStatement(extractors) val query = select.query println(query) query shouldBe @@ -2944,91 +2991,91 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "y": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.YEAR)); param1" | } | }, | "m": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MONTH_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MONTH_OF_YEAR)); param1" | } | }, | "wd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z'))); (param1 == null) ? null : (param1.get(ChronoField.DAY_OF_WEEK) + 6) % 7" | } | }, | "yd": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_YEAR)); param1" | } | }, | "d": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.DAY_OF_MONTH)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.DAY_OF_MONTH)); param1" | } | }, | "h": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.HOUR_OF_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.HOUR_OF_DAY)); param1" | } | }, | "minutes": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MINUTE_OF_HOUR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MINUTE_OF_HOUR)); param1" | } | }, | "s": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.SECOND_OF_MINUTE)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.SECOND_OF_MINUTE)); param1" | } | }, | "nano": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.NANO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.NANO_OF_SECOND)); param1" | } | }, | "micro": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MICRO_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MICRO_OF_SECOND)); param1" | } | }, | "milli": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.MILLI_OF_SECOND)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.MILLI_OF_SECOND)); param1" | } | }, | "epoch": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.EPOCH_DAY)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.EPOCH_DAY)); param1" | } | }, | "off": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(ChronoField.OFFSET_SECONDS)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(ChronoField.OFFSET_SECONDS)); param1" | } | }, | "w": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR)); param1" | } | }, | "q": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" + | "source": "def param1 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toInstant().atZone(ZoneId.of('Z')).get(java.time.temporal.IsoFields.QUARTER_OF_YEAR)); param1" | } | } | }, @@ -3069,7 +3116,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle geo distance as script fields and criteria" in { val select: ElasticSearchRequest = - SQLQuery(geoDistance) + SelectStatement(geoDistance) val query = select.query println(query) query shouldBe @@ -3084,7 +3131,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)) >= 4000000.0", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon) >= 4000000.0", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3108,7 +3155,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon)) < 2000000.0" + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); def arg1 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null || arg1 == null) ? null : arg0.arcDistance(arg1.lat, arg1.lon) < 2000000.0" | } | } | }, @@ -3127,7 +3174,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d1": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['toLocation'].size() == 0 ? null : doc['toLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3137,7 +3184,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "d2": { | "script": { | "lang": "painless", - | "source": "(def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon))", + | "source": "def arg0 = (doc['fromLocation'].size() == 0 ? null : doc['fromLocation']); (arg0 == null) ? null : arg0.arcDistance(params.lat, params.lon)", | "params": { | "lat": -70.0, | "lon": 40.0 @@ -3191,7 +3238,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle between with temporal" in { // 50 val select: ElasticSearchRequest = - SQLQuery(betweenTemporal) + SelectStatement(betweenTemporal) val query = select.query println(query) query shouldBe @@ -3276,7 +3323,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested of nested" in { val select: ElasticSearchRequest = - SQLQuery(nestedOfNested) + SelectStatement(nestedOfNested) val query = select.query println(query) query shouldBe @@ -3314,26 +3361,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "comments.replies.reply_author", - | "comments.replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.replies.reply_author", + | "comments.replies.reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3384,7 +3427,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle predicate with distinct nested" in { val select: ElasticSearchRequest = - SQLQuery(predicateWithDistinctNested) + SelectStatement(predicateWithDistinctNested) val query = select.query println(query) query shouldBe @@ -3409,13 +3452,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "replies.reply_author", - | "replies.reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "replies.reply_author", + | "replies.reply_text" + | ] | } | } | } @@ -3434,13 +3475,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "comments.author", - | "comments.comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "comments.author", + | "comments.comments" + | ] | } | } | } @@ -3494,7 +3533,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle nested without criteria" in { val select: ElasticSearchRequest = - SQLQuery(nestedWithoutCriteria) + SelectStatement(nestedWithoutCriteria) val query = select.query println(query) query shouldBe @@ -3509,7 +3548,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "script": { | "script": { | "lang": "painless", - | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))" + | "source": "def param1 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate(); param1 == null ? false : (param1.isBefore(param2))", + | "params": { + | "__now__": 1767139200000 + | } | } | } | } @@ -3528,26 +3570,22 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "inner_hits": { | "name": "matched_replies", | "from": 0, - | "_source": { - | "includes": [ - | "reply_author", - | "reply_text" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "reply_author", + | "reply_text" + | ] | } | } | }, | "inner_hits": { | "name": "matched_comments", | "from": 0, - | "_source": { - | "includes": [ - | "author", - | "comments" - | ] - | }, - | "size": 5 + | "size": 5, + | "docvalue_fields": [ + | "author", + | "comments" + | ] | } | } | } @@ -3559,8 +3597,6 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "_source": true |}""".stripMargin .replaceAll("\\s+", "") - .replaceAll("\\s+", "") - .replaceAll("\\s+", "") .replaceAll("defp", "def p") .replaceAll("defa", "def a") .replaceAll("defe", "def e") @@ -3594,11 +3630,12 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { .replaceAll("lat,arg", "lat, arg") .replaceAll("false:", "false : ") .replaceAll("DateTimeFormatter", " DateTimeFormatter") + .replaceAll(",ZoneId.of", ", ZoneId.of") } it should "determine the aggregation context" in { val select: ElasticSearchRequest = - SQLQuery(determinationOfTheAggregationContext) + SelectStatement(determinationOfTheAggregationContext) val query = select.query println(query) query shouldBe @@ -3632,7 +3669,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle aggregation with nested of nested context" in { val select: ElasticSearchRequest = - SQLQuery(aggregationWithNestedOfNestedContext) + SelectStatement(aggregationWithNestedOfNestedContext) val query = select.query println(query) query shouldBe @@ -3668,7 +3705,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { it should "handle where filters according to scope" in { val select: ElasticSearchRequest = - SQLQuery(whereFiltersAccordingToScope) + SelectStatement(whereFiltersAccordingToScope) val query = select.query println(query) query shouldBe @@ -3753,7 +3790,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { "\n", " " ) - val select: ElasticSearchRequest = SQLQuery(query) + val select: ElasticSearchRequest = SelectStatement(query) println(select.query)*/ val query = """ SELECT @@ -3778,8 +3815,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | GROUP BY product_id, product_name, DATE_TRUNC( sale_date, MONTH ) | ORDER BY product_id, DATE_TRUNC( sale_date, MONTH ) |""".stripMargin - SQLQuery(query).request.flatMap(_.left.toOption) match { - case Some(request) => + SelectStatement(query).statement match { + case Some(request: SingleSearch) => val aggRequest = request .copy( diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala index 3e773733..7d84a327 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestAliasApi.scala @@ -16,8 +16,9 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{AliasApi, IndicesApi} +import app.softnetwork.elastic.client.AliasApi import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.schema.TableAlias import io.searchbox.client.JestResult import io.searchbox.indices.aliases.{AddAliasMapping, GetAliases, ModifyAliases, RemoveAliasMapping} @@ -28,20 +29,26 @@ import scala.jdk.CollectionConverters._ * [[AliasApi]] for generic API documentation */ trait JestAliasApi extends AliasApi with JestClientHelpers { - _: IndicesApi with JestClientCompanion => + _: JestIndicesApi with JestClientCompanion => /** Add an alias to an index. * @see * [[AliasApi.addAlias]] */ - private[client] def executeAddAlias(index: String, alias: String): ElasticResult[Boolean] = + private[client] def executeAddAlias(alias: TableAlias): ElasticResult[Boolean] = executeJestBooleanAction( operation = "addAlias", - index = Some(s"$index -> $alias"), + index = Some(s"${alias.table} -> ${alias.alias}"), retryable = false // Aliases operations can not be retried ) { + val builder = new AddAliasMapping.Builder(alias.table, alias.alias) + if (alias.filter.nonEmpty) + builder.setFilter(alias.filter.asJava.asInstanceOf[java.util.Map[String, AnyRef]]) + alias.routing.foreach(builder.addRouting) + alias.indexRouting.foreach(builder.addIndexRouting) + alias.searchRouting.foreach(builder.addSearchRouting) new ModifyAliases.Builder( - new AddAliasMapping.Builder(index, alias).build() + builder.build() ).build() } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala index 9399ecec..9a9c4fec 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestBulkApi.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.client.jest import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Flow -import app.softnetwork.elastic.client.{BulkApi, IndexApi, RefreshApi, SettingsApi} +import app.softnetwork.elastic.client.BulkApi import app.softnetwork.elastic.client.bulk.{ BulkAction, BulkElasticAction, @@ -40,7 +40,7 @@ import scala.language.implicitConversions import scala.util.{Failure, Success, Try} trait JestBulkApi extends BulkApi { - _: RefreshApi with SettingsApi with IndexApi with JestClientCompanion => + _: JestRefreshApi with JestSettingsApi with JestIndexApi with JestClientCompanion => // ======================================================================== // TYPE ALIASES FOR JEST diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala index f6cd2e76..9a31eaa1 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -17,7 +17,6 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client._ -import app.softnetwork.elastic.sql.query.SQLQuery import app.softnetwork.elastic.sql.bridge._ import io.searchbox.action.BulkableAction import io.searchbox.core._ @@ -45,6 +44,8 @@ trait JestClientApi with JestScrollApi with JestBulkApi with JestVersionApi + with JestPipelineApi + with JestTemplateApi with JestClientCompanion object JestClientApi extends SerializationApi { @@ -56,19 +57,8 @@ object JestClientApi extends SerializationApi { search.build() } - implicit class SearchSQLQuery(sqlQuery: SQLQuery) { - def jestSearch: Option[Search] = { - sqlQuery.request match { - case Some(Left(value)) => - val request: ElasticSearchRequest = value - Some(request) - case _ => None - } - } - } - implicit class SearchElasticQuery(elasticQuery: ElasticQuery) { - def search: (Search, JSONQuery) = { + def search: (Search, String) = { import elasticQuery._ val _search = new Search.Builder(query) for (indice <- indices) _search.addIndex(indice) diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala index 3b85ae2f..f5a2c666 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientHelpers.scala @@ -18,7 +18,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.ElasticClientHelpers import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} -import io.searchbox.action.Action +import io.searchbox.action.{AbstractAction, Action} import io.searchbox.client.JestResult import scala.concurrent.Promise @@ -71,7 +71,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( transformer: R => T ): ElasticResult[T] = { @@ -166,7 +166,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] ): ElasticResult[Boolean] = { executeJestAction[R, Boolean](operation, index, retryable)(action)(_.isSucceeded) } @@ -208,7 +208,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( extractor: com.google.gson.JsonObject => T ): ElasticResult[T] = { @@ -259,7 +259,7 @@ trait JestClientHelpers extends ElasticClientHelpers { _: JestClientCompanion => index: Option[String] = None, retryable: Boolean = true )( - action: => Action[R] + action: => AbstractAction[R] )( parser: String => T ): ElasticResult[T] = { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala index 7dd90f95..c5bdcc99 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestDeleteApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{DeleteApi, SettingsApi} +import app.softnetwork.elastic.client.DeleteApi import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Delete @@ -27,7 +27,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[DeleteApi]] for generic API documentation */ trait JestDeleteApi extends DeleteApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion => + _: JestSettingsApi with JestClientCompanion => /** Delete an entity from the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala index 60404a0c..f914c461 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndexApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndexApi, SerializationApi, SettingsApi} +import app.softnetwork.elastic.client.{IndexApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Index @@ -27,7 +27,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[IndexApi]] for generic API documentation */ trait JestIndexApi extends IndexApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion with SerializationApi => + _: JestSettingsApi with JestClientCompanion with SerializationApi => /** Index a document in the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala index 13f93df5..65433051 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestIndicesApi.scala @@ -17,8 +17,12 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.IndicesApi +import app.softnetwork.elastic.client.jest.actions.{GetIndex, WaitForShards} import app.softnetwork.elastic.client.result.ElasticResult +import app.softnetwork.elastic.sql.schema.{mapper, TableAlias} +import com.fasterxml.jackson.databind.node.ObjectNode import io.searchbox.client.JestResult +import io.searchbox.core.{Cat, CatResult, DeleteByQuery, UpdateByQuery, UpdateByQueryResult} import io.searchbox.indices.{CloseIndex, CreateIndex, DeleteIndex, IndicesExists, OpenIndex} import io.searchbox.indices.reindex.Reindex @@ -28,8 +32,14 @@ import scala.util.Try * @see * [[IndicesApi]] for generic API documentation */ -trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpers { - _: JestClientCompanion => +trait JestIndicesApi extends IndicesApi with JestClientHelpers { + _: JestRefreshApi + with JestPipelineApi + with JestScrollApi + with JestBulkApi + with JestVersionApi + with JestTemplateApi + with JestClientCompanion => /** Create an index with the given settings. * @see @@ -37,20 +47,49 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe */ private[client] def executeCreateIndex( index: String, - settings: String = defaultSettings + settings: String = defaultSettings, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = { executeJestBooleanAction( operation = "createIndex", index = Some(index), retryable = false // Creation can not be retried ) { - new CreateIndex.Builder(index) + val builder = new CreateIndex.Builder(index) .settings(settings) - .build() + if (aliases.nonEmpty) { + val as = mapper.createObjectNode() + aliases.foreach { alias => + as.set[ObjectNode](alias.alias, alias.node) + } + builder.aliases(as.toString) + } + mappings.foreach { mapping => + builder.mappings(mapping) + } + builder.build() + } + } + + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJestAction[JestResult, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + ) { + new GetIndex.Builder(index).build() + } { result => + if (result.isSucceeded) { + Some(result.getJsonString) + } else { + None + } } } /** Delete an index. + * * @see * [[IndicesApi.deleteIndex]] */ @@ -99,7 +138,8 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = executeJestAction[JestResult, (Boolean, Option[Long])]( operation = "reindex", @@ -131,4 +171,96 @@ trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientHelpe new IndicesExists.Builder(index).build() } } + + private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJestAction[JestResult, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + ) { + val builder = new DeleteByQuery.Builder(jsonQuery) + .addIndex(index) + + builder.setParameter("conflicts", "proceed") + + if (refresh) + builder.setParameter("refresh", true) + + builder.build() + } { result => + val deleted = Try { + result.getJsonObject.get("deleted").getAsLong + }.getOrElse(0L) + + deleted + } + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJestAction[CatResult, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + ) { + new Cat.IndicesBuilder() + .addIndex(index) + .setParameter("format", "json") + .build() + } { result => + val json = result.getJsonObject + val arr = json.getAsJsonArray("result") + + if (arr == null || arr.size() == 0) + false + else { + val entry = arr.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } + + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = { + executeJestBooleanAction( + operation = "waitForShards", + index = Some(index), + retryable = true + ) { + new WaitForShards.Builder(index = index, status = status, timeout = timeout).build() + }.map(_ => ()) + } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + executeJestAction[UpdateByQueryResult, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + ) { + val builder = new UpdateByQuery.Builder(jsonQuery) + .addIndex(index) + + if (refresh) + builder.setParameter("refresh", true) + + pipelineId.foreach { id => + builder.setParameter("pipeline", id) + } + + builder.build() + } { result => + val json = result.getJsonObject + json.get("updated").getAsLong + } + } } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala index f9a3cec0..fe53c9c3 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestMappingApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndicesApi, MappingApi, RefreshApi, SettingsApi} +import app.softnetwork.elastic.client.MappingApi import app.softnetwork.elastic.client.result.{ ElasticError, ElasticFailure, @@ -33,7 +33,12 @@ import scala.util.Try * [[MappingApi]] for generic API documentation */ trait JestMappingApi extends MappingApi with JestClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JestClientCompanion => + _: JestSettingsApi + with JestIndicesApi + with JestRefreshApi + with JestVersionApi + with JestAliasApi + with JestClientCompanion => /** Set the mapping for an index. * @see @@ -71,7 +76,7 @@ trait JestMappingApi extends MappingApi with JestClientHelpers { getMapping(index).flatMap { jsonString => // βœ… Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala new file mode 100644 index 00000000..35d392de --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestPipelineApi.scala @@ -0,0 +1,94 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.jest.actions.Pipeline +import app.softnetwork.elastic.client.{result, PipelineApi, SerializationApi} +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import io.searchbox.client.JestResult + +trait JestPipelineApi extends PipelineApi with JestClientHelpers { + _: JestVersionApi with SerializationApi with JestClientCompanion => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): result.ElasticResult[Boolean] = { + // There is no direct API to create a pipeline in Jest. + apply().execute(Pipeline.Create(pipelineName, pipelineDefinition)) match { + case jestResult: JestResult if jestResult.isSucceeded => + result.ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to create pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): result.ElasticResult[Boolean] = { + // There is no direct API to delete a pipeline in Jest. + apply().execute(Pipeline.Delete(pipelineName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + result.ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to delete pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + + override private[client] def executeGetPipeline( + pipelineName: String + ): result.ElasticResult[Option[String]] = { + // There is no direct API to get a pipeline in Jest. + apply().execute(Pipeline.Get(pipelineName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode if objectNode.has(pipelineName) => + val pipelineNode = objectNode.get(pipelineName) + result.ElasticSuccess(Some(pipelineNode)) + case _ => + result.ElasticSuccess(None) + } + } else { + result.ElasticSuccess(None) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + result.ElasticFailure( + result.ElasticError( + s"Failed to get pipeline '$pipelineName': $errorMessage" + ) + ) + } + } + +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala index af2cf47f..8fefa648 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala @@ -19,15 +19,7 @@ package app.softnetwork.elastic.client.jest import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Source -import app.softnetwork.elastic.client.{ - retryWithBackoff, - ClientAggregation, - ElasticQuery, - ElasticResponse, - ScrollApi, - SearchApi, - VersionApi -} +import app.softnetwork.elastic.client.{retryWithBackoff, ClientAggregation, ElasticQuery, ScrollApi} import app.softnetwork.elastic.client.scroll.ScrollConfig import app.softnetwork.elastic.sql.query.SQLAggregation import com.google.gson.{JsonNull, JsonObject, JsonParser} @@ -40,7 +32,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} trait JestScrollApi extends ScrollApi with JestClientHelpers { - _: VersionApi with SearchApi with JestClientCompanion => + _: JestVersionApi with JestSearchApi with JestClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -145,7 +137,7 @@ trait JestScrollApi extends ScrollApi with JestClientHelpers { logger.debug(s"Fetching next search_after batch (after: ${values.mkString(", ")})") } - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Check if sorts already exist in the query if (!hasSorts && !queryJson.has("sort")) { diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 36e0881a..21809931 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.client.jest import app.softnetwork.elastic.client.{ElasticQueries, ElasticQuery, SearchApi, SerializationApi} import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.sql.bridge.ElasticSearchRequest -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.MultiSearch import scala.concurrent.{ExecutionContext, Future} @@ -29,7 +29,9 @@ import scala.language.implicitConversions trait JestSearchApi extends SearchApi with JestClientHelpers { _: JestClientCompanion with SerializationApi => - private[client] implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + private[client] implicit def sqlSearchRequestToJsonQuery( + sqlSearch: SingleSearch + )(implicit timestamp: Long): String = implicitly[ElasticSearchRequest](sqlSearch).query import JestClientApi._ diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala index dd29e89e..cb022a5a 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSettingsApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{IndicesApi, SettingsApi} +import app.softnetwork.elastic.client.SettingsApi import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.indices.settings.{GetSettings, UpdateSettings} @@ -25,7 +25,7 @@ import io.searchbox.indices.settings.{GetSettings, UpdateSettings} * [[SettingsApi]] for generic API documentation */ trait JestSettingsApi extends SettingsApi with JestClientHelpers { - _: IndicesApi with JestClientCompanion => + _: JestIndicesApi with JestClientCompanion => /** Update index settings. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala new file mode 100644 index 00000000..c7c86c3b --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestTemplateApi.scala @@ -0,0 +1,214 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.jest.actions.Template +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.client.{SerializationApi, TemplateApi} +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import io.searchbox.client.JestResult + +import scala.jdk.CollectionConverters._ + +trait JestTemplateApi extends TemplateApi with JestClientHelpers { + _: JestVersionApi with SerializationApi with JestClientCompanion => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("createTemplate") + ) + ) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("deleteTemplate") + ) + ) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("getTemplate") + ) + ) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("listTemplates") + ) + ) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + ElasticFailure( + ElasticError( + message = "Composable templates are not supported by Jest client (ES < 7.8 only)", + statusCode = Some(501), // Not Implemented + operation = Some("templateExists") + ) + ) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + apply().execute(Template.Create(templateName, templateDefinition)) match { + case jestResult: JestResult if jestResult.isSucceeded => + ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to create template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + apply().execute(Template.Delete(templateName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + ElasticSuccess(true) + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to delete template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + apply().execute(Template.Get(templateName)) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode if objectNode.has(templateName) => + val templateNode = objectNode.get(templateName) + ElasticSuccess(Some(templateNode)) + case _ => + ElasticSuccess(None) + } + } else { + ElasticSuccess(None) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to get template '$templateName': $errorMessage" + ) + ) + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + apply().execute(Template.GetAll()) match { + case jestResult: JestResult if jestResult.isSucceeded => + val jsonString = jestResult.getJsonString + if (jsonString != null && jsonString.nonEmpty) { + val node: JsonNode = jsonString + node match { + case objectNode: ObjectNode => + val templates = objectNode + .fields() + .asScala + .map { entry => + entry.getKey -> entry.getValue.toString + } + .toMap + ElasticSuccess(templates) + case _ => + ElasticSuccess(Map.empty[String, String]) + } + } else { + ElasticSuccess(Map.empty[String, String]) + } + case jestResult: JestResult => + val errorMessage = jestResult.getErrorMessage + ElasticFailure( + ElasticError( + s"Failed to list templates: $errorMessage" + ) + ) + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + apply().execute(Template.Exists(templateName)) match { + case jestResult: JestResult => + val statusCode = jestResult.getResponseCode + ElasticSuccess(statusCode == 200) + case _ => + ElasticSuccess(false) + } + } + +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala index 6a6a924c..9ef0648f 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestUpdateApi.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.client.jest -import app.softnetwork.elastic.client.{SerializationApi, SettingsApi, UpdateApi} +import app.softnetwork.elastic.client.{SerializationApi, UpdateApi} import app.softnetwork.elastic.client.bulk.docAsUpsert import app.softnetwork.elastic.client.result.ElasticResult import io.searchbox.core.Update @@ -28,7 +28,7 @@ import scala.concurrent.{ExecutionContext, Future} * [[UpdateApi]] for generic API documentation */ trait JestUpdateApi extends UpdateApi with JestClientHelpers { - _: SettingsApi with JestClientCompanion with SerializationApi => + _: JestSettingsApi with JestClientCompanion with SerializationApi => /** Update an entity in the given index. * @see diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala new file mode 100644 index 00000000..9f109c96 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/GetIndex.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.action.AbstractAction +import io.searchbox.action.GenericResultAbstractAction +import io.searchbox.client.config.ElasticsearchVersion + +object GetIndex { + class Builder(var index: String) extends AbstractAction.Builder[GetIndex, GetIndex.Builder] { + override def build = new GetIndex(this) + } +} + +class GetIndex protected (builder: GetIndex.Builder) extends GenericResultAbstractAction(builder) { + override def getRestMethodName = "GET" + + override def buildURI(elasticsearchVersion: ElasticsearchVersion): String = s"/${builder.index}" + +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala new file mode 100644 index 00000000..1e0f168d --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Pipeline.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.client.config.ElasticsearchVersion + +object Pipeline { + + import io.searchbox.action.AbstractAction + import io.searchbox.client.JestResult + import com.google.gson.Gson + + case class Create(pipelineId: String, json: String) extends AbstractAction[JestResult] { + + payload = json + + override def getRestMethodName: String = "PUT" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineId" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Get(pipelineId: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineId" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201 || statusCode == 404) + if (statusCode != 404) { + Option(json).foreach(result.setJsonString) + } + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Delete(pipelineName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "DELETE" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_ingest/pipeline/$pipelineName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala new file mode 100644 index 00000000..1c38c2fa --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/Template.scala @@ -0,0 +1,140 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.client.config.ElasticsearchVersion + +object Template { + + import io.searchbox.action.AbstractAction + import io.searchbox.client.JestResult + import com.google.gson.Gson + + case class Create(templateName: String, json: String) extends AbstractAction[JestResult] { + + payload = json + + override def getRestMethodName: String = "PUT" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Get(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201 || statusCode == 404) + if (statusCode != 404) { + Option(json).foreach(result.setJsonString) + } + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class GetAll() extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "GET" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + "/_template" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Delete(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "DELETE" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 201) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } + + case class Exists(templateName: String) extends AbstractAction[JestResult] { + + override def getRestMethodName: String = "HEAD" + + override def getURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_template/$templateName" + + override def createNewElasticSearchResult( + json: String, + statusCode: Int, + reasonPhrase: String, + gson: Gson + ): JestResult = { + val result = new JestResult(gson) + result.setResponseCode(statusCode) + result.setSucceeded(statusCode == 200 || statusCode == 404) + Option(json).foreach(result.setJsonString) + Option(reasonPhrase).foreach(result.setErrorMessage) + result + } + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala new file mode 100644 index 00000000..dcfc5c43 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/actions/WaitForShards.scala @@ -0,0 +1,36 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.jest.actions + +import io.searchbox.action.{AbstractAction, GenericResultAbstractAction} +import io.searchbox.client.config.ElasticsearchVersion + +object WaitForShards { + class Builder(var index: String, var status: String = "yellow", var timeout: Int = 30) + extends AbstractAction.Builder[WaitForShards, WaitForShards.Builder] { + override def build = new WaitForShards(this) + } +} + +class WaitForShards protected (builder: WaitForShards.Builder) + extends GenericResultAbstractAction(builder) { + override def getRestMethodName = "GET" + + override def buildURI(elasticsearchVersion: ElasticsearchVersion): String = + s"/_cluster/health/${builder.index}?wait_for_status=${builder.status}&timeout=${builder.timeout}s" + +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala new file mode 100644 index 00000000..631ca626 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientInsertByQuerySpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JestClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override def client: ElasticClientApi = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala new file mode 100644 index 00000000..aa59f134 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit + +class JestClientPipelineApiSpec extends PipelineApiSpec with EmbeddedElasticTestKit { + override def client: PipelineApi = new JestClientSpi().client(elasticConfig) +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala index fbbf9b0e..8808ca47 100644 --- a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala @@ -1,3 +1,5 @@ package app.softnetwork.elastic.client -class JestClientSpec extends ElasticClientSpec +class JestClientSpec extends ElasticClientSpec { + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala new file mode 100644 index 00000000..3882b343 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientTemplateApiSpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JestClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override def client: TemplateApi with VersionApi = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala new file mode 100644 index 00000000..a79e53e9 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestGatewayApiSpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JestClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JestGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override def client: GatewayApi = new JestClientSpi().client(elasticConfig) + + override def elasticVersion: String = "6.7.2" +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 9758537f..66dc032f 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,12 +21,16 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.{ObjectValue, Value} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.schema.TableAlias +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest +import org.elasticsearch.action.admin.indices.alias.{Alias, IndicesAliasesRequest} import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest import org.elasticsearch.action.admin.indices.close.CloseIndexRequest @@ -36,10 +40,17 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.admin.indices.refresh.{RefreshRequest, RefreshResponse} import org.elasticsearch.action.admin.indices.settings.get.{GetSettingsRequest, GetSettingsResponse} import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + GetPipelineResponse, + PutPipelineRequest +} import org.elasticsearch.action.search.{ ClearScrollRequest, MultiSearchRequest, @@ -52,15 +63,23 @@ import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} -import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions} +import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CreateIndexRequest, GetIndexRequest, + GetIndexTemplatesRequest, + GetIndexTemplatesResponse, GetMappingsRequest, + GetMappingsResponse, + IndexTemplateMetaData, + IndexTemplatesExistRequest, + PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.cluster.metadata.AliasMetaData import org.elasticsearch.common.Strings +import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.xcontent.{DeprecationHandler, XContentType} import org.elasticsearch.rest.RestStatus @@ -92,15 +111,17 @@ trait RestHighLevelClientApi with RestHighLevelClientBulkApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion - with RestHighLevelClientVersion + with RestHighLevelClientVersionApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientTemplateApi /** Version API implementation for RestHighLevelClient * @see * [[VersionApi]] for generic API documentation */ -trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { +trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeRestLowLevelAction[String]( operation = "version", index = None, @@ -122,23 +143,71 @@ trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelp * [[IndicesApi]] for generic API documentation */ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { - _: RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientScrollApi + with RestHighLevelClientBulkApi + with RestHighLevelClientVersionApi + with RestHighLevelClientTemplateApi + with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String - ): result.ElasticResult[Boolean] = { + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), retryable = false - )( - request = new CreateIndexRequest(index).settings(settings, XContentType.JSON) - )( + )(request = { + val req = new CreateIndexRequest(index) + .settings(settings, XContentType.JSON) + .aliases( + aliases + .map(alias => { + var a = new Alias(alias.alias).writeIndex(alias.isWriteIndex) + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + a = a.filter(filterNode.toString) + } + alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) + alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) + a + }) + .asJava + ) + mappings match { + case Some(m) if m.trim.startsWith("{") && m.trim.endsWith("}") => + req.mapping(m, XContentType.JSON) + case _ => req + } + })( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) } - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeRestAction[Request, Response, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + )( + request = new Request("GET", s"/$index") + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )(resp => { + resp.getStatusLine match { + case statusLine if statusLine.getStatusCode >= 400 => + None + case _ => + val json = scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + Some(json) + } + }) + } + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", index = Some(index), @@ -149,7 +218,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().delete(req, RequestOptions.DEFAULT) ) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[CloseIndexRequest, AcknowledgedResponse]( operation = "closeIndex", index = Some(index), @@ -160,7 +229,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().close(req, RequestOptions.DEFAULT) ) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[OpenIndexRequest, AcknowledgedResponse]( operation = "openIndex", index = Some(index), @@ -174,8 +243,9 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean - ): result.ElasticResult[(Boolean, Option[Long])] = + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", index = Some(s"$sourceIndex->$targetIndex"), @@ -204,8 +274,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH case statusLine if statusLine.getStatusCode >= 400 => (false, None) case _ => - val json = new JsonParser() - .parse( + val json = JsonParser + .parseString( scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString ) .getAsJsonObject @@ -217,7 +287,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } }) - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeRestAction[GetIndexRequest, Boolean, Boolean]( operation = "indexExists", index = Some(index), @@ -230,6 +300,127 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH identity ) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeRestAction[Request, Response, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_delete_by_query?refresh=$refresh&conflicts=proceed" + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonObject + + // ES6/ES7 return "deleted" + val deleted = + if (json.has("deleted")) json.get("deleted").getAsLong + else 0L + + deleted + } + ) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeRestAction[Request, Response, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + request = { + val req = new Request("GET", s"/_cat/indices/$index?format=json") + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonArray + + if (json.size() == 0) + false + else { + val entry = json.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } + ) + + override private[client] def waitForShards( + index: String, + status: String, + timeout: Int + ): ElasticResult[Unit] = { + executeRestAction[Request, Response, Unit]( + operation = "waitForShards", + index = Some(index.toString), + retryable = true + )( + request = { + val req = new Request( + "GET", + s"/_cluster/health/${index}?wait_for_status=${status}&timeout=${timeout}s" + ) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = _ => () + ) + } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeRestAction[Request, Response, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_update_by_query?refresh=$refresh" + + pipelineId.map(id => s"&pipeline=$id").getOrElse("") + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString( + scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + ) + .getAsJsonObject + json.get("updated").getAsLong + } + ) + } } /** Alias management API for RestHighLevelClient @@ -237,23 +428,32 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH * [[AliasApi]] for generic API documentation */ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String - ): result.ElasticResult[Boolean] = + alias: TableAlias + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false )( - request = new IndicesAliasesRequest() - .addAliasAction( - new AliasActions(AliasActions.Type.ADD) - .index(index) - .alias(alias) - ) + request = { + val aliasAction = new AliasActions(AliasActions.Type.ADD) + .index(alias.table) + .alias(alias.alias) + if (alias.isWriteIndex) { + aliasAction.writeIndex(true) + } + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + aliasAction.filter(filterNode.toString) + } + alias.routing.foreach(aliasAction.routing) + alias.indexRouting.foreach(aliasAction.indexRouting) + alias.searchRouting.foreach(aliasAction.searchRouting) + new IndicesAliasesRequest().addAliasAction(aliasAction) + } )( executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) @@ -261,7 +461,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "removeAlias", index = Some(index), @@ -277,7 +477,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeRestAction[GetAliasesRequest, GetAliasesResponse, Boolean]( operation = "aliasExists", index = Some(alias), @@ -288,7 +488,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().getAlias(req, RequestOptions.DEFAULT) )(response => !response.getAliases.isEmpty) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeRestAction[GetAliasesRequest, GetAliasesResponse, String]( operation = "getAliases", index = Some(index), @@ -303,7 +503,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex -> $newIndex"), @@ -331,12 +531,12 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe * [[SettingsApi]] for generic API documentation */ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "updateSettings", index = Some(index), @@ -348,7 +548,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien executor = req => apply().indices().putSettings(req, RequestOptions.DEFAULT) ) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeRestAction[GetSettingsRequest, GetSettingsResponse, String]( operation = "loadSettings", index = Some(index), @@ -366,11 +566,16 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien * [[MappingApi]] for generic API documentation */ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientVersionApi + with RestHighLevelClientAliasApi + with RestHighLevelClientCompanion => override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "setMapping", index = Some(index), @@ -382,10 +587,10 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH executor = req => apply().indices().putMapping(req, RequestOptions.DEFAULT) ) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeRestAction[ GetMappingsRequest, - org.elasticsearch.client.indices.GetMappingsResponse, + GetMappingsResponse, String ]( operation = "getMapping", @@ -411,7 +616,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH */ trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeRestAction[RefreshRequest, RefreshResponse, Boolean]( operation = "refresh", index = Some(index), @@ -434,7 +639,7 @@ trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientHelpe index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[FlushRequest, FlushResponse, Boolean]( operation = "flush", index = Some(index), @@ -455,7 +660,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe _: RestHighLevelClientCompanion => override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeRestAction[CountRequest, CountResponse, Option[Double]]( operation = "count", index = Some(query.indices.mkString(",")), @@ -468,7 +673,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = { + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = { executeAsyncRestAction[CountRequest, CountResponse, Option[Double]]( operation = "countAsync", index = Some(query.indices.mkString(",")), @@ -487,13 +692,13 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe * [[IndexApi]] for generic API documentation */ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[IndexRequest, IndexResponse, Boolean]( operation = "index", index = Some(index), @@ -525,7 +730,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[IndexRequest, IndexResponse, Boolean]( operation = "indexAsync", index = Some(index), @@ -557,14 +762,14 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe * [[UpdateApi]] for generic API documentation */ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, id: String, source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "update", index = Some(index), @@ -598,7 +803,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = executeAsyncRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "updateAsync", index = Some(index), @@ -633,13 +838,13 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel * [[DeleteApi]] for generic API documentation */ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => override private[client] def executeDelete( index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "delete", index = Some(index), @@ -661,7 +866,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "deleteAsync", index = Some(index), @@ -692,7 +897,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[GetRequest, GetResponse, Option[String]]( operation = "get", index = Some(index), @@ -711,7 +916,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[GetRequest, GetResponse, Option[String]]( operation = "getAsync", index = Some(index), @@ -737,12 +942,14 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -772,7 +979,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "multiSearch", index = Some( @@ -808,7 +1015,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "executeSingleSearchAsync", index = Some(elasticQuery.indices.mkString(",")), @@ -838,7 +1045,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "executeMultiSearchAsync", index = Some( @@ -879,7 +1086,10 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel * [[BulkApi]] for generic API documentation */ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndexApi + with RestHighLevelClientCompanion => override type BulkActionType = DocWriteRequest[_] override type BulkResultType = BulkResponse @@ -1112,7 +1322,9 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers * [[ScrollApi]] for generic API documentation */ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { - _: VersionApi with SearchApi with RestHighLevelClientCompanion => + _: RestHighLevelClientVersionApi + with RestHighLevelClientSearchApi + with RestHighLevelClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1385,3 +1597,305 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } } + +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( + operation = "createPipeline", + retryable = false + )( + request = new PutPipelineRequest( + pipelineName, + new BytesArray(pipelineDefinition), + XContentType.JSON + ) + )( + executor = req => apply().ingest().putPipeline(req, RequestOptions.DEFAULT) + ) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( + operation = "deletePipeline", + retryable = false + )( + request = new DeletePipelineRequest(pipelineName) + )( + executor = req => apply().ingest().deletePipeline(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[String]]( + operation = "getPipeline", + retryable = true + )( + request = new GetPipelineRequest(pipelineName) + )( + executor = req => apply().ingest().getPipeline(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val pipelines = resp.pipelines().asScala + if (pipelines.nonEmpty) { + val pipeline = pipelines.head + val config = pipeline.getConfigAsMap + Some(mapper.writeValueAsString(config)) + } else { + None + } + } + ) + } +} + +trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = ElasticSuccess(false) + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = ElasticSuccess(None) + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = ElasticSuccess(Map.empty[String, String]) + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = ElasticSuccess(false) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Option[String]]( + operation = "getTemplate", + retryable = true + )( + request = new GetIndexTemplatesRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates.asScala + if (templates.nonEmpty) { + val template = templates.head + Some(legacyTemplateToJson(template)) + } else { + None + } + } + ) + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Map[String, String]]( + operation = "listTemplates", + retryable = true + )( + request = new GetIndexTemplatesRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + resp.getIndexTemplates.asScala.map { template => + template.name() -> legacyTemplateToJson(template) + }.toMap + } + ) + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[IndexTemplatesExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new IndexTemplatesExistRequest(templateName) + )( + executor = req => apply().indices().existsTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== CONVERSION HELPERS ==================== + + private def legacyTemplateToJson(template: IndexTemplateMetaData): String = { + + val root = mapper.createObjectNode() + + // index_patterns + val patternsArray = mapper.createArrayNode() + template.patterns().asScala.foreach(patternsArray.add) + root.set("index_patterns", patternsArray) + + // order + root.put("order", template.order()) + + // version + if (template.version() != null) { + root.put("version", template.version()) + } + + // settings + if (template.settings() != null && !template.settings().isEmpty) { + val settingsNode = mapper.createObjectNode() + template.settings().keySet().asScala.foreach { key => + val value = template.settings().get(key) + if (value != null) { + settingsNode.put(key, value) + } + } + root.set("settings", settingsNode) + } + + // mappings + if (template.mappings() != null && template.mappings().source().string().nonEmpty) { + try { + val mappingsNode = mapper.readTree(template.mappings().source().string()) + root.set("mappings", mappingsNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse mappings: ${e.getMessage}") + } + } + + // aliases + if (template.aliases() != null && !template.aliases().isEmpty) { + val aliasesNode = mapper.createObjectNode() + + template.aliases().keysIt().asScala.foreach { aliasName => + // Type explicite pour Γ©viter l'infΓ©rence vers Nothing$ + val aliasObjectNode: ObjectNode = + try { + val aliasMetadata = template.aliases().get(aliasName) + + // Convert AliasMetadata to JSON + val aliasJson = convertAliasMetadataToJson(aliasMetadata) + + // Parse and validate + val parsedAlias = mapper.readTree(aliasJson) + + if (parsedAlias.isInstanceOf[ObjectNode]) { + parsedAlias.asInstanceOf[ObjectNode].get(aliasName) match { + case objNode: ObjectNode => objNode + case _ => + logger.debug( + s"Alias '$aliasName' does not contain an object node, creating empty object" + ) + mapper.createObjectNode() + } + } else { + logger.debug( + s"Alias '$aliasName' is not an ObjectNode (type: ${parsedAlias.getClass.getName}), creating empty object" + ) + mapper.createObjectNode() + } + + } catch { + case e: Exception => + logger.warn(s"Failed to process alias '$aliasName': ${e.getMessage}", e) + mapper.createObjectNode() + } + + // Set with explicit type + aliasesNode.set[ObjectNode](aliasName, aliasObjectNode) + } + + root.set("aliases", aliasesNode) + } + + mapper.writeValueAsString(root) + } + + /** Convert AliasMetadata to JSON string + * + * @param aliasMetadata + * the alias metadata + * @return + * JSON string representation + */ + private def convertAliasMetadataToJson( + aliasMetadata: AliasMetaData + ): String = { + try { + import org.elasticsearch.common.xcontent.{XContentFactory, ToXContent} + + val builder = XContentFactory.jsonBuilder() + builder.startObject() + aliasMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.endObject() + builder.close() + + org.elasticsearch.common.Strings.toString(builder) + + } catch { + case e: Exception => + logger.warn(s"Failed to convert AliasMetadata to JSON: ${e.getMessage}", e) + "{}" + } + } +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala index afa5f20e..522cbdac 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala @@ -106,6 +106,20 @@ trait RestHighLevelClientHelpers extends ElasticClientHelpers { _: RestHighLevel operation = Some(operation) ) ) + case Failure(ex: org.elasticsearch.client.ResponseException) => + val statusCode = Some(ex.getResponse.getStatusLine.getStatusCode) + logger.error( + s"Response exception during operation '$operation'$indexStr: ${ex.getMessage}", + ex + ) + ElasticResult.failure( + ElasticError( + message = s"HTTP error during $operation: ${ex.getMessage}", + cause = Some(ex), + statusCode = statusCode, + operation = Some(operation) + ) + ) case Failure(ex) => logger.error(s"Exception during operation '$operation'$indexStr: ${ex.getMessage}", ex) ElasticResult.failure( diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala index e8db8fbc..9193e155 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelBulkApiSpec.scala @@ -1,8 +1,8 @@ package app.softnetwork.elastic.client import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -class RestHighLevelBulkApiSpec extends BulkApiSpec with EmbeddedElasticTestKit { +class RestHighLevelBulkApiSpec extends BulkApiSpec with ElasticDockerTestKit { override lazy val client: BulkApi = new RestHighLevelClientSpi().client(elasticConfig) } diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala index 89867667..71d5e8f9 100644 --- a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientCompanionSpec.scala @@ -2,7 +2,7 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.rest.RestHighLevelClientCompanion -import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit import app.softnetwork.persistence.generateUUID import org.scalatest.wordspec.AnyWordSpec import org.scalatest.matchers.should.Matchers @@ -16,7 +16,7 @@ import scala.util.Try class RestHighLevelClientCompanionSpec extends AnyWordSpec - with EmbeddedElasticTestKit + with ElasticDockerTestKit with Matchers with ScalaFutures { diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala new file mode 100644 index 00000000..bd302beb --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientGatewayApiSpec + extends GatewayApiIntegrationSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala new file mode 100644 index 00000000..5dbbd428 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala new file mode 100644 index 00000000..b10763b7 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala new file mode 100644 index 00000000..67b1622c --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = + new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 63356f2e..c21aaf84 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -21,11 +21,17 @@ import akka.actor.ActorSystem import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ +import app.softnetwork.elastic.sql.{ObjectValue, Value} import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import org.apache.http.util.EntityUtils +import org.elasticsearch.action.admin.indices.alias.Alias import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest @@ -35,10 +41,17 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexRequest import org.elasticsearch.action.admin.indices.refresh.{RefreshRequest, RefreshResponse} import org.elasticsearch.action.admin.indices.settings.get.{GetSettingsRequest, GetSettingsResponse} import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest import org.elasticsearch.action.bulk.{BulkRequest, BulkResponse} import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} import org.elasticsearch.action.get.{GetRequest, GetResponse} import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + GetPipelineResponse, + PutPipelineRequest +} import org.elasticsearch.action.search.{ ClearScrollRequest, ClosePointInTimeRequest, @@ -53,18 +66,29 @@ import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} -import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions} +import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.indices.{ CloseIndexRequest, + ComposableIndexTemplateExistRequest, CreateIndexRequest, + DeleteComposableIndexTemplateRequest, + GetComposableIndexTemplateRequest, GetIndexRequest, + GetIndexTemplatesRequest, + GetIndexTemplatesResponse, GetMappingsRequest, + IndexTemplateMetadata, + IndexTemplatesExistRequest, + PutComposableIndexTemplateRequest, + PutIndexTemplateRequest, PutMappingRequest } +import org.elasticsearch.cluster.metadata.{AliasMetadata, ComposableIndexTemplate} import org.elasticsearch.common.Strings +import org.elasticsearch.common.bytes.BytesArray import org.elasticsearch.core.TimeValue -import org.elasticsearch.xcontent.{DeprecationHandler, XContentType} +import org.elasticsearch.xcontent.{DeprecationHandler, ToXContent, XContentFactory, XContentType} import org.elasticsearch.rest.RestStatus import org.elasticsearch.search.builder.{PointInTimeBuilder, SearchSourceBuilder} import org.elasticsearch.search.sort.{FieldSortBuilder, SortOrder} @@ -94,16 +118,18 @@ trait RestHighLevelClientApi with RestHighLevelClientBulkApi with RestHighLevelClientScrollApi with RestHighLevelClientCompanion - with RestHighLevelClientVersion + with RestHighLevelClientVersionApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientTemplateApi /** Version API implementation for RestHighLevelClient * @see * [[VersionApi]] for generic API documentation */ -trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelpers { +trait RestHighLevelClientVersionApi extends VersionApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion with SerializationApi => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeRestLowLevelAction[String]( operation = "version", index = None, @@ -126,24 +152,75 @@ trait RestHighLevelClientVersion extends VersionApi with RestHighLevelClientHelp * [[IndicesApi]] for generic API documentation */ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientHelpers { - _: RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientPipelineApi + with RestHighLevelClientScrollApi + with RestHighLevelClientBulkApi + with RestHighLevelClientVersionApi + with RestHighLevelClientTemplateApi + with RestHighLevelClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String - ): result.ElasticResult[Boolean] = { + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = { executeRestBooleanAction[CreateIndexRequest, AcknowledgedResponse]( operation = "createIndex", index = Some(index), retryable = false )( - request = new CreateIndexRequest(index).settings(settings, XContentType.JSON) + request = { + val req = new CreateIndexRequest(index) + .settings(settings, XContentType.JSON) + .aliases( + aliases + .map(alias => { + var a = + new Alias(alias.alias).writeIndex(alias.isWriteIndex).isHidden(alias.isHidden) + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + a = a.filter(filterNode.toString) + } + alias.indexRouting.foreach(ir => a = a.indexRouting(ir)) + alias.searchRouting.foreach(sr => a = a.searchRouting(sr)) + a + }) + .asJava + ) + mappings match { + case Some(m) if m.trim.startsWith("{") && m.trim.endsWith("}") => + req.mapping(m, XContentType.JSON) + case _ => req + } + } )( executor = req => apply().indices().create(req, RequestOptions.DEFAULT) ) } - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeRestAction[Request, Response, Option[String]]( + operation = "getIndex", + index = Some(index), + retryable = true + )( + request = new Request("GET", s"/$index") + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )(resp => { + resp.getStatusLine match { + case statusLine if statusLine.getStatusCode >= 400 => + None + case _ => + val json = scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + Some(json) + } + }) + } + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[DeleteIndexRequest, AcknowledgedResponse]( operation = "deleteIndex", index = Some(index), @@ -154,7 +231,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().delete(req, RequestOptions.DEFAULT) ) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[CloseIndexRequest, AcknowledgedResponse]( operation = "closeIndex", index = Some(index), @@ -165,7 +242,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH executor = req => apply().indices().close(req, RequestOptions.DEFAULT) ) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeRestBooleanAction[OpenIndexRequest, AcknowledgedResponse]( operation = "openIndex", index = Some(index), @@ -179,8 +256,9 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean - ): result.ElasticResult[(Boolean, Option[Long])] = + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = executeRestAction[Request, org.elasticsearch.client.Response, (Boolean, Option[Long])]( operation = "reindex", index = Some(s"$sourceIndex->$targetIndex"), @@ -209,8 +287,8 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH case statusLine if statusLine.getStatusCode >= 400 => (false, None) case _ => - val json = new JsonParser() - .parse( + val json = JsonParser + .parseString( scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString ) .getAsJsonObject @@ -222,7 +300,7 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH } }) - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeRestAction[GetIndexRequest, Boolean, Boolean]( operation = "indexExists", index = Some(index), @@ -235,6 +313,103 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH identity ) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeRestAction[Request, Response, Long]( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_delete_by_query?refresh=$refresh&conflicts=proceed" + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonObject + + // ES6/ES7 return "deleted" + val deleted = + if (json.has("deleted")) json.get("deleted").getAsLong + else 0L + + deleted + } + ) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeRestAction[Request, Response, Boolean]( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + request = { + val req = new Request("GET", s"/_cat/indices/$index?format=json") + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString(scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString) + .getAsJsonArray + + if (json.size() == 0) + false + else { + val entry = json.get(0).getAsJsonObject + val status = entry.get("status").getAsString // "open" or "close" + status == "close" + } + } + ) + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeRestAction[Request, Response, Long]( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + request = { + val req = new Request( + "POST", + s"/$index/_update_by_query?refresh=$refresh" + + pipelineId.map(id => s"&pipeline=$id").getOrElse("") + ) + req.setJsonEntity(jsonQuery) + req + } + )( + executor = req => apply().getLowLevelClient.performRequest(req) + )( + transformer = resp => { + val json = JsonParser + .parseString( + scala.io.Source.fromInputStream(resp.getEntity.getContent).mkString + ) + .getAsJsonObject + json.get("updated").getAsLong + } + ) + } } /** Alias management API for RestHighLevelClient @@ -242,23 +417,32 @@ trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientH * [[AliasApi]] for generic API documentation */ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String - ): result.ElasticResult[Boolean] = + alias: TableAlias + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false )( - request = new IndicesAliasesRequest() - .addAliasAction( - new AliasActions(AliasActions.Type.ADD) - .index(index) - .alias(alias) - ) + request = { + val aliasAction = new AliasActions(AliasActions.Type.ADD) + .index(alias.table) + .alias(alias.alias) + if (alias.isWriteIndex) { + aliasAction.writeIndex(true) + } + if (alias.filter.nonEmpty) { + val filterNode = Value(alias.filter).asInstanceOf[ObjectValue].toJson + aliasAction.filter(filterNode.toString) + } + alias.routing.foreach(aliasAction.routing) + alias.indexRouting.foreach(aliasAction.indexRouting) + alias.searchRouting.foreach(aliasAction.searchRouting) + new IndicesAliasesRequest().addAliasAction(aliasAction) + } )( executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) @@ -266,7 +450,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "removeAlias", index = Some(index), @@ -282,7 +466,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().updateAliases(req, RequestOptions.DEFAULT) ) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeRestAction[GetAliasesRequest, GetAliasesResponse, Boolean]( operation = "aliasExists", index = Some(alias), @@ -293,7 +477,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe executor = req => apply().indices().getAlias(req, RequestOptions.DEFAULT) )(response => !response.getAliases.isEmpty) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeRestAction[GetAliasesRequest, GetAliasesResponse, String]( operation = "getAliases", index = Some(index), @@ -308,7 +492,7 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex -> $newIndex"), @@ -335,12 +519,12 @@ trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientHelpe * [[SettingsApi]] for generic API documentation */ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientHelpers { - _: IndicesApi with RestHighLevelClientCompanion => + _: RestHighLevelClientIndicesApi with RestHighLevelClientCompanion => override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "updateSettings", index = Some(index), @@ -352,7 +536,7 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien executor = req => apply().indices().putSettings(req, RequestOptions.DEFAULT) ) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeRestAction[GetSettingsRequest, GetSettingsResponse, String]( operation = "loadSettings", index = Some(index), @@ -370,12 +554,17 @@ trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClien * [[MappingApi]] for generic API documentation */ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientVersionApi + with RestHighLevelClientAliasApi + with RestHighLevelClientCompanion => override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestBooleanAction( operation = "setMapping", index = Some(index), @@ -387,7 +576,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH executor = req => apply().indices().putMapping(req, RequestOptions.DEFAULT) ) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeRestAction[ GetMappingsRequest, org.elasticsearch.client.indices.GetMappingsResponse, @@ -417,7 +606,7 @@ trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientH trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientHelpers { _: RestHighLevelClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeRestAction[RefreshRequest, RefreshResponse, Boolean]( operation = "refresh", index = Some(index), @@ -440,7 +629,7 @@ trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientHelpe index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[FlushRequest, FlushResponse, Boolean]( operation = "flush", index = Some(index), @@ -461,7 +650,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe _: RestHighLevelClientCompanion => override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeRestAction[CountRequest, CountResponse, Option[Double]]( operation = "count", index = Some(query.indices.mkString(",")), @@ -474,7 +663,7 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = { + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = { executeAsyncRestAction[CountRequest, CountResponse, Option[Double]]( operation = "countAsync", index = Some(query.indices.mkString(",")), @@ -493,13 +682,13 @@ trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientHelpe * [[IndexApi]] for generic API documentation */ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[IndexRequest, IndexResponse, Boolean]( operation = "index", index = Some(index), @@ -530,7 +719,7 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[IndexRequest, IndexResponse, Boolean]( operation = "indexAsync", index = Some(index), @@ -561,14 +750,14 @@ trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientHelpe * [[UpdateApi]] for generic API documentation */ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion with SerializationApi => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, id: String, source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "update", index = Some(index), @@ -602,7 +791,7 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = executeAsyncRestAction[UpdateRequest, UpdateResponse, Boolean]( operation = "updateAsync", index = Some(index), @@ -637,13 +826,13 @@ trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientHel * [[DeleteApi]] for generic API documentation */ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHelpers { - _: SettingsApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => override private[client] def executeDelete( index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "delete", index = Some(index), @@ -665,7 +854,7 @@ trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientHel override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = executeAsyncRestAction[DeleteRequest, DeleteResponse, Boolean]( operation = "deleteAsync", index = Some(index), @@ -696,7 +885,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[GetRequest, GetResponse, Option[String]]( operation = "get", index = Some(index), @@ -715,7 +904,7 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[GetRequest, GetResponse, Option[String]]( operation = "getAsync", index = Some(index), @@ -741,12 +930,14 @@ trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientHelpers { trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHelpers { _: ElasticConversion with RestHighLevelClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -776,7 +967,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "multiSearch", index = Some( @@ -812,7 +1003,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[SearchRequest, SearchResponse, Option[String]]( operation = "executeSingleSearchAsync", index = Some(elasticQuery.indices.mkString(",")), @@ -842,7 +1033,7 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = executeAsyncRestAction[MultiSearchRequest, MultiSearchResponse, Option[String]]( operation = "executeMultiSearchAsync", index = Some( @@ -883,7 +1074,10 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel * [[BulkApi]] for generic API documentation */ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with RestHighLevelClientCompanion => + _: RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndexApi + with RestHighLevelClientCompanion => override type BulkActionType = DocWriteRequest[_] override type BulkResultType = BulkResponse @@ -1109,7 +1303,9 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers * [[ScrollApi]] for generic API documentation */ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { - _: SearchApi with VersionApi with RestHighLevelClientCompanion => + _: RestHighLevelClientSearchApi + with RestHighLevelClientVersionApi + with RestHighLevelClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1558,3 +1754,415 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } } } + +trait RestHighLevelClientPipelineApi extends PipelineApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + executeRestBooleanAction[PutPipelineRequest, AcknowledgedResponse]( + operation = "createPipeline", + retryable = false + )( + request = new PutPipelineRequest( + pipelineName, + new BytesArray(pipelineDefinition), + XContentType.JSON + ) + )( + executor = req => apply().ingest().putPipeline(req, RequestOptions.DEFAULT) + ) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + executeRestBooleanAction[DeletePipelineRequest, AcknowledgedResponse]( + operation = "deletePipeline", + retryable = false + )( + request = new DeletePipelineRequest(pipelineName) + )( + executor = req => apply().ingest().deletePipeline(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetPipeline( + pipelineName: JSONQuery + ): ElasticResult[Option[JSONQuery]] = { + executeRestAction[GetPipelineRequest, GetPipelineResponse, Option[JSONQuery]]( + operation = "getPipeline", + retryable = true + )( + request = new GetPipelineRequest(pipelineName) + )( + executor = req => apply().ingest().getPipeline(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val pipelines = resp.pipelines().asScala + if (pipelines.nonEmpty) { + val pipeline = pipelines.head + val config = pipeline.getConfigAsMap + val mapper = JacksonConfig.objectMapper + Some(mapper.writeValueAsString(config)) + } else { + None + } + } + ) + } +} + +trait RestHighLevelClientTemplateApi extends TemplateApi with RestHighLevelClientHelpers { + _: RestHighLevelClientVersionApi with RestHighLevelClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutComposableIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutComposableIndexTemplateRequest() + req.name(templateName) + req.indexTemplate( + ComposableIndexTemplate.parse( + org.elasticsearch.common.xcontent.XContentHelper.createParser( + org.elasticsearch.xcontent.NamedXContentRegistry.EMPTY, + org.elasticsearch.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(templateDefinition), + XContentType.JSON + ) + ) + ) + req + } + )( + executor = req => apply().indices().putIndexTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteComposableIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteComposableIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteIndexTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeRestAction( + operation = "getTemplate", + retryable = true + )( + request = new GetComposableIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates + if (templates != null && !templates.isEmpty) { + val template = templates.get(templateName) + if (template != null) { + Some(composableTemplateToJson(template)) + } else { + None + } + } else { + None + } + } + ) + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = { + executeRestAction( + operation = "listTemplates", + retryable = true + )( + request = new GetComposableIndexTemplateRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates + if (templates != null) { + templates.asScala.map { case (name, template) => + name -> composableTemplateToJson(template) + }.toMap + } else { + Map.empty + } + } + ) + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[ComposableIndexTemplateExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new ComposableIndexTemplateExistRequest(templateName) + )( + executor = req => apply().indices().existsIndexTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = { + executeRestBooleanAction[PutIndexTemplateRequest, AcknowledgedResponse]( + operation = "createTemplate", + retryable = false + )( + request = { + val req = new PutIndexTemplateRequest(templateName) + req.source(new BytesArray(templateDefinition), XContentType.JSON) + req + } + )( + executor = req => apply().indices().putTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeRestBooleanAction[DeleteIndexTemplateRequest, AcknowledgedResponse]( + operation = "deleteTemplate", + retryable = false + )( + request = new DeleteIndexTemplateRequest(templateName) + )( + executor = req => apply().indices().deleteTemplate(req, RequestOptions.DEFAULT) + ) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Option[String]]( + operation = "getTemplate", + retryable = true + )( + request = new GetIndexTemplatesRequest(templateName) + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val templates = resp.getIndexTemplates.asScala + if (templates.nonEmpty) { + val template = templates.head + Some(legacyTemplateToJson(template)) + } else { + None + } + } + ) + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeRestAction[GetIndexTemplatesRequest, GetIndexTemplatesResponse, Map[String, String]]( + operation = "listTemplates", + retryable = true + )( + request = new GetIndexTemplatesRequest("*") + )( + executor = req => apply().indices().getIndexTemplate(req, RequestOptions.DEFAULT) + )( + transformer = resp => { + val mapper = JacksonConfig.objectMapper + resp.getIndexTemplates.asScala.map { template => + template.name() -> legacyTemplateToJson(template) + }.toMap + } + ) + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeRestAction[IndexTemplatesExistRequest, Boolean, Boolean]( + operation = "existsTemplate", + retryable = false + )( + request = new IndexTemplatesExistRequest(templateName) + )( + executor = req => apply().indices().existsTemplate(req, RequestOptions.DEFAULT) + )(response => response) + } + + // ==================== CONVERSION HELPERS ==================== + + private def composableTemplateToJson(template: ComposableIndexTemplate): String = { + try { + val builder = XContentFactory.jsonBuilder() + template.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.close() + org.elasticsearch.common.Strings.toString(builder) + } catch { + case e: Exception => + logger.error(s"Failed to convert composable template to JSON: ${e.getMessage}", e) + "{}" + } + } + + private def legacyTemplateToJson(template: IndexTemplateMetadata): String = { + + val root = mapper.createObjectNode() + + // index_patterns + val patternsArray = mapper.createArrayNode() + template.patterns().asScala.foreach(patternsArray.add) + root.set("index_patterns", patternsArray) + + // order + root.put("order", template.order()) + + // version + if (template.version() != null) { + root.put("version", template.version()) + } + + // settings + if (template.settings() != null && !template.settings().isEmpty) { + val settingsNode = mapper.createObjectNode() + template.settings().keySet().asScala.foreach { key => + val value = template.settings().get(key) + if (value != null) { + settingsNode.put(key, value) + } + } + root.set("settings", settingsNode) + } + + // mappings + if (template.mappings() != null && template.mappings().source().string().nonEmpty) { + try { + val mappingsNode = mapper.readTree(template.mappings().source().string()) + root.set("mappings", mappingsNode) + } catch { + case e: Exception => + logger.warn(s"Failed to parse mappings: ${e.getMessage}") + } + } + + // aliases + if (template.aliases() != null && !template.aliases().isEmpty) { + val aliasesNode = mapper.createObjectNode() + + template.aliases().keysIt().asScala.foreach { aliasName => + // Type explicite pour Γ©viter l'infΓ©rence vers Nothing$ + val aliasObjectNode: ObjectNode = + try { + val aliasMetadata = template.aliases().get(aliasName) + + // Convert AliasMetadata to JSON + val aliasJson = convertAliasMetadataToJson(aliasMetadata) + + // Parse and validate + val parsedAlias = mapper.readTree(aliasJson) + + if (parsedAlias.isInstanceOf[ObjectNode]) { + parsedAlias.asInstanceOf[ObjectNode].get(aliasName) match { + case objNode: ObjectNode => objNode + case _ => + logger.debug( + s"Alias '$aliasName' does not contain an object node, creating empty object" + ) + mapper.createObjectNode() + } + } else { + logger.debug( + s"Alias '$aliasName' is not an ObjectNode (type: ${parsedAlias.getClass.getName}), creating empty object" + ) + mapper.createObjectNode() + } + + } catch { + case e: Exception => + logger.warn(s"Failed to process alias '$aliasName': ${e.getMessage}", e) + mapper.createObjectNode() + } + + // Set with explicit type + aliasesNode.set[ObjectNode](aliasName, aliasObjectNode) + } + + root.set("aliases", aliasesNode) + } + + mapper.writeValueAsString(root) + } + + /** Convert AliasMetadata to JSON string + * + * @param aliasMetadata + * the alias metadata + * @return + * JSON string representation + */ + private def convertAliasMetadataToJson( + aliasMetadata: AliasMetadata + ): String = { + try { + import org.elasticsearch.xcontent.{ToXContent, XContentFactory} + + val builder = XContentFactory.jsonBuilder() + builder.startObject() + aliasMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS) + builder.endObject() + builder.close() + + org.elasticsearch.common.Strings.toString(builder) + + } catch { + case e: Exception => + logger.warn(s"Failed to convert AliasMetadata to JSON: ${e.getMessage}", e) + "{}" + } + } +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala index afa5f20e..522cbdac 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientHelpers.scala @@ -106,6 +106,20 @@ trait RestHighLevelClientHelpers extends ElasticClientHelpers { _: RestHighLevel operation = Some(operation) ) ) + case Failure(ex: org.elasticsearch.client.ResponseException) => + val statusCode = Some(ex.getResponse.getStatusLine.getStatusCode) + logger.error( + s"Response exception during operation '$operation'$indexStr: ${ex.getMessage}", + ex + ) + ElasticResult.failure( + ElasticError( + message = s"HTTP error during $operation: ${ex.getMessage}", + cause = Some(ex), + statusCode = statusCode, + operation = Some(operation) + ) + ) case Failure(ex) => logger.error(s"Exception during operation '$operation'$indexStr: ${ex.getMessage}", ex) ElasticResult.failure( diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala new file mode 100644 index 00000000..bd302beb --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientGatewayApiSpec.scala @@ -0,0 +1,10 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientGatewayApiSpec + extends GatewayApiIntegrationSpec + with ElasticDockerTestKit { + override lazy val client: GatewayApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala new file mode 100644 index 00000000..5dbbd428 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala new file mode 100644 index 00000000..b10763b7 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala new file mode 100644 index 00000000..67b1622c --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientTemplateApiSpec.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.RestHighLevelClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class RestHighLevelClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = + new RestHighLevelClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index ec367082..9005dee0 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,10 +24,18 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} -import app.softnetwork.elastic.client -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ + Conflicts, FieldSort, FieldValue, Refresh, @@ -36,6 +44,7 @@ import co.elastic.clients.elasticsearch._types.{ SortOrder, Time } +import co.elastic.clients.elasticsearch.cat.IndicesRequest import co.elastic.clients.elasticsearch.core.bulk.{ BulkOperation, DeleteOperation, @@ -53,6 +62,12 @@ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESS import co.elastic.clients.elasticsearch.core.search.PointInTimeReference import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} +import co.elastic.clients.elasticsearch.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + PutPipelineRequest +} +import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -80,6 +95,8 @@ trait JavaClientApi with JavaClientScrollApi with JavaClientCompanion with JavaClientVersionApi + with JavaClientPipelineApi + with JavaClientTemplateApi /** Elasticsearch client implementation using the Java Client * @see @@ -87,7 +104,7 @@ trait JavaClientApi */ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { _: SerializationApi with JavaClientCompanion => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeJavaAction( operation = "version", index = None, @@ -103,12 +120,20 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * @see * [[IndicesApi]] for index management operations */ -trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHelpers { - _: JavaClientCompanion => +trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { + _: JavaClientRefreshApi + with JavaClientPipelineApi + with JavaClientScrollApi + with JavaClientBulkApi + with JavaClientVersionApi + with JavaClientTemplateApi + with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String - ): result.ElasticResult[Boolean] = + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", index = Some(index), @@ -116,15 +141,60 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel )( apply() .indices() - .create( - new CreateIndexRequest.Builder() + .create { + val req = new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) - .build() - ) + .aliases( + aliases + .map(alias => { + var builder = + new Alias.Builder().isWriteIndex(alias.isWriteIndex).isHidden(alias.isHidden) + alias.routing.foreach(r => builder = builder.routing(r)) + alias.indexRouting.foreach(ir => builder = builder.indexRouting(ir)) + alias.searchRouting.foreach(sr => builder = builder.searchRouting(sr)) + (alias.alias, builder.build()) + }) + .toMap + .asJava + ) + mappings match { + case None => req.build() + case Some(m) => + req + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(m) + ) + .build() + ) + .build() + } + } )(_.acknowledged()) - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getIndex", + index = Some(index), + retryable = true + )( + apply() + .indices() + .get( + new GetIndexRequest.Builder().index(index).build() + ) + )(response => { + val valueOpt = response.result.asScala.get(index) + valueOpt match { + case Some(value) => Some(convertToJson(value)) + case None => None + } + }) + } + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", index = Some(index), @@ -135,7 +205,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .delete(new DeleteIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "closeIndex", index = Some(index), @@ -146,7 +216,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .close(new CloseIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "openIndex", index = Some(index), @@ -160,8 +230,9 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean - ): result.ElasticResult[(Boolean, Option[Long])] = + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", index = Some(s"$sourceIndex -> $targetIndex"), @@ -185,7 +256,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel (failures.isEmpty, Option(response.total())) } - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "indexExists", index = Some(index), @@ -198,6 +269,70 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.value()) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJavaAction( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + apply().deleteByQuery( + new DeleteByQueryRequest.Builder() + .index(index) + .withJson(new StringReader(jsonQuery)) // JSON query as raw string + .refresh(refresh) + .conflicts(Conflicts.Proceed) + .build() + ) + )(_.deleted()) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJavaAction( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + apply() + .cat() + .indices( + new IndicesRequest.Builder() + .index(index) + .build() + ) + ) { response => + response.valueBody().asScala.headOption match { + case Some(info) => info.status() == "close" + case None => false + } + } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeJavaAction( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + apply().updateByQuery( + new UpdateByQueryRequest.Builder() + .index(index) + .refresh(refresh) + .pipeline(pipelineId.orNull) + .withJson(new StringReader(jsonQuery)) + .build() + ) + ) { response => + response.updated() + } + } } /** Elasticsearch client implementation of Alias API using the Java Client @@ -205,34 +340,39 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel * [[AliasApi]] for alias management operations */ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String - ): result.ElasticResult[Boolean] = + alias: TableAlias + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false - )( + ) { + val node = alias.node + node.put("index", alias.table) + node.put("alias", alias.alias) + + val action = mapper.createObjectNode() + action.set("add", node) + + val addAction = new Action.Builder().withJson(new StringReader(action)) apply() .indices() .updateAliases( new UpdateAliasesRequest.Builder() .actions( - new Action.Builder() - .add(new AddAction.Builder().index(index).alias(alias).build()) - .build() + addAction.build() ) .build() ) - )(_.acknowledged()) + }(_.acknowledged()) override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "removeAlias", index = Some(index), @@ -251,7 +391,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "aliasExists", index = None, @@ -264,7 +404,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.value()) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeJavaAction( operation = "getAliases", index = Some(index), @@ -281,7 +421,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex <-> $newIndex"), @@ -312,12 +452,12 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { * [[SettingsApi]] for settings management operations */ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "updateSettings", index = Some(index), @@ -333,7 +473,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeJavaAction( operation = "loadSettings", index = Some(index), @@ -353,12 +493,17 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { * [[MappingApi]] for mapping management operations */ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JavaClientCompanion => + _: JavaClientSettingsApi + with JavaClientIndicesApi + with JavaClientRefreshApi + with JavaClientVersionApi + with JavaClientAliasApi + with JavaClientCompanion => override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "setMapping", index = Some(index), @@ -371,7 +516,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeJavaAction( operation = "getMapping", index = Some(index), @@ -401,7 +546,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { getMapping(index).flatMap { jsonString => // βœ… Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") @@ -422,7 +567,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { trait JavaClientRefreshApi extends RefreshApi with JavaClientHelpers { _: JavaClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "refresh", index = Some(index), @@ -452,7 +597,7 @@ trait JavaClientFlushApi extends FlushApi with JavaClientHelpers { index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "flush", index = Some(index), @@ -480,7 +625,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeJavaAction( operation = "count", index = Some(query.indices.mkString(",")), @@ -496,14 +641,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = fromCompletableFuture( async() .count( new CountRequest.Builder().index(query.indices.asJava).build() ) ).map { response => - result.ElasticSuccess(Option(response.count().toDouble)) + ElasticSuccess(Option(response.count().toDouble)) } } @@ -513,14 +658,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { * [[IndexApi]] for index operations */ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "index", index = Some(index), @@ -549,7 +694,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .index( @@ -562,8 +707,8 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Created | Result.Updated | Result.NoOp => result.ElasticSuccess(true) - case _ => result.ElasticSuccess(false) + case Result.Created | Result.Updated | Result.NoOp => ElasticSuccess(true) + case _ => ElasticSuccess(false) } } } @@ -573,7 +718,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { * [[UpdateApi]] for update operations */ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, @@ -581,7 +726,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "update", index = Some(index), @@ -615,7 +760,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .update( @@ -630,14 +775,14 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Created | Result.Updated | Result.NoOp => result.ElasticSuccess(true) + case Result.Created | Result.Updated | Result.NoOp => ElasticSuccess(true) case Result.NotFound => - result.ElasticFailure( - result.ElasticError( + ElasticFailure( + ElasticError( s"Document with id: $id not found in index: $index" ) // if upsert is false ) - case _ => result.ElasticSuccess(false) + case _ => ElasticSuccess(false) } } @@ -648,13 +793,13 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { * [[DeleteApi]] for delete operations */ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion => + _: JavaClientSettingsApi with JavaClientCompanion => override private[client] def executeDelete( index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "delete", index = Some(index), @@ -677,7 +822,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .delete( @@ -689,8 +834,8 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { ) ).map { resp => resp.result() match { - case Result.Deleted | Result.NoOp => result.ElasticSuccess(true) - case _ => result.ElasticSuccess(false) + case Result.Deleted | Result.NoOp => ElasticSuccess(true) + case _ => ElasticSuccess(false) } } @@ -706,7 +851,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "get", index = Some(index), @@ -730,7 +875,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .get( @@ -742,9 +887,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { ) ).map { response => if (response.found()) { - result.ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) + ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) } else { - result.ElasticSuccess(None) + ElasticSuccess(None) } } @@ -757,12 +902,14 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -782,7 +929,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "multiSearch", index = Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")), @@ -801,7 +948,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .search( @@ -812,12 +959,12 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture { val items = elasticQueries.queries.map { q => new RequestItem.Builder() @@ -830,7 +977,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } } @@ -840,7 +987,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * [[BulkApi]] for bulk operations */ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientSettingsApi + with JavaClientIndexApi + with JavaClientCompanion => override type BulkActionType = BulkOperation override type BulkResultType = BulkResponse @@ -1015,14 +1165,15 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { action match { case BulkAction.UPDATE => + val doc: JsonNode = document new BulkOperation.Builder() .update( new UpdateOperation.Builder() .index(bulkItem.index) .id(id.orNull) .action( - new UpdateAction.Builder[JMap[String, Object], JMap[String, Object]]() - .doc(mapper.readValue(document, classOf[JMap[String, Object]])) + new UpdateAction.Builder[JsonNode, JsonNode]() + .doc(doc) .docAsUpsert(true) .build() ) @@ -1039,12 +1190,13 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { .build() case _ => + val doc: JsonNode = document new BulkOperation.Builder() .index( - new IndexOperation.Builder[JMap[String, Object]]() + new IndexOperation.Builder[JsonNode]() .index(bulkItem.index) .id(id.orNull) - .document(mapper.readValue(document, classOf[JMap[String, Object]])) + .document(doc) .build() ) .build() @@ -1089,7 +1241,7 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: VersionApi with SearchApi with JavaClientCompanion => + _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1253,7 +1405,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ) // Parse query to add query clause (not indices, they're in PIT) - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Extract query clause if present if (queryJson.has("query")) { @@ -1500,3 +1652,301 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } } + +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createPipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .putPipeline( + new PutPipelineRequest.Builder() + .id(pipelineName) + .withJson(new StringReader(pipelineDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deletePipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .deletePipeline( + new DeletePipelineRequest.Builder() + .id(pipelineName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getPipeline", + index = None, + retryable = true + )( + apply() + .ingest() + .getPipeline( + new GetPipelineRequest.Builder() + .id(pipelineName) + .build() + ) + ) { resp => + resp.result().asScala.get(pipelineName).map { pipeline => + convertToJson(pipeline) + } + } + } +} + +trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers { + _: JavaClientVersionApi with JavaClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteIndexTemplate( + DeleteIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + ) { resp => + resp.indexTemplates().asScala.headOption.map { template => + convertToJson(template) + } + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name("*") + } + ) + ) { resp => + resp + .indexTemplates() + .asScala + .map { template => + template.name -> convertToJson(template) + } + .toMap + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsIndexTemplate( + ExistsIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.value()) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .putTemplate( + new PutTemplateRequest.Builder() + .name(templateName) + .withJson(new StringReader(templateDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteTemplate( + new DeleteTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name(templateName) + .build() + ) + ) { resp => + resp.result().asScala.get(templateName).map { template => + convertToJson(template) + } + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name("*") + .build() + ) + ) { resp => + resp + .result() + .asScala + .map { case (name, template) => + name -> convertToJson(template) + } + .toMap + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsTemplate( + new ExistsTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.value()) + } + +} diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index e1841ff2..4ecb88b4 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -20,8 +20,6 @@ import app.softnetwork.elastic.client.ElasticClientCompanion import co.elastic.clients.elasticsearch.{ElasticsearchAsyncClient, ElasticsearchClient} import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.ClassTagExtensions import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -38,8 +36,6 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { private val asyncRef = new AtomicReference[Option[ElasticsearchAsyncClient]](None) - lazy val mapper: ObjectMapper with ClassTagExtensions = new ObjectMapper() with ClassTagExtensions - def async(): ElasticsearchAsyncClient = { asyncRef.get() match { case Some(c) => c diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 20729e6b..4295d333 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -16,13 +16,14 @@ package app.softnetwork.elastic.client.java +import app.softnetwork.elastic.client.SerializationApi import co.elastic.clients.json.JsonpSerializable import co.elastic.clients.json.jackson.JacksonJsonpMapper import java.io.{IOException, StringWriter} import scala.util.Try -trait JavaClientConversion { _: JavaClientCompanion => +trait JavaClientConversion { _: JavaClientCompanion with SerializationApi => private[this] val jsonpMapper = new JacksonJsonpMapper(mapper) /** Convert any Elasticsearch response to JSON string */ diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 940aa01e..00e4d600 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -16,12 +16,15 @@ package app.softnetwork.elastic.client.java -import app.softnetwork.elastic.client.ElasticClientHelpers +import app.softnetwork.elastic.client.{ElasticClientHelpers, SerializationApi} import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} import scala.util.{Failure, Success, Try} -trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { +trait JavaClientHelpers + extends ElasticClientHelpers + with JavaClientConversion + with SerializationApi { _: JavaClientCompanion => // ======================================================================== diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala new file mode 100644 index 00000000..d84c3345 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala new file mode 100644 index 00000000..94e8a1d0 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala new file mode 100644 index 00000000..46efe3bd --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala new file mode 100644 index 00000000..3dbe1131 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index a6fc8e2e..563120b2 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -24,10 +24,18 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.sql.bridge._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} -import app.softnetwork.elastic.client -import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.sql.schema.TableAlias +import app.softnetwork.elastic.sql.serialization._ +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ + Conflicts, FieldSort, FieldValue, Refresh, @@ -35,6 +43,7 @@ import co.elastic.clients.elasticsearch._types.{ SortOrder, Time } +import co.elastic.clients.elasticsearch.cat.IndicesRequest import co.elastic.clients.elasticsearch.core.bulk.{ BulkOperation, DeleteOperation, @@ -48,6 +57,12 @@ import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source => ESS import co.elastic.clients.elasticsearch.core.search.{PointInTimeReference, SearchRequestBody} import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} +import co.elastic.clients.elasticsearch.ingest.{ + DeletePipelineRequest, + GetPipelineRequest, + PutPipelineRequest +} +import com.fasterxml.jackson.databind.JsonNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} @@ -75,6 +90,8 @@ trait JavaClientApi with JavaClientScrollApi with JavaClientCompanion with JavaClientVersionApi + with JavaClientPipelineApi + with JavaClientTemplateApi /** Elasticsearch client implementation using the Java Client * @see @@ -82,7 +99,7 @@ trait JavaClientApi */ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { _: SerializationApi with JavaClientCompanion => - override private[client] def executeVersion(): result.ElasticResult[String] = + override private[client] def executeVersion(): ElasticResult[String] = executeJavaAction( operation = "version", index = None, @@ -98,12 +115,19 @@ trait JavaClientVersionApi extends VersionApi with JavaClientHelpers { * @see * [[IndicesApi]] for index management operations */ -trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHelpers { - _: JavaClientCompanion => +trait JavaClientIndicesApi extends IndicesApi with JavaClientHelpers { + _: JavaClientRefreshApi + with JavaClientPipelineApi + with JavaClientScrollApi + with JavaClientBulkApi + with JavaClientTemplateApi + with JavaClientCompanion => override private[client] def executeCreateIndex( index: String, - settings: String - ): result.ElasticResult[Boolean] = + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "createIndex", index = Some(index), @@ -111,15 +135,60 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel )( apply() .indices() - .create( - new CreateIndexRequest.Builder() + .create { + val req = new CreateIndexRequest.Builder() .index(index) .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) - .build() - ) + .aliases( + aliases + .map(alias => { + var builder = + new Alias.Builder().isWriteIndex(alias.isWriteIndex).isHidden(alias.isHidden) + alias.routing.foreach(r => builder = builder.routing(r)) + alias.indexRouting.foreach(ir => builder = builder.indexRouting(ir)) + alias.searchRouting.foreach(sr => builder = builder.searchRouting(sr)) + (alias.alias, builder.build()) + }) + .toMap + .asJava + ) + mappings match { + case None => req.build() + case Some(m) => + req + .mappings( + new TypeMapping.Builder() + .withJson( + new StringReader(m) + ) + .build() + ) + .build() + } + } )(_.acknowledged()) - override private[client] def executeDeleteIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeGetIndex(index: String): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getIndex", + index = Some(index), + retryable = true + )( + apply() + .indices() + .get( + new GetIndexRequest.Builder().index(index).build() + ) + )(response => { + val valueOpt = response.indices().asScala.get(index) + valueOpt match { + case Some(value) => Some(convertToJson(value)) + case None => None + } + }) + } + + override private[client] def executeDeleteIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "deleteIndex", index = Some(index), @@ -130,7 +199,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .delete(new DeleteIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeCloseIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeCloseIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "closeIndex", index = Some(index), @@ -141,7 +210,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel .close(new CloseIndexRequest.Builder().index(index).build()) )(_.acknowledged()) - override private[client] def executeOpenIndex(index: String): result.ElasticResult[Boolean] = + override private[client] def executeOpenIndex(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "openIndex", index = Some(index), @@ -155,8 +224,9 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean - ): result.ElasticResult[(Boolean, Option[Long])] = + refresh: Boolean, + pipeline: Option[String] + ): ElasticResult[(Boolean, Option[Long])] = executeJavaAction( operation = "reindex", index = Some(s"$sourceIndex -> $targetIndex"), @@ -180,7 +250,7 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel (failures.isEmpty, Option(response.total())) } - override private[client] def executeIndexExists(index: String): result.ElasticResult[Boolean] = + override private[client] def executeIndexExists(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "indexExists", index = Some(index), @@ -193,6 +263,75 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel ) )(_.value()) + override private[client] def executeDeleteByQuery( + index: String, + jsonQuery: String, + refresh: Boolean + ): ElasticResult[Long] = + executeJavaAction( + operation = "deleteByQuery", + index = Some(index), + retryable = true + )( + apply().deleteByQuery( + new DeleteByQueryRequest.Builder() + .index(index) + .withJson(new StringReader(jsonQuery)) // JSON query as raw string + .refresh(refresh) + .conflicts(Conflicts.Proceed) + .build() + ) + )(_.deleted()) + + override private[client] def executeIsIndexClosed(index: String): ElasticResult[Boolean] = + executeJavaAction( + operation = "isIndexClosed", + index = Some(index), + retryable = true + )( + apply() + .cat() + .indices( + new IndicesRequest.Builder() + .index(index) + .build() + ) + ) { response => + val entries = response.indices().asScala.headOption + if (entries.isEmpty) false + else { + val entry = entries.head + entry.status() match { + case "close" => true + case _ => false + } + } + } + + override private[client] def executeUpdateByQuery( + index: String, + jsonQuery: String, + pipelineId: Option[String], + refresh: Boolean + ): ElasticResult[Long] = { + + executeJavaAction( + operation = "updateByQuery", + index = Some(index), + retryable = true + )( + apply().updateByQuery( + new UpdateByQueryRequest.Builder() + .index(index) + .refresh(refresh) + .pipeline(pipelineId.orNull) + .withJson(new StringReader(jsonQuery)) + .build() + ) + ) { response => + response.updated() + } + } } /** Elasticsearch client implementation of Alias API using the Java Client @@ -200,34 +339,39 @@ trait JavaClientIndicesApi extends IndicesApi with RefreshApi with JavaClientHel * [[AliasApi]] for alias management operations */ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeAddAlias( - index: String, - alias: String - ): result.ElasticResult[Boolean] = + alias: TableAlias + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "addAlias", - index = Some(index), + index = Some(alias.table), retryable = false - )( + ) { + val node = alias.node + node.put("index", alias.table) + node.put("alias", alias.alias) + + val action = mapper.createObjectNode() + action.set("add", node) + + val addAction = new Action.Builder().withJson(new StringReader(action)) apply() .indices() .updateAliases( new UpdateAliasesRequest.Builder() .actions( - new Action.Builder() - .add(new AddAction.Builder().index(index).alias(alias).build()) - .build() + addAction.build() ) .build() ) - )(_.acknowledged()) + }(_.acknowledged()) override private[client] def executeRemoveAlias( index: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "removeAlias", index = Some(index), @@ -246,7 +390,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeAliasExists(alias: String): result.ElasticResult[Boolean] = + override private[client] def executeAliasExists(alias: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "aliasExists", index = None, @@ -259,7 +403,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { ) )(_.value()) - override private[client] def executeGetAliases(index: String): result.ElasticResult[String] = + override private[client] def executeGetAliases(index: String): ElasticResult[String] = executeJavaAction( operation = "getAliases", index = Some(index), @@ -276,7 +420,7 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { oldIndex: String, newIndex: String, alias: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "swapAlias", index = Some(s"$oldIndex <-> $newIndex"), @@ -307,12 +451,12 @@ trait JavaClientAliasApi extends AliasApi with JavaClientHelpers { * [[SettingsApi]] for settings management operations */ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { - _: IndicesApi with JavaClientCompanion => + _: JavaClientIndicesApi with JavaClientCompanion => override private[client] def executeUpdateSettings( index: String, settings: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "updateSettings", index = Some(index), @@ -328,7 +472,7 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeLoadSettings(index: String): result.ElasticResult[String] = + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = executeJavaAction( operation = "loadSettings", index = Some(index), @@ -348,12 +492,17 @@ trait JavaClientSettingsApi extends SettingsApi with JavaClientHelpers { * [[MappingApi]] for mapping management operations */ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { - _: SettingsApi with IndicesApi with RefreshApi with JavaClientCompanion => + _: JavaClientSettingsApi + with JavaClientIndicesApi + with JavaClientRefreshApi + with JavaClientVersionApi + with JavaClientAliasApi + with JavaClientCompanion => override private[client] def executeSetMapping( index: String, mapping: String - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "setMapping", index = Some(index), @@ -366,7 +515,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { ) )(_.acknowledged()) - override private[client] def executeGetMapping(index: String): result.ElasticResult[String] = + override private[client] def executeGetMapping(index: String): ElasticResult[String] = executeJavaAction( operation = "getMapping", index = Some(index), @@ -396,7 +545,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { getMapping(index).flatMap { jsonString => // βœ… Extracting mapping from JSON ElasticResult.attempt( - new JsonParser().parse(jsonString).getAsJsonObject + JsonParser.parseString(jsonString).getAsJsonObject ) match { case ElasticFailure(error) => logger.error(s"❌ Failed to parse JSON mapping for index '$index': ${error.message}") @@ -418,7 +567,7 @@ trait JavaClientMappingApi extends MappingApi with JavaClientHelpers { trait JavaClientRefreshApi extends RefreshApi with JavaClientHelpers { _: JavaClientCompanion => - override private[client] def executeRefresh(index: String): result.ElasticResult[Boolean] = + override private[client] def executeRefresh(index: String): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "refresh", index = Some(index), @@ -448,7 +597,7 @@ trait JavaClientFlushApi extends FlushApi with JavaClientHelpers { index: String, force: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "flush", index = Some(index), @@ -476,7 +625,7 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCount( query: ElasticQuery - ): result.ElasticResult[Option[Double]] = + ): ElasticResult[Option[Double]] = executeJavaAction( operation = "count", index = Some(query.indices.mkString(",")), @@ -492,14 +641,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { override private[client] def executeCountAsync( query: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[Double]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[Double]]] = fromCompletableFuture( async() .count( new CountRequest.Builder().index(query.indices.asJava).build() ) ).map { response => - result.ElasticSuccess(Option(response.count().toDouble)) + ElasticSuccess(Option(response.count().toDouble)) } } @@ -509,14 +658,14 @@ trait JavaClientCountApi extends CountApi with JavaClientHelpers { * [[IndexApi]] for index operations */ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeIndex( index: String, id: String, source: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "index", index = Some(index), @@ -544,7 +693,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { wait: Boolean )(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .index( @@ -557,10 +706,10 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to index document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to index document with id: $id in index: $index") ) } } @@ -572,7 +721,7 @@ trait JavaClientIndexApi extends IndexApi with JavaClientHelpers { * [[UpdateApi]] for update operations */ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion with SerializationApi => + _: JavaClientSettingsApi with JavaClientCompanion with SerializationApi => override private[client] def executeUpdate( index: String, @@ -580,7 +729,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "update", index = Some(index), @@ -609,7 +758,7 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { source: String, upsert: Boolean, wait: Boolean - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Boolean]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .update( @@ -624,10 +773,10 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to update document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to update document with id: $id in index: $index") ) } } @@ -639,13 +788,13 @@ trait JavaClientUpdateApi extends UpdateApi with JavaClientHelpers { * [[DeleteApi]] for delete operations */ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { - _: SettingsApi with JavaClientCompanion => + _: JavaClientSettingsApi with JavaClientCompanion => override private[client] def executeDelete( index: String, id: String, wait: Boolean - ): result.ElasticResult[Boolean] = + ): ElasticResult[Boolean] = executeJavaBooleanAction( operation = "delete", index = Some(index), @@ -667,7 +816,7 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { override private[client] def executeDeleteAsync(index: String, id: String, wait: Boolean)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Boolean]] = + ): Future[ElasticResult[Boolean]] = fromCompletableFuture( async() .delete( @@ -679,10 +828,10 @@ trait JavaClientDeleteApi extends DeleteApi with JavaClientHelpers { ) ).map { response => if (response.shards().failed().intValue() == 0) { - result.ElasticSuccess(true) + ElasticSuccess(true) } else { - result.ElasticFailure( - client.result.ElasticError(s"Failed to delete document with id: $id in index: $index") + ElasticFailure( + ElasticError(s"Failed to delete document with id: $id in index: $index") ) } } @@ -699,7 +848,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGet( index: String, id: String - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "get", index = Some(index), @@ -723,7 +872,7 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { override private[client] def executeGetAsync(index: String, id: String)(implicit ec: ExecutionContext - ): Future[result.ElasticResult[Option[String]]] = + ): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .get( @@ -735,9 +884,9 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { ) ).map { response => if (response.found()) { - result.ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) + ElasticSuccess(Some(mapper.writeValueAsString(response.source()))) } else { - result.ElasticSuccess(None) + ElasticSuccess(None) } } @@ -750,12 +899,14 @@ trait JavaClientGetApi extends GetApi with JavaClientHelpers { trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { _: JavaClientCompanion with SerializationApi => - override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SingleSearch)(implicit + timestamp: Long + ): String = implicitly[ElasticSearchRequest](sqlSearch).query override private[client] def executeSingleSearch( elasticQuery: ElasticQuery - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "singleSearch", index = Some(elasticQuery.indices.mkString(",")), @@ -775,7 +926,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeMultiSearch( elasticQueries: ElasticQueries - ): result.ElasticResult[Option[String]] = + ): ElasticResult[Option[String]] = executeJavaAction( operation = "multiSearch", index = Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")), @@ -794,7 +945,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { override private[client] def executeSingleSearchAsync( elasticQuery: ElasticQuery - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture( async() .search( @@ -805,12 +956,12 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries - )(implicit ec: ExecutionContext): Future[result.ElasticResult[Option[String]]] = + )(implicit ec: ExecutionContext): Future[ElasticResult[Option[String]]] = fromCompletableFuture { val items = elasticQueries.queries.map { q => new RequestItem.Builder() @@ -823,7 +974,7 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - result.ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))) } } @@ -833,7 +984,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { * [[BulkApi]] for bulk operations */ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { - _: RefreshApi with SettingsApi with IndexApi with JavaClientCompanion => + _: JavaClientRefreshApi + with JavaClientSettingsApi + with JavaClientIndexApi + with JavaClientCompanion => override type BulkActionType = BulkOperation override type BulkResultType = BulkResponse @@ -1008,14 +1162,15 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { action match { case BulkAction.UPDATE => + val doc: JsonNode = document new BulkOperation.Builder() .update( new UpdateOperation.Builder() .index(bulkItem.index) .id(id.orNull) .action( - new UpdateAction.Builder[JMap[String, Object], JMap[String, Object]]() - .doc(mapper.readValue(document, classOf[JMap[String, Object]])) + new UpdateAction.Builder[JsonNode, JsonNode]() + .doc(doc) .docAsUpsert(true) .build() ) @@ -1032,12 +1187,13 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { .build() case _ => + val doc: JsonNode = document new BulkOperation.Builder() .index( - new IndexOperation.Builder[JMap[String, Object]]() + new IndexOperation.Builder[JsonNode]() .index(bulkItem.index) .id(id.orNull) - .document(mapper.readValue(document, classOf[JMap[String, Object]])) + .document(doc) .build() ) .build() @@ -1082,7 +1238,7 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: VersionApi with SearchApi with JavaClientCompanion => + _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1246,7 +1402,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ) // Parse query to add query clause (not indices, they're in PIT) - val queryJson = new JsonParser().parse(elasticQuery.query).getAsJsonObject + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject // Extract query clause if present if (queryJson.has("query")) { @@ -1493,3 +1649,301 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } } } + +trait JavaClientPipelineApi extends PipelineApi with JavaClientHelpers with JavaClientVersionApi { + _: JavaClientCompanion with SerializationApi => + + override private[client] def executeCreatePipeline( + pipelineName: String, + pipelineDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createPipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .putPipeline( + new PutPipelineRequest.Builder() + .id(pipelineName) + .withJson(new StringReader(pipelineDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeletePipeline( + pipelineName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "deletePipeline", + index = None, + retryable = false + )( + apply() + .ingest() + .deletePipeline( + new DeletePipelineRequest.Builder() + .id(pipelineName) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeGetPipeline( + pipelineName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getPipeline", + index = None, + retryable = true + )( + apply() + .ingest() + .getPipeline( + new GetPipelineRequest.Builder() + .id(pipelineName) + .build() + ) + ) { resp => + resp.pipelines().asScala.get(pipelineName).map { pipeline => + convertToJson(pipeline) + } + } + } +} + +trait JavaClientTemplateApi extends TemplateApi with JavaClientHelpers with JavaClientVersionApi { + _: JavaClientCompanion with SerializationApi => + + // ==================== COMPOSABLE TEMPLATES (ES 7.8+) ==================== + + override private[client] def executeCreateComposableTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + retryable = false + )( + apply() + .indices() + .putIndexTemplate( + PutIndexTemplateRequest.of { builder => + builder + .name(templateName) + .withJson(new StringReader(templateDefinition)) + } + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteComposableTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeComposableTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Composable template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteIndexTemplate( + DeleteIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetComposableTemplate( + templateName: String + ): ElasticResult[Option[String]] = + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + ) { resp => + resp.indexTemplates().asScala.headOption.map { template => + convertToJson(template) + } + } + + override private[client] def executeListComposableTemplates() + : ElasticResult[Map[String, String]] = + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getIndexTemplate( + GetIndexTemplateRequest.of { builder => + builder.name("*") + } + ) + ) { resp => + resp + .indexTemplates() + .asScala + .map { template => + template.name -> convertToJson(template) + } + .toMap + } + + override private[client] def executeComposableTemplateExists( + templateName: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsIndexTemplate( + ExistsIndexTemplateRequest.of { builder => + builder.name(templateName) + } + ) + )(resp => resp.value()) + + // ==================== LEGACY TEMPLATES ==================== + + override private[client] def executeCreateLegacyTemplate( + templateName: String, + templateDefinition: String + ): ElasticResult[Boolean] = + executeJavaBooleanAction( + operation = "createTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .putTemplate( + new PutTemplateRequest.Builder() + .name(templateName) + .withJson(new StringReader(templateDefinition)) + .build() + ) + )(resp => resp.acknowledged()) + + override private[client] def executeDeleteLegacyTemplate( + templateName: String, + ifExists: Boolean + ): ElasticResult[Boolean] = { + if (ifExists) { + // Check existence first + executeLegacyTemplateExists(templateName) match { + case ElasticSuccess(exists) => + if (!exists) { + logger.debug(s"Legacy template '$templateName' does not exist, skipping deletion") + return ElasticSuccess(false) + } + case failure @ ElasticFailure(_) => + return failure + } + } + executeJavaBooleanAction( + operation = "deleteTemplate", + index = None, + retryable = false + )( + apply() + .indices() + .deleteTemplate( + new DeleteTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.acknowledged()) + } + + override private[client] def executeGetLegacyTemplate( + templateName: String + ): ElasticResult[Option[String]] = { + executeJavaAction( + operation = "getTemplate", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name(templateName) + .build() + ) + ) { resp => + resp.templates().asScala.get(templateName).map { template => + convertToJson(template) + } + } + } + + override private[client] def executeListLegacyTemplates(): ElasticResult[Map[String, String]] = { + executeJavaAction( + operation = "listTemplates", + index = None, + retryable = true + )( + apply() + .indices() + .getTemplate( + new GetTemplateRequest.Builder() + .name("*") + .build() + ) + ) { resp => + resp + .templates() + .asScala + .map { case (name, template) => + name -> convertToJson(template) + } + .toMap + } + } + + override private[client] def executeLegacyTemplateExists( + templateName: String + ): ElasticResult[Boolean] = { + executeJavaBooleanAction( + operation = "templateExists", + index = None, + retryable = true + )( + apply() + .indices() + .existsTemplate( + new ExistsTemplateRequest.Builder() + .name(templateName) + .build() + ) + )(resp => resp.value()) + } + +} diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index e1841ff2..4ecb88b4 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -20,8 +20,6 @@ import app.softnetwork.elastic.client.ElasticClientCompanion import co.elastic.clients.elasticsearch.{ElasticsearchAsyncClient, ElasticsearchClient} import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.ClassTagExtensions import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder @@ -38,8 +36,6 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { private val asyncRef = new AtomicReference[Option[ElasticsearchAsyncClient]](None) - lazy val mapper: ObjectMapper with ClassTagExtensions = new ObjectMapper() with ClassTagExtensions - def async(): ElasticsearchAsyncClient = { asyncRef.get() match { case Some(c) => c diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala index 20729e6b..4295d333 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientConversion.scala @@ -16,13 +16,14 @@ package app.softnetwork.elastic.client.java +import app.softnetwork.elastic.client.SerializationApi import co.elastic.clients.json.JsonpSerializable import co.elastic.clients.json.jackson.JacksonJsonpMapper import java.io.{IOException, StringWriter} import scala.util.Try -trait JavaClientConversion { _: JavaClientCompanion => +trait JavaClientConversion { _: JavaClientCompanion with SerializationApi => private[this] val jsonpMapper = new JacksonJsonpMapper(mapper) /** Convert any Elasticsearch response to JSON string */ diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 940aa01e..00e4d600 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -16,12 +16,15 @@ package app.softnetwork.elastic.client.java -import app.softnetwork.elastic.client.ElasticClientHelpers +import app.softnetwork.elastic.client.{ElasticClientHelpers, SerializationApi} import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} import scala.util.{Failure, Success, Try} -trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { +trait JavaClientHelpers + extends ElasticClientHelpers + with JavaClientConversion + with SerializationApi { _: JavaClientCompanion => // ======================================================================== diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala new file mode 100644 index 00000000..d84c3345 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientGatewayApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientGatewayApiSpec extends GatewayApiIntegrationSpec with ElasticDockerTestKit { + override lazy val client: GatewayApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala new file mode 100644 index 00000000..94e8a1d0 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientInsertByQuerySpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientInsertByQuerySpec extends InsertByQuerySpec with ElasticDockerTestKit { + override lazy val client: ElasticClientApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala new file mode 100644 index 00000000..46efe3bd --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientPipelineApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientPipelineApiSpec extends PipelineApiSpec with ElasticDockerTestKit { + override lazy val client: PipelineApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala new file mode 100644 index 00000000..3dbe1131 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientTemplateApiSpec.scala @@ -0,0 +1,8 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.spi.JavaClientSpi +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit + +class JavaClientTemplateApiSpec extends TemplateApiSpec with ElasticDockerTestKit { + override lazy val client: TemplateApi with VersionApi = new JavaClientSpi().client(elasticConfig) +} diff --git a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala index 4733479d..7e7bc00b 100644 --- a/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala +++ b/macros-tests/src/test/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidatorSpec.scala @@ -14,7 +14,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Numbers - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Numbers]( "SELECT tiny::TINYINT as tiny, small::SMALLINT as small, normal::INT as normal, big::BIGINT as big, huge::BIGINT as huge, decimal::DOUBLE as decimal, r::REAL as r FROM numbers" @@ -26,7 +26,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Strings - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Strings]( "SELECT vchar::VARCHAR, c::CHAR, text FROM strings" @@ -38,7 +38,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Temporal - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Temporal]( "SELECT d::DATE, t::TIME, dt::DATETIME, ts::TIMESTAMP FROM temporal" @@ -50,7 +50,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, name, price::DOUBLE, stock::INT, active::BOOLEAN, createdAt::DATETIME FROM products" @@ -62,7 +62,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT product_id AS id, product_name AS name, product_price::DOUBLE AS price, product_stock::INT AS stock, is_active::BOOLEAN AS active, created_at::TIMESTAMP AS createdAt FROM products" @@ -74,7 +74,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.ProductWithOptional - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[ProductWithOptional]( "SELECT id, name FROM products" @@ -87,7 +87,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.ProductWithDefaults - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[ProductWithDefaults]( "SELECT id, name FROM products" @@ -100,10 +100,10 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAsUnchecked[Product]( - SQLQuery("SELECT * FROM products") + SelectStatement("SELECT * FROM products") ) """) } @@ -139,7 +139,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, name FROM products" @@ -151,7 +151,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, invalid_name, price, stock, active, createdAt FROM products" @@ -162,7 +162,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { assertDoesNotCompile(""" import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement case class WrongTypes(id: Int, name: Int) @@ -176,7 +176,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT id, nam, price, stock, active, createdAt FROM products" @@ -188,7 +188,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement val dynamicField = "name" TestElasticClientApi.searchAs[Product]( @@ -201,7 +201,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT * FROM products" @@ -214,7 +214,7 @@ class SQLQueryValidatorSpec extends AnyFlatSpec with Matchers { import app.softnetwork.elastic.client.macros.TestElasticClientApi import app.softnetwork.elastic.client.macros.TestElasticClientApi.defaultFormats import app.softnetwork.elastic.sql.macros.SQLQueryValidatorSpec.Product - import app.softnetwork.elastic.sql.query.SQLQuery + import app.softnetwork.elastic.sql.query.SelectStatement TestElasticClientApi.searchAs[Product]( "SELECT * FROM products WHERE active = true" diff --git a/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala b/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala index ecfaed86..a31d591d 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/client/macros/TestElasticClientApi.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.client.macros import app.softnetwork.elastic.sql.macros.SQLQueryMacros -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import org.json4s.{DefaultFormats, Formats} import scala.language.experimental.macros @@ -34,7 +34,7 @@ trait TestElasticClientApi { /** Search without compile-time validation (runtime). */ def searchAsUnchecked[T]( - sqlQuery: SQLQuery + sqlQuery: SelectStatement )(implicit m: Manifest[T], formats: Formats): Seq[T] = { // Dummy implementation for tests Seq.empty[T] diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala index 8371e771..d4ff1eba 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryMacros.scala @@ -44,7 +44,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.searchAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery) + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery) )($m, $formats) """ } @@ -71,7 +71,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.searchAsyncAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery) + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery) )($m, $ec, $formats) """ } @@ -99,7 +99,7 @@ object SQLQueryMacros extends SQLQueryValidator { // 3. Generate the call to searchAsUnchecked q""" ${c.prefix}.scrollAsUnchecked[$tpe]( - _root_.app.softnetwork.elastic.sql.query.SQLQuery($validatedQuery), + _root_.app.softnetwork.elastic.sql.query.SelectStatement($validatedQuery), $config )($system, $m, $formats) """ diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala index 0afd31b1..7d1e445e 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala @@ -19,7 +19,7 @@ package app.softnetwork.elastic.sql.macros import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.{COUNT, WindowFunction} import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.{MultiSearch, SingleSearch} import scala.language.experimental.macros import scala.reflect.macros.blackbox @@ -156,7 +156,7 @@ trait SQLQueryValidator { | scrollAs[Product](\"\"\"SELECT id, name FROM products\"\"\".stripMargin) | |❌ For dynamic queries, use: - | scrollAsUnchecked[Product](SQLQuery(dynamicSql), ScrollConfig()) + | scrollAsUnchecked[Product](SelectStatement(dynamicSql), ScrollConfig()) | |""".stripMargin ) @@ -170,12 +170,12 @@ trait SQLQueryValidator { // ============================================================ // Helper: Parse SQL query into SQLSearchRequest // ============================================================ - private def parseSQLQuery(c: blackbox.Context)(sqlQuery: String): SQLSearchRequest = { + private def parseSQLQuery(c: blackbox.Context)(sqlQuery: String): SingleSearch = { Parser(sqlQuery) match { - case Right(Left(request)) => + case Right(request: SingleSearch) => request - case Right(Right(multi)) => + case Right(multi: MultiSearch) => multi.requests.headOption.getOrElse { c.abort(c.enclosingPosition, "❌ Empty multi-search query") } @@ -193,7 +193,7 @@ trait SQLQueryValidator { // Reject SELECT * (incompatible with compile-time validation) // ============================================================ private def rejectSelectStar[T: c.WeakTypeTag](c: blackbox.Context)( - parsedQuery: SQLSearchRequest, + parsedQuery: SingleSearch, sqlQuery: String ): Unit = { import c.universe._ @@ -237,7 +237,7 @@ trait SQLQueryValidator { | SELECT $fieldNames FROM ... | | 2. Use the *Unchecked() variant for dynamic queries: - | searchAsUnchecked[${tpe.typeSymbol.name}](SQLQuery("SELECT * FROM ...")) + | searchAsUnchecked[${tpe.typeSymbol.name}](SelectStatement("SELECT * FROM ...")) | |Best Practice: | Always explicitly select only the fields you need. @@ -330,7 +330,7 @@ trait SQLQueryValidator { // ============================================================ // Helper: Extract selected fields from parsed SQL query // ============================================================ - private def extractQueryFields(parsedQuery: SQLSearchRequest): Set[String] = { + private def extractQueryFields(parsedQuery: SingleSearch): Set[String] = { parsedQuery.select.fields.map { field => field.fieldAlias.map(_.alias).getOrElse(field.identifier.name) }.toSet @@ -339,7 +339,7 @@ trait SQLQueryValidator { // ============================================================ // Helper: Extract UNNEST collections from the query // ============================================================ - private def extractUnnestedCollections(parsedQuery: SQLSearchRequest): Set[String] = { + private def extractUnnestedCollections(parsedQuery: SingleSearch): Set[String] = { // Check if the query has nested elements (UNNEST) parsedQuery.select.fields.flatMap { field => field.identifier.nestedElement.map { nested => @@ -630,7 +630,7 @@ trait SQLQueryValidator { // Helper: Validate Type compatibility // ============================================================ private def validateTypes(c: blackbox.Context)( - parsedQuery: SQLSearchRequest, + parsedQuery: SingleSearch, requiredFields: Map[String, c.universe.Type] ): Unit = { diff --git a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala index 533cfb84..eef61be8 100644 --- a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala +++ b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala @@ -19,14 +19,14 @@ package app.softnetwork.elastic.persistence.query import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} import app.softnetwork.elastic.client.spi.ElasticClientFactory import app.softnetwork.elastic.client.{ElasticClientApi, ElasticClientDelegator} -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.sql.query.SelectStatement import mustache.Mustache import org.json4s.Formats import app.softnetwork.persistence._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.persistence.query.ExternalPersistenceProvider import app.softnetwork.serialization.commonFormats -import app.softnetwork.elastic.persistence.typed.Elastic._ +import app.softnetwork.elastic.persistence.typed.Elastic import org.slf4j.Logger import scala.reflect.ClassTag @@ -45,11 +45,11 @@ trait ElasticProvider[T <: Timestamped] implicit def formats: Formats = commonFormats - protected lazy val index: String = getIndex[T](manifestWrapper.wrapped) + protected lazy val index: String = Elastic.getIndex[T](manifestWrapper.wrapped) protected lazy val _type: String = getType[T](manifestWrapper.wrapped) - protected lazy val alias: String = getAlias[T](manifestWrapper.wrapped) + protected lazy val alias: String = Elastic.getAlias[T](manifestWrapper.wrapped) protected def mappingPath: Option[String] = None @@ -188,7 +188,7 @@ trait ElasticProvider[T <: Timestamped] override def searchDocuments( query: String )(implicit m: Manifest[T], formats: Formats): List[T] = { - searchAsUnchecked[T](SQLQuery(query)) match { + searchAsUnchecked[T](SelectStatement(query)) match { case ElasticSuccess(results) => results.toList case ElasticFailure(elasticError) => logger.error(s"searchDocuments failed -> ${elasticError.message}") diff --git a/sql/build.sbt b/sql/build.sbt index eb0e89c6..cc74c484 100644 --- a/sql/build.sbt +++ b/sql/build.sbt @@ -7,18 +7,23 @@ elasticSearchVersion := Versions.es9 name := s"softclient4es-sql" +val typesafeConfig = Seq( + "com.typesafe" % "config" % Versions.typesafeConfig, + "com.github.kxbmap" %% "configs" % Versions.kxbmap +) + val scalatest = Seq( "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) libraryDependencies ++= jacksonDependencies(elasticSearchVersion.value) ++ // elastic4sDependencies(elasticSearchVersion.value) ++ + typesafeConfig ++ scalatest ++ Seq( "javax.activation" % "activation" % "1.1.1" % Test ) :+ + "com.typesafe.scala-logging" %% "scala-logging" % Versions.scalaLogging :+ // ("app.softnetwork.persistence" %% "persistence-core" % Versions.genericPersistence excludeAll(jacksonExclusions: _*)) :+ "org.scala-lang" % "scala-reflect" % scalaVersion.value :+ "com.google.code.gson" % "gson" % Versions.gson % Test - - diff --git a/sql/src/main/resources/softnetwork-sql.conf b/sql/src/main/resources/softnetwork-sql.conf new file mode 100644 index 00000000..74fdf8e4 --- /dev/null +++ b/sql/src/main/resources/softnetwork-sql.conf @@ -0,0 +1,4 @@ +sql { + composite-key-separator = "\\|\\|" # regex separator for composite keys in SQL ddl queries + artificial-primary-key-column-name = "id" # name of the artificial primary key column added to tables without primary key(s) defined +} diff --git a/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala new file mode 100644 index 00000000..75c56386 --- /dev/null +++ b/sql/src/main/scala-2.12/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -0,0 +1,25 @@ +package app.softnetwork.elastic.sql.config + +import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.scalalogging.StrictLogging +import configs.Configs + +case class ElasticSqlConfig( + compositeKeySeparator: String, + artificialPrimaryKeyColumnName: String +) + +object ElasticSqlConfig extends StrictLogging { + def apply(config: Config): ElasticSqlConfig = { + Configs[ElasticSqlConfig] + .get(config.withFallback(ConfigFactory.load("softnetwork-sql.conf")), "sql") + .toEither match { + case Left(configError) => + logger.error(s"Something went wrong with the provided arguments $configError") + throw configError.configException + case Right(r) => r + } + } + + def apply(): ElasticSqlConfig = apply(com.typesafe.config.ConfigFactory.load()) +} diff --git a/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala new file mode 100644 index 00000000..c777f29d --- /dev/null +++ b/sql/src/main/scala-2.13/app/softnetwork/elastic/sql/config/ElasticSqlConfig.scala @@ -0,0 +1,41 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.config + +import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.scalalogging.StrictLogging +import configs.ConfigReader + +case class ElasticSqlConfig( + compositeKeySeparator: String, + artificialPrimaryKeyColumnName: String +) + +object ElasticSqlConfig extends StrictLogging { + def apply(config: Config): ElasticSqlConfig = { + ConfigReader[ElasticSqlConfig] + .read(config.withFallback(ConfigFactory.load("softnetwork-sql.conf")), "sql") + .toEither match { + case Left(configError) => + logger.error(s"Something went wrong with the provided arguments $configError") + throw configError.configException + case Right(r) => r + } + } + + def apply(): ElasticSqlConfig = apply(com.typesafe.config.ConfigFactory.load()) +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala new file mode 100644 index 00000000..897b8392 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -0,0 +1,507 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic + +import app.softnetwork.elastic.sql.{BooleanValue, ObjectValue, StringValue, StringValues, Value} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.schema.{ + Column, + DateIndexNameProcessor, + IngestPipelineType, + IngestProcessor, + PartitionDate, + PrimaryKeyProcessor, + Schema, + ScriptProcessor, + SetProcessor, + Table +} +import app.softnetwork.elastic.sql.serialization._ +import app.softnetwork.elastic.sql.time.TimeUnit +import com.fasterxml.jackson.databind.JsonNode + +import scala.jdk.CollectionConverters._ + +package object schema { + + final case class IndexField( + name: String, + `type`: String, + script: Option[ScriptProcessor] = None, + null_value: Option[Value[_]] = None, + not_null: Option[Boolean] = None, + comment: Option[String] = None, + fields: List[IndexField] = Nil, + options: Map[String, Value[_]] = Map.empty + ) { + lazy val ddlColumn: Column = { + Column( + name = name, + dataType = SQLTypes(this), + script = script, + multiFields = fields.map(_.ddlColumn), + defaultValue = null_value, + notNull = not_null.getOrElse(false), + comment = comment, + options = options + ) + } + } + + object IndexField { + def apply(name: String, node: JsonNode, _meta: Option[ObjectValue]): IndexField = { + val tpe = _meta + .flatMap { + case m: ObjectValue => + m.value.get("data_type") match { + case Some(v: StringValue) => Some(v.value) + case _ => None + } + case _ => None + } + .getOrElse(Option(node.get("type")).map(_.asText()).getOrElse("object")) + + val nullValue = + Option(node.get("null_value")) + .flatMap(Value(_)) + .map(Value(_)) + .orElse(_meta.flatMap { + case m: ObjectValue => + m.value.get("default_value") match { + case Some(v: Value[_]) => Some(v) + case _ => None + } + case _ => None + }) + + val fields_meta = _meta.flatMap { + case m: ObjectValue => + m.value.get("multi_fields") match { + case Some(f: ObjectValue) => Some(f) + case _ => None + } + case _ => None + } + + val fields = + Option(node.get("fields")) // multi-fields + .orElse(Option(node.get("properties"))) // object/nested fields + .map(_.properties().asScala.map { entry => + val name = entry.getKey + val value = entry.getValue + val _meta = fields_meta.flatMap { + case m: ObjectValue => + m.value.get(name) match { + case Some(colMeta: ObjectValue) => Some(colMeta) + case _ => None + } + case _ => None + } + apply(name, value, _meta) + }.toList) + .getOrElse(Nil) + + val options = + extractObject(node, ignoredKeys = Set("type", "null_value", "fields", "properties")) + + val comment = _meta.flatMap { + case m: ObjectValue => + m.value.get("comment") match { + case Some(c: StringValue) => Some(c.value) + case _ => None + } + case _ => None + } + + val notNull = _meta.flatMap { + case m: ObjectValue => + m.value.get("not_null") match { + case Some(c: BooleanValue) => Some(c.value) + case Some(c: StringValue) => Some(c.value.toBoolean) + case _ => None + } + case _ => None + } + + val script = _meta.flatMap { + case m: ObjectValue => + m.value.get("script") match { + case Some(st: ObjectValue) => + val map = st.value + map.get("sql") match { + case Some(script: StringValue) => + map.get("painless") match { + case Some(source: StringValue) => + Some( + ScriptProcessor( + script = script.value, + column = name, + dataType = SQLTypes(tpe), + source = source.value + ) + ) + } + case _ => None + } + case _ => None + } + case _ => None + } + IndexField( + name = name, + `type` = tpe, + script = script, + null_value = nullValue, + not_null = notNull, + comment = comment, + fields = fields, + options = options + ) + + } + } + + final case class IndexMappings( + fields: List[IndexField] = Nil, + primaryKey: List[String] = Nil, + partitionBy: Option[IndexDatePartition] = None, + options: Map[String, Value[_]] = Map.empty + ) + + object IndexMappings { + def apply(root: JsonNode): IndexMappings = { + if (root.has("mappings")) { + val mappingsNode = root.path("mappings") + return apply(mappingsNode) + } + if (root.has("_doc")) { + val docNode = root.path("_doc") + return apply(docNode) + } + val options = extractObject(root, ignoredKeys = Set("properties")) + val meta = options.get("_meta") + val columns = meta.flatMap { + case m: ObjectValue => + m.value.get("columns") match { + case Some(cols: ObjectValue) => Some(cols) + case _ => None + } + case _ => None + } + val fields = Option(root.get("properties")) + .map(_.properties().asScala.map { entry => + val name = entry.getKey + val value = entry.getValue + val _meta = columns.flatMap { + case c: ObjectValue => + c.value.get(name) match { + case Some(colMeta: ObjectValue) => Some(colMeta) + case _ => None + } + case _ => None + } + IndexField(name, value, _meta) + }.toList) + .getOrElse(Nil) + + val primaryKey: List[String] = meta + .map { + case m: ObjectValue => + m.value.get("primary_key") match { + case Some(pk: StringValues) => pk.values.map(_.ddl.replaceAll("\"", "")).toList + case Some(pk: StringValue) => List(pk.ddl.replaceAll("\"", "")) + case _ => List.empty + } + case _ => List.empty + } + .getOrElse(List.empty) + + val partitionBy: Option[IndexDatePartition] = meta.flatMap { + case m: ObjectValue => + m.value.get("partition_by") match { + case Some(pb: ObjectValue) => + pb.value.get("column") match { + case Some(column: StringValue) => // valid + pb.value.get("granularity") match { + case Some(granularity: StringValue) => + Some(IndexDatePartition(column.value, granularity.value)) + case _ => Some(IndexDatePartition(column.value, "d")) + } + case _ => None + } + case _ => None + } + case _ => None + } + + IndexMappings( + fields = fields, + primaryKey = primaryKey, + partitionBy = partitionBy, + options = options + ) + } + + } + + final case class IndexDatePartition( + column: String, + granularity: String // "d", "M", "y", etc. + ) + + final case class IndexSettings( + options: Map[String, Value[_]] = Map.empty + ) + + object IndexSettings { + def apply(settings: JsonNode): IndexSettings = { + if (settings.has("settings")) { + val settingsNode = settings.path("settings") + return apply(settingsNode) + } + val index = settings.path("index") + + val options = extractObject(index) + + IndexSettings( + options = options + ) + } + } + + final case class IndexIngestProcessor( + pipelineType: IngestPipelineType, + processor: JsonNode + ) { + lazy val ddlProcesor: IngestProcessor = IngestProcessor(pipelineType, processor) + } + + final case class IndexIngestPipeline( + pipelineType: IngestPipelineType, + pipeline: JsonNode + ) { + lazy val processors: Seq[IngestProcessor] = { + val processorsNode = pipeline.get("processors") + if (processorsNode != null && processorsNode.isArray) { + processorsNode + .elements() + .asScala + .toSeq + .map(IndexIngestProcessor(pipelineType, _).ddlProcesor) + } else { + Seq.empty + } + } + } + + final case class IndexAlias( + name: String, + filter: Map[String, Any] = Map.empty, + routing: Option[String] = None, + indexRouting: Option[String] = None, + searchRouting: Option[String] = None, + isWriteIndex: Boolean = false, + isHidden: Boolean = false, + node: JsonNode + ) + + object IndexAlias { + def apply(name: String, node: JsonNode): IndexAlias = { + if (node.has(name)) { + val aliasNode = node.path(name) + return apply(name, aliasNode) + } + val filter: Map[String, Any] = + if (node.has("filter")) node.path("filter") + else Map.empty + val routing = + Option(node.get("routing")).map(_.asText()) + val indexRouting = + Option(node.get("index_routing")).map(_.asText()) + val searchRouting = + Option(node.get("search_routing")).map(_.asText()) + val isWriteIndex = + Option(node.get("is_write_index")).exists(_.asBoolean()) + val isHidden = + Option(node.get("is_hidden")).exists(_.asBoolean()) + IndexAlias( + name = name, + filter = filter, + routing = routing, + indexRouting = indexRouting, + searchRouting = searchRouting, + isWriteIndex = isWriteIndex, + isHidden = isHidden, + node = node + ) + } + } + + final case class IndexAliases( + aliases: Map[String, IndexAlias] = Map.empty + ) + + object IndexAliases { + def apply(nodes: Seq[(String, JsonNode)]): IndexAliases = { + IndexAliases(nodes.map(entry => entry._1 -> IndexAlias(entry._1, entry._2)).toMap) + } + } + + final case class Index( + name: String, + mappings: JsonNode, + settings: JsonNode, + aliases: Map[String, JsonNode] = Map.empty, + defaultPipeline: Option[JsonNode] = None, + finalPipeline: Option[JsonNode] = None + ) { + + lazy val defaultIngestPipelineName: Option[String] = esSettings.options.get("index") match { + case Some(obj: ObjectValue) => + obj.value.get("default_pipeline") match { + case Some(s: StringValue) => Some(s.value) + case _ => None + } + case _ => None + } + + lazy val finalIngestPipelineName: Option[String] = esSettings.options.get("index") match { + case Some(obj: ObjectValue) => + obj.value.get("final_pipeline") match { + case Some(s: StringValue) => Some(s.value) + case _ => None + } + case _ => None + } + + lazy val esMappings: IndexMappings = IndexMappings(mappings) + + lazy val esSettings: IndexSettings = IndexSettings(settings) + + lazy val esAliases: IndexAliases = IndexAliases(aliases.toSeq) + + lazy val esDefaultPipeline: Option[IndexIngestPipeline] = + defaultPipeline.map(IndexIngestPipeline(IngestPipelineType.Default, _)) + + lazy val esFinalPipeline: Option[IndexIngestPipeline] = + finalPipeline.map(IndexIngestPipeline(IngestPipelineType.Final, _)) + + lazy val esProcessors: Seq[IngestProcessor] = { + val defaultProcessors = esDefaultPipeline.map(_.processors).getOrElse(Seq.empty) + val finalProcessors = esFinalPipeline.map(_.processors).getOrElse(Seq.empty) + defaultProcessors ++ finalProcessors + } + + lazy val schema: Schema = { + // 1. Columns from the mapping + val initialCols: Map[String, Column] = + esMappings.fields.map { field => + val name = field.name + name -> field.ddlColumn + }.toMap + + // 2. PK + partition + pipelines from index mappings and settings + var primaryKey: List[String] = esMappings.primaryKey + var partitionBy: Option[PartitionDate] = esMappings.partitionBy.map { p => + val granularity = TimeUnit(p.granularity) + PartitionDate(p.column, granularity) + } + + // 3. Enrichment from the pipeline (if provided) + val enrichedCols = scala.collection.mutable.Map(initialCols.toSeq: _*) + + var processors: collection.mutable.Seq[IngestProcessor] = collection.mutable.Seq.empty + + esProcessors.foreach { + case p: ScriptProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(script = Some(p))) + } + + case p: SetProcessor => + val col = p.column + enrichedCols.get(col).foreach { c => + enrichedCols.update(col, c.copy(defaultValue = Some(p.value))) + } + + case p: DateIndexNameProcessor => + if (partitionBy.isEmpty) { + val granularity = TimeUnit(p.dateRounding) + partitionBy = Some(PartitionDate(p.column, granularity)) + } + + case p: PrimaryKeyProcessor => + if (primaryKey.isEmpty) { + primaryKey = p.value.toList + } + + case p: IngestProcessor => processors = processors :+ p + + } + + // 4. Final construction of the Table + Table( + name = name, + columns = enrichedCols.values.toList.sortBy(_.name), + primaryKey = primaryKey, + partitionBy = partitionBy, + mappings = esMappings.options, + settings = esSettings.options, + processors = processors.toSeq, + aliases = esAliases.aliases.map(entry => entry._1 -> entry._2.node) + ).update() + } + } + + object Index { + def apply(name: String, json: String): Index = { + val root: JsonNode = json + apply(name, root) + } + + def apply(name: String, root: JsonNode): Index = { + if (root.has(name)) { + val indexNode = root.path(name) + return apply(name, indexNode) + } + val mappings = root.path("mappings") + val settings = root.path("settings") + val aliasesNode = root.path("aliases") + val aliases: Map[String, JsonNode] = + if (aliasesNode != null && aliasesNode.isObject) { + aliasesNode + .properties() + .asScala + .map { entry => + val aliasName = entry.getKey + val aliasValue = entry.getValue + aliasName -> aliasValue + } + .toMap + } else { + Map.empty + } + Index( + name = name, + mappings = mappings, + settings = settings, + aliases = aliases + ) + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index 924270c2..726ff774 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -17,7 +17,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.parser.Parser -import app.softnetwork.elastic.sql.query.{Criteria, SQLMultiSearchRequest, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{Criteria, MultiSearch, SingleSearch, Statement} import scala.util.matching.Regex @@ -27,16 +27,21 @@ object SQLImplicits { import scala.language.implicitConversions implicit def queryToSQLCriteria(query: String): Option[Criteria] = { - val maybeQuery: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = query + val maybeQuery: Option[Statement] = query maybeQuery match { - case Some(Left(l)) => l.where.flatMap(_.criteria) - case _ => None + case Some(statement) => + statement match { + case single: SingleSearch => + single.where.flatMap(_.criteria) + case _ => None + } + case _ => None } } - implicit def queryToSQLQuery( + implicit def queryToStatement( query: String - ): Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + ): Option[Statement] = { Parser(query) match { case Left(_) => None case Right(r) => Some(r) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index 1e853ae5..16bf57b1 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -16,14 +16,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.query.{ - Bucket, - BucketPath, - Field, - Limit, - OrderBy, - SQLSearchRequest -} +import app.softnetwork.elastic.sql.query.{Bucket, BucketPath, Field, Limit, OrderBy, SingleSearch} import app.softnetwork.elastic.sql.{Expr, Identifier, TokenRegex, Updateable} package object aggregate { @@ -94,7 +87,7 @@ package object aggregate { override lazy val bucketPath: String = aggregations.map(_.bucketPath).distinct.sortBy(_.length).reverse.headOption.getOrElse("") - override def update(request: SQLSearchRequest): BucketScriptAggregation = { + override def update(request: SingleSearch): BucketScriptAggregation = { val identifiers = FunctionUtils.aggregateIdentifiers(identifier) val params = identifiers.flatMap { case identifier: Identifier => @@ -120,7 +113,7 @@ package object aggregate { def window: Window def limit: Option[Limit] - override def isWindowing: Boolean = buckets.nonEmpty + override def isWindowing: Boolean = buckets.nonEmpty || orderBy.isDefined lazy val buckets: Seq[Bucket] = partitionBy.map(identifier => Bucket(identifier, None)) @@ -148,7 +141,7 @@ package object aggregate { def withFields(fields: Seq[Field]): WindowFunction - def update(request: SQLSearchRequest): WindowFunction = { + def update(request: SingleSearch): WindowFunction = { val updated = this .withPartitionBy(partitionBy = partitionBy.map(_.update(request))) updated.withFields( @@ -177,7 +170,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[FirstValue] .copy( @@ -197,7 +190,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[LastValue] .copy( @@ -217,7 +210,7 @@ package object aggregate { override def withPartitionBy(partitionBy: Seq[Identifier]): WindowFunction = this.copy(partitionBy = partitionBy) override def withFields(fields: Seq[Field]): WindowFunction = this - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[ArrayAgg] .copy( @@ -244,7 +237,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[CountAgg] .copy( @@ -268,7 +261,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[MinAgg] .copy( @@ -292,7 +285,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[MaxAgg] .copy( @@ -316,7 +309,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[AvgAgg] .copy( @@ -340,7 +333,7 @@ package object aggregate { override def withFields(fields: Seq[Field]): WindowFunction = this.copy(fields = fields) - override def update(request: SQLSearchRequest): WindowFunction = super + override def update(request: SingleSearch): WindowFunction = super .update(request) .asInstanceOf[SumAgg] .copy( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala index 6f450160..c914deb7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/geo/package.scala @@ -29,7 +29,7 @@ import app.softnetwork.elastic.sql.{ Updateable } import app.softnetwork.elastic.sql.operator.Operator -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.SingleSearch package object geo { @@ -97,7 +97,7 @@ package object geo { with PainlessParams with Updateable { - override def update(request: SQLSearchRequest): Distance = this.copy( + override def update(request: SingleSearch): Distance = this.copy( from = from.fold(id => Left(id.update(request)), p => Right(p)), to = to.fold(id => Left(id.update(request)), p => Right(p)) ) @@ -171,7 +171,7 @@ package object geo { } if (identifiers.nonEmpty) - s"($assignments ($nullCheck) ? null : $ret)" + s"$assignments ($nullCheck) ? null : $ret" else ret } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 79752d8a..adcc4fae 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -18,7 +18,6 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ Expr, - Identifier, IntValue, PainlessContext, PainlessParam, @@ -69,9 +68,7 @@ package object math { case object Atan extends Expr("ATAN") with Trigonometric case object Atan2 extends Expr("ATAN2") with Trigonometric - sealed trait MathematicalFunction - extends TransformFunction[SQLNumeric, SQLNumeric] - with FunctionWithIdentifier { + sealed trait MathematicalFunction extends TransformFunction[SQLNumeric, SQLNumeric] { override def inputType: SQLNumeric = SQLTypes.Numeric override def outputType: SQLNumeric = mathOp.baseType @@ -80,8 +77,13 @@ package object math { override def fun: Option[PainlessScript] = Some(mathOp) - override def identifier: Identifier = Identifier(this) - + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + val ret = super.toPainlessCall(callArgs, context) + s"Double.valueOf($ret)" + } } case class MathematicalFunctionWithOp( @@ -120,7 +122,7 @@ package object math { override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = callArgs match { - case List(a, p) => s"${mathOp.painless(context)}(($a * $p) / $p)" + case List(a, p) => s"Long.valueOf(${mathOp.painless(context)}(($a * $p) / $p))" case _ => throw new IllegalArgumentException("Round function requires exactly one argument") } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index 3667718b..064e6b25 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -20,7 +20,7 @@ import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator -import app.softnetwork.elastic.sql.query.SQLSearchRequest +import app.softnetwork.elastic.sql.query.{NestedElement, SingleSearch} package object function { @@ -33,11 +33,15 @@ package object function { } def expr: Token = _expr override def nullable: Boolean = expr.nullable + def functionNestedElement: Option[NestedElement] = None } trait FunctionWithIdentifier extends Function { def identifier: Identifier + override def functionNestedElement: Option[NestedElement] = + identifier.nestedElement.orElse(identifier.functionNestedElement) + override def shouldBeScripted: Boolean = identifier.shouldBeScripted } @@ -189,7 +193,7 @@ package object function { functions.indexOf(function) } - def updateFunctions(request: SQLSearchRequest): List[Function] = { + def updateFunctions(request: SingleSearch): List[Function] = { functions.map { case f: Updateable => f.update(request).asInstanceOf[Function] @@ -199,6 +203,8 @@ package object function { override def shouldBeScripted: Boolean = functions.exists(_.shouldBeScripted) + override def functionNestedElement: Option[NestedElement] = + functions.flatMap(_.functionNestedElement).headOption } trait FunctionN[In <: SQLType, Out <: SQLType] extends Function with PainlessScript { @@ -268,7 +274,7 @@ package object function { val ret = SQLTypeUtils .coerce( a, - argTypes(i), + in, context ) if (ret.startsWith(".")) { @@ -285,18 +291,36 @@ package object function { ctx.addParam(LiteralParam(ret)) } case identifier: Identifier => - identifier.baseType match { - case SQLTypes.Any => // in painless context, Any is ZonedDateTime - out match { + identifier.originalType match { + case SQLTypes.Any if !ctx.isProcessor => + in match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + identifier.addPainlessMethod(".toInstant().atZone(ZoneId.of('Z'))") case SQLTypes.Date => - identifier.addPainlessMethod(".toLocalDate()") + identifier.addPainlessMethod( + ".toInstant().atZone(ZoneId.of('Z')).toLocalDate()" + ) case SQLTypes.Time => - identifier.addPainlessMethod(".toLocalTime()") + identifier.addPainlessMethod( + ".toInstant().atZone(ZoneId.of('Z')).toLocalTime()" + ) case _ => } - case _ => + Option(paramName) + case SQLTypes.Any if ctx.isProcessor => + in match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + val param = SQLTypeUtils + .coerce( + a, + in, + context + ) + ctx.addParam(LiteralParam(param)) + case _ => Option(paramName) + } + case _ => Option(paramName) } - Option(paramName) case _ => Option(paramName) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala index 4629692c..bbd98d01 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/string/package.scala @@ -16,14 +16,7 @@ package app.softnetwork.elastic.sql.function -import app.softnetwork.elastic.sql.{ - Expr, - Identifier, - IntValue, - PainlessContext, - PainlessScript, - TokenRegex -} +import app.softnetwork.elastic.sql.{Expr, IntValue, PainlessContext, PainlessScript, TokenRegex} import app.softnetwork.elastic.sql.`type`.{ SQLBigInt, SQLBool, @@ -108,17 +101,13 @@ package object string { override def nullable: Boolean = false } - sealed trait StringFunction[Out <: SQLType] - extends TransformFunction[SQLVarchar, Out] - with FunctionWithIdentifier { + sealed trait StringFunction[Out <: SQLType] extends TransformFunction[SQLVarchar, Out] { override def inputType: SQLVarchar = SQLTypes.Varchar override def outputType: Out def stringOp: StringOp - override def identifier: Identifier = Identifier(this) - override def toSQL(base: String): String = if (base.nonEmpty) s"$sql($base)" else sql @@ -163,11 +152,11 @@ package object string { callArgs match { // SUBSTRING(expr, start, length) case List(arg0, _, _) => - s"$arg0.substring(${start - 1}, Math.min(${start - 1 + length.get}, $arg0.length()))" + s"((String)$arg0).substring(${start - 1}, (int)Math.min(${start - 1 + length.get}, ((String)$arg0).length()))" // SUBSTRING(expr, start) case List(arg0, arg1) => - s"$arg0.substring(Math.min(${start - 1}, $arg0.length() - 1))" + s"((String)$arg0).substring((int)Math.min(${start - 1}, ((String)$arg0).length() - 1))" case _ => throw new IllegalArgumentException("SUBSTRING requires 2 or 3 arguments") } @@ -244,7 +233,7 @@ package object string { ): String = { callArgs match { case List(arg0, arg1) => - s"$arg0.substring(0, Math.min($arg1, $arg0.length()))" + s"((String)$arg0).substring(0, (int)Math.min($arg1, ((String)$arg0).length()))" case _ => throw new IllegalArgumentException("LEFT requires 2 arguments") } } @@ -273,7 +262,8 @@ package object string { callArgs match { case List(arg0, arg1) => if (length == 0) "" - else s"""$arg0.substring($arg0.length() - Math.min($arg1, $arg0.length()))""" + else + s"""((String)$arg0).substring(((String)$arg0).length() - (int)Math.min($arg1, ((String)$arg0).length()))""" case _ => throw new IllegalArgumentException("RIGHT requires 2 arguments") } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 55a9bd44..1856002f 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -38,6 +38,7 @@ import app.softnetwork.elastic.sql.`type`.{ SQLTypes, SQLVarchar } +import app.softnetwork.elastic.sql.function.time.CurrentFunction.queryTimestamp import app.softnetwork.elastic.sql.time.{IsoField, TimeField, TimeInterval, TimeUnit} package object time { @@ -116,7 +117,7 @@ package object time { } sealed trait DateTimeFunction extends Function { - def now: String = "ZonedDateTime.now(ZoneId.of('Z'))" + def now: String = "ZonedDateTime.ofInstant(Instant.ofEpochMilli({{__now__}}), ZoneId.of('Z'))" override def baseType: SQLType = SQLTypes.DateTime } @@ -140,17 +141,28 @@ package object time { override def painless(context: Option[PainlessContext]): String = { context match { case Some(ctx) => - ctx.addParam(LiteralParam(param)) match { + ctx.addParam(LiteralParam(param.replaceAll("\\{\\{__now__}}", ctx.timestamp))) match { case Some(p) => return SQLTypeUtils.coerce(p, this.baseType, this.out, nullable = false, context) case _ => } case _ => } - SQLTypeUtils.coerce(param, this.baseType, this.out, nullable = false, context) + SQLTypeUtils.coerce( + param.replaceAll("\\{\\{__now__}}", queryTimestamp), + this.baseType, + this.out, + nullable = false, + context + ) } } + object CurrentFunction { + val processorTimestamp: String = "ctx['_ingest']['timestamp']" + val queryTimestamp: String = "params.__now__" + } + sealed trait CurrentDateTimeFunction extends DateTimeFunction with CurrentFunction { override def param: String = now } @@ -388,7 +400,7 @@ package object time { override lazy val words: List[String] = List(sql, "DATEDIFF") } - case class DateDiff(end: PainlessScript, start: PainlessScript, unit: TimeUnit) + case class DateDiff(start: PainlessScript, end: PainlessScript, unit: TimeUnit) extends DateTimeFunction with BinaryFunction[SQLDateTime, SQLDateTime, SQLNumeric] with PainlessScript { @@ -402,10 +414,27 @@ package object time { override def sql: String = DateDiff.sql - override def toSQL(base: String): String = s"$sql(${end.sql}, ${start.sql}, ${unit.sql})" + override def toSQL(base: String): String = s"$sql(${start.sql}, ${end.sql}, ${unit.sql})" - override def toPainlessCall(callArgs: List[String], context: Option[PainlessContext]): String = - s"${unit.painless(context)}${DateDiff.painless(context)}(${callArgs.mkString(", ")})" + override def in: SQLType = SQLTypes.Date + + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + val ret = + s"Long.valueOf(${unit.painless(context)}${DateDiff.painless(context)}(${callArgs.mkString(", ")}))" + context match { + case Some(ctx) + if ctx.isProcessor => // to fix bug in painless script processor context with elasticsearch v6 + ctx.addParam(LiteralParam(ret)) match { + case Some(p) => return p + case _ => + } + case _ => + } + ret + } } case object DateAdd extends Expr("DATE_ADD") with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 24e199c9..c953dcf0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -20,6 +20,7 @@ import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} import app.softnetwork.elastic.sql.parser.Validator +import app.softnetwork.elastic.sql.query.NestedElement case class ArithmeticExpression( left: PainlessScript, @@ -27,7 +28,8 @@ case class ArithmeticExpression( right: PainlessScript, group: Boolean = false ) extends TransformFunction[SQLNumeric, SQLNumeric] - with BinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] { + with BinaryFunction[SQLNumeric, SQLNumeric, SQLNumeric] + with Updateable { override def fun: Option[ArithmeticOperator] = Some(operator) @@ -129,4 +131,35 @@ case class ArithmeticExpression( expr } + override def update(request: query.SingleSearch): ArithmeticExpression = { + (left, right) match { + case (l: Updateable, r: Updateable) => + this.copy( + left = l.update(request).asInstanceOf[PainlessScript], + right = r.update(request).asInstanceOf[PainlessScript] + ) + case (l: Updateable, _) => + this.copy( + left = l.update(request).asInstanceOf[PainlessScript] + ) + case (_, r: Updateable) => + this.copy( + right = r.update(request).asInstanceOf[PainlessScript] + ) + case _ => + this + } + } + + override def functionNestedElement: Option[NestedElement] = + (left, right) match { + case (l: Identifier, r: Identifier) => + l.nestedElement.orElse(r.nestedElement) + case (l: Identifier, _) => + l.nestedElement + case (_, r: Identifier) => + r.nestedElement + case _ => + None + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala index b698a751..5569800c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/package.scala @@ -74,7 +74,7 @@ package object operator { case object AND extends Expr("AND") with PredicateOperator case object OR extends Expr("OR") with PredicateOperator - case object UNION extends Expr("UNION") with Operator with TokenRegex + case object UNION extends Expr("UNION ALL") with Operator with TokenRegex sealed trait ElasticOperator extends Operator with TokenRegex diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index d58d26d7..ad54ce26 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -16,21 +16,17 @@ package app.softnetwork.elastic -import app.softnetwork.elastic.sql.function.aggregate.{ - AggregateFunction, - COUNT, - MAX, - MIN, - WindowFunction -} +import app.softnetwork.elastic.sql.function.aggregate.{AggregateFunction, COUNT, WindowFunction} import app.softnetwork.elastic.sql.function.geo.DistanceUnit import app.softnetwork.elastic.sql.function.time.CurrentFunction -import app.softnetwork.elastic.sql.operator._ import app.softnetwork.elastic.sql.parser.{Validation, Validator} import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.Column +import com.fasterxml.jackson.databind.JsonNode import java.security.MessageDigest -import java.util.regex.Pattern +import scala.annotation.tailrec +import scala.jdk.CollectionConverters._ import scala.reflect.runtime.universe._ import scala.util.Try import scala.util.matching.Regex @@ -78,6 +74,10 @@ package object sql { def shouldBeScripted: Boolean = false } + trait DdlToken extends Token { + def ddl: String = sql + } + trait TokenValue extends Token { def value: Any } @@ -110,12 +110,6 @@ package object sql { } } - def paramValue: String = - if (nullable && checkNotNull.nonEmpty) - checkNotNull - else - s"$param${painlessMethods.mkString("")}" - private[this] var _painlessMethods: collection.mutable.Seq[String] = collection.mutable.Seq.empty @@ -129,14 +123,26 @@ package object sql { } - case class LiteralParam(param: String) extends PainlessParam { + case class LiteralParam(literal: String, maybeCheckNotNull: Option[String] = None) + extends PainlessParam { + override def param: String = literal override def sql: String = "" - override def checkNotNull: String = "" + override def nullable: Boolean = maybeCheckNotNull.nonEmpty + override def checkNotNull: String = maybeCheckNotNull.getOrElse("") + } + + sealed trait PainlessContextType + + case object PainlessContextType { + case object Processor extends PainlessContextType + case object Query extends PainlessContextType } /** Context for painless scripts + * @param context + * the context type */ - case class PainlessContext() { + case class PainlessContext(context: PainlessContextType = PainlessContextType.Query) { // List of parameter keys private[this] var _keys: collection.mutable.Seq[PainlessParam] = collection.mutable.Seq.empty @@ -146,6 +152,15 @@ package object sql { // Last parameter name added private[this] var _lastParam: Option[String] = None + def isProcessor: Boolean = context == PainlessContextType.Processor + + lazy val timestamp: String = { + context match { + case PainlessContextType.Processor => CurrentFunction.processorTimestamp + case PainlessContextType.Query => CurrentFunction.queryTimestamp + } + } + /** Add a token parameter to the context if not already present * * @param token @@ -155,6 +170,10 @@ package object sql { */ def addParam(token: Token): Option[String] = { token match { + case identifier: Identifier if isProcessor => + addParam( + LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ ) + ) case param: PainlessParam if param.param.nonEmpty && (param.isInstanceOf[LiteralParam] || param.nullable) => get(param) match { @@ -177,6 +196,8 @@ package object sql { def get(token: Token): Option[String] = { token match { + case identifier: Identifier if isProcessor => + get(LiteralParam(identifier.processParamName, None /*identifier.processCheckNotNull*/ )) case param: PainlessParam => if (exists(param)) Try(_values(_keys.indexOf(param))).toOption else None @@ -205,13 +226,19 @@ package object sql { else None } + private[this] def paramValue(param: PainlessParam): String = + if (param.nullable && param.checkNotNull.nonEmpty) + param.checkNotNull + else + s"${param.param}${param.painlessMethods.mkString("")}" + override def toString: String = { if (isEmpty) "" else _keys .flatMap { param => get(param) match { - case Some(v) => Some(s"def $v = ${param.paramValue}; ") + case Some(v) => Some(s"def $v = ${paramValue(param)}; ") case None => None // should not happen } } @@ -248,35 +275,17 @@ package object sql { } trait Updateable extends Token { - def update(request: SQLSearchRequest): Updateable + def update(request: SingleSearch): Updateable } abstract class Expr(override val sql: String) extends Token case object Distinct extends Expr("DISTINCT") with TokenRegex - abstract class Value[+T](val value: T)(implicit ev$1: T => Ordered[T]) - extends Token + abstract class Value[+T](val value: T) + extends DdlToken with PainlessScript with FunctionWithValue[T] { - def choose[R >: T]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - if (values.isEmpty) - None - else - operator match { - case Some(EQ) => values.find(_ == value) - case Some(NE | DIFF) => values.find(_ != value) - case Some(GE) => values.filter(_ >= value).sorted.reverse.headOption - case Some(GT) => values.filter(_ > value).sorted.reverse.headOption - case Some(LE) => values.filter(_ <= value).sorted.headOption - case Some(LT) => values.filter(_ < value).sorted.headOption - case _ => values.headOption - } - } override def painless(context: Option[PainlessContext]): String = SQLTypeUtils.coerce( value match { @@ -294,6 +303,228 @@ package object sql { override def nullable: Boolean = false } + object Value { + def apply[R: TypeTag, T <: Value[R]](value: Any): Value[_] = { + value match { + case null => Null + case b: Boolean => BooleanValue(b) + case c: Char => CharValue(c) + case s: String => + s match { + case "null" => Null + case "_id" | "{{_id]]" => IdValue + case "_ingest.timestamp" | "{{_ingest.timestamp}}" => IngestTimestampValue + case _ => StringValue(s) + } + case b: Byte => ByteValue(b) + case s: Short => ShortValue(s) + case i: Int => IntValue(i) + case l: Long => LongValue(l) + case f: Float => FloatValue(f) + case d: Double => DoubleValue(d) + case a: Array[T] => apply(a.toSeq) + case a: Seq[T] => + val values = a.map(apply) + values.headOption match { + case Some(_: StringValue) => + StringValues(values.asInstanceOf[Seq[StringValue]]).asInstanceOf[Values[R, T]] + case Some(_: ByteValue) => + ByteValues(values.asInstanceOf[Seq[ByteValue]]).asInstanceOf[Values[R, T]] + case Some(_: ShortValue) => + ShortValues(values.asInstanceOf[Seq[ShortValue]]).asInstanceOf[Values[R, T]] + case Some(_: IntValue) => + IntValues(values.asInstanceOf[Seq[IntValue]]).asInstanceOf[Values[R, T]] + case Some(_: LongValue) => + LongValues(values.asInstanceOf[Seq[LongValue]]).asInstanceOf[Values[R, T]] + case Some(_: FloatValue) => + FloatValues(values.asInstanceOf[Seq[FloatValue]]).asInstanceOf[Values[R, T]] + case Some(_: DoubleValue) => + DoubleValues(values.asInstanceOf[Seq[DoubleValue]]).asInstanceOf[Values[R, T]] + case Some(_: BooleanValue) => + BooleanValues(values.asInstanceOf[Seq[BooleanValue]]) + .asInstanceOf[Values[R, T]] + case Some(_: ObjectValue) => + ObjectValues( + values + .asInstanceOf[Seq[ObjectValue]] + ).asInstanceOf[Values[R, T]] + case _ => throw new IllegalArgumentException("Unsupported Values type") + } + case o: Map[_, _] => + val map = o.asInstanceOf[Map[String, Any]].map { case (k, v) => k -> apply(v) } + ObjectValue(map) + case other => StringValue(other.toString) + } + } + + def apply(node: JsonNode): Option[Any] = { + node match { + case n if n.isNull => Some(null) + case n if n.isTextual => Some(n.asText()) + case n if n.isBoolean => Some(n.asBoolean()) + case n if n.isShort => Some(node.asInstanceOf[Short]) + case n if n.isInt => Some(n.asInt()) + case n if n.isLong => Some(n.asLong()) + case n if n.isDouble => Some(n.asDouble()) + case n if n.isFloat => Some(node.asInstanceOf[Float]) + case n if n.isArray => + import scala.jdk.CollectionConverters._ + val arr = n + .elements() + .asScala + .flatMap(apply) + .toList + Some(arr) + case n if n.isObject => + val map = n + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val valueNode = entry.getValue + apply(valueNode).map(value => key -> value) + } + .toMap + Some(map) + case _ => + None + } + } + } + + sealed trait Diff + + case class Altered(name: String, value: Value[_]) extends Diff + + case class Removed(name: String) extends Diff + + case class ObjectValue(override val value: Map[String, Value[_]]) + extends Value[Map[String, Value[_]]](value) { + override def sql: String = value + .map { case (k, v) => s"""$k = ${v.sql}""" } + .mkString("(", ", ", ")") + override def baseType: SQLType = SQLTypes.Struct + override def ddl: String = value + .map { case (k, v) => + v match { + case IdValue | IngestTimestampValue => s"""$k = "${v.ddl}"""" + case _ => s"""$k = ${v.ddl}""" + } + } + .mkString("(", ", ", ")") + + def set(path: String, newValue: Value[_]): ObjectValue = { + val keys = path.split("\\.") + val updatedValue = { + if (keys.length == 1) { + value + (keys.head -> newValue) + } else { + val parentPath = keys.dropRight(1).mkString(".") + val parentKey = keys.last + val parentObject = find(parentPath) match { + case Some(obj: ObjectValue) => obj + case _ => ObjectValue.empty + } + val updatedParent = parentObject.set(parentKey, newValue) + value + (keys.head -> updatedParent) + } + } + ObjectValue(updatedValue) + } + + def remove(path: String): ObjectValue = { + val keys = path.split("\\.") + val updatedValue = { + if (keys.length == 1) { + value - keys.head + } else { + val parentPath = keys.dropRight(1).mkString(".") + val parentKey = keys.last + val parentObject = find(parentPath) match { + case Some(obj: ObjectValue) => obj + case _ => ObjectValue.empty + } + val updatedParent = parentObject.remove(parentKey) + value + (keys.head -> updatedParent) + } + } + ObjectValue(updatedValue) + } + + def find(path: String): Option[Value[_]] = { + val keys = path.split("\\.") + @tailrec + def loop(current: Map[String, Value[_]], remainingKeys: Seq[String]): Option[Value[_]] = { + remainingKeys match { + case Seq() => None + case Seq(k) => current.get(k) + case k +: ks => + current.get(k) match { + case Some(obj: ObjectValue) => loop(obj.value, ks) + case _ => None + } + } + } + loop(value, keys.toSeq) + } + + def diff(other: ObjectValue, path: Option[String] = None): List[Diff] = { + val diffs = scala.collection.mutable.ListBuffer[Diff]() + + val actual = this.value + val desired = other.value + + val allKeys = actual.keySet ++ desired.keySet + + allKeys.foreach { key => + val computedKey = s"${path.map(p => s"$p.").getOrElse("")}$key" + (actual.get(key), desired.get(key)) match { + case (None, Some(v)) => + diffs += Altered(computedKey, v) + case (Some(_), None) => + diffs += Removed(computedKey) + case (Some(a), Some(b)) => + b match { + case ObjectValue(_) => + a match { + case ObjectValue(_) => + diffs ++= a + .asInstanceOf[ObjectValue] + .diff( + b.asInstanceOf[ObjectValue], + Some(computedKey) + ) + case _ => + diffs += Altered(computedKey, b) + } + case _ => + if (a != b) { + diffs += Altered(computedKey, b) + } + } + case _ => + } + } + + diffs.toList + } + + import app.softnetwork.elastic.sql.serialization._ + + def toJson: JsonNode = this + + } + + object ObjectValue { + import app.softnetwork.elastic.sql.serialization._ + + def empty: ObjectValue = ObjectValue(Map.empty) + + def fromJson(jsonNode: JsonNode): ObjectValue = jsonNode + + def parseJson(jsonString: String): ObjectValue = jsonString + } + case object Null extends Value[Null](null) with TokenRegex { override def sql: String = "NULL" override def painless(context: Option[PainlessContext]): String = "null" @@ -301,6 +532,13 @@ package object sql { override def baseType: SQLType = SQLTypes.Null } + case object ParamValue extends Value[String](null) with TokenRegex { + override def sql: String = "?" + override def painless(context: Option[PainlessContext]): String = "params.paramValue" + override def nullable: Boolean = true + override def baseType: SQLType = SQLTypes.Any + } + case class BooleanValue(override val value: Boolean) extends Value[Boolean](value) { override def sql: String = value.toString override def baseType: SQLType = SQLTypes.Boolean @@ -313,47 +551,28 @@ package object sql { case class StringValue(override val value: String) extends Value[String](value) { override def sql: String = s"""'$value'""" - import SQLImplicits._ - private lazy val pattern: Pattern = value.pattern - def like: Seq[String] => Boolean = { - _.exists { pattern.matcher(_).matches() } - } - def eq: Seq[String] => Boolean = { - _.exists { _.contentEquals(value) } - } - def ne: Seq[String] => Boolean = { - _.forall { !_.contentEquals(value) } - } - override def choose[R >: String]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - operator match { - case Some(EQ) => values.find(v => v.toString contentEquals value) - case Some(NE | DIFF) => values.find(v => !(v.toString contentEquals value)) - case Some(LIKE | RLIKE) => values.find(v => pattern.matcher(v.toString).matches()) - case None => Some(values.mkString(separator)) - case _ => super.choose(values, operator, separator) - } - } override def baseType: SQLType = SQLTypes.Varchar + + override def ddl: String = s""""$value"""" + } + + case object IdValue extends Value[String]("_id") with TokenRegex { + override def sql: String = value + override def painless(context: Option[PainlessContext]): String = s"{{$value}}" + override def baseType: SQLType = SQLTypes.Varchar + override def ddl: String = value + } + + case object IngestTimestampValue extends Value[String]("_ingest.timestamp") with TokenRegex { + override def sql: String = value + override def painless(context: Option[PainlessContext]): String = + s"{{$value}}" + override def baseType: SQLType = SQLTypes.Timestamp + override def ddl: String = value } - sealed abstract class NumericValue[T: Numeric](override val value: T)(implicit - ev$1: T => Ordered[T] - ) extends Value[T](value) { + sealed abstract class NumericValue[T: Numeric](override val value: T) extends Value[T](value) { override def sql: String = value.toString - override def choose[R >: T]( - values: Seq[R], - operator: Option[ExpressionOperator], - separator: String = "|" - )(implicit ev: R => Ordered[R]): Option[R] = { - operator match { - case None => if (values.isEmpty) None else Some(values.max) - case _ => super.choose(values, operator, separator) - } - } private[this] val num: Numeric[T] = implicitly[Numeric[T]] def toDouble: Double = num.toDouble(value) def toEither: Either[Long, Double] = value match { @@ -363,14 +582,6 @@ package object sql { case f: Float => Right(f.toDouble) case _ => Right(toDouble) } - def max: Seq[T] => T = x => Try(x.max).getOrElse(num.zero) - def min: Seq[T] => T = x => Try(x.min).getOrElse(num.zero) - def eq: Seq[T] => Boolean = { - _.exists { _ == value } - } - def ne: Seq[T] => Boolean = { - _.forall { _ != value } - } override def baseType: SQLNumeric = SQLTypes.Numeric } @@ -478,14 +689,15 @@ package object sql { extends FromTo(from, to) sealed abstract class Values[+R: TypeTag, +T <: Value[R]](val values: Seq[T]) - extends Token - with PainlessScript { + extends Value[Seq[T]](values) { override def sql = s"(${values.map(_.sql).mkString(",")})" override def painless(context: Option[PainlessContext]): String = s"[${values.map(_.painless(context)).mkString(",")}]" lazy val innerValues: Seq[R] = values.map(_.value) override def nullable: Boolean = values.exists(_.nullable) override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Any) + + override def ddl: String = s"[${values.map(_.ddl).mkString(",")}]" } case class StringValues(override val values: Seq[StringValue]) @@ -537,23 +749,23 @@ package object sql { override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Double) } - def choose[T]( - values: Seq[T], - criteria: Option[Criteria], - function: Option[Function] = None - )(implicit ev$1: T => Ordered[T]): Option[T] = { - criteria match { - case Some(GenericExpression(_, operator, value: Value[T] @unchecked, _)) => - value.choose[T](values, Some(operator)) - case _ => - function match { - case Some(MIN) => Some(values.min) - case Some(MAX) => Some(values.max) - // FIXME case Some(SQLSum) => Some(values.sum) - // FIXME case Some(SQLAvg) => Some(values.sum / values.length ) - case _ => values.headOption - } + case class BooleanValues(override val values: Seq[BooleanValue]) + extends Values[Boolean, Value[Boolean]](values) { + def eq: Seq[Boolean] => Boolean = { + _.exists { b => innerValues.contains(b) } } + def ne: Seq[Boolean] => Boolean = { + _.forall { b => !innerValues.contains(b) } + } + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Boolean) + } + + case class ObjectValues(override val values: Seq[ObjectValue]) + extends Values[Map[String, Value[_]], ObjectValue](values) { + override def baseType: SQLArray = SQLTypes.Array(SQLTypes.Struct) + import app.softnetwork.elastic.sql.serialization._ + + def toJson: JsonNode = this } def toRegex(value: String): String = { @@ -606,7 +818,7 @@ package object sql { trait Source extends Updateable { def name: String - def update(request: SQLSearchRequest): Source + def update(request: SingleSearch): Source } sealed trait Identifier @@ -620,9 +832,10 @@ package object sql { def withFunctions(functions: List[Function]): Identifier - def update(request: SQLSearchRequest): Identifier + def update(request: SingleSearch): Identifier def tableAlias: Option[String] + def table: Option[String] def distinct: Boolean def nested: Boolean def nestedElement: Option[NestedElement] @@ -765,7 +978,46 @@ package object sql { else s"(doc['$path'].size() == 0 ? $nullValue : doc['$path'].value${painlessMethods.mkString("")})" + lazy val processParamName: String = { + if (path.nonEmpty) { + if (path.contains(".")) + s"ctx.${path.split("\\.").mkString("?.")}" + else + s"ctx.$path" + } else "" + } + + lazy val processCheckNotNull: Option[String] = + if (path.isEmpty || !nullable) None + else + Option(s"(ctx.$path == null ? $nullValue : ctx.$path${painlessMethods.mkString("")})") + + def originalType: SQLType = + if (name.trim.nonEmpty) SQLTypes.Any + else this.baseType + override def painless(context: Option[PainlessContext]): String = { + val orderedFunctions = FunctionUtils.transformFunctions(this).reverse + var currType = this.originalType + currType match { + case SQLTypes.Any => + orderedFunctions.headOption match { + case Some(f: TransformFunction[_, _]) => + f.in match { + case SQLTypes.Temporal => // the first function to apply required a Temporal as input type + context match { + case Some(_) => + // compatible ES6+ + this.addPainlessMethod(".toInstant().atZone(ZoneId.of('Z'))") + currType = SQLTypes.Timestamp + case _ => // do nothing + } + case _ => // do nothing + } + case _ => // do nothing + } + case _ => // do nothing + } val base = context match { case Some(ctx) => @@ -776,7 +1028,6 @@ package object sql { else paramName } - val orderedFunctions = FunctionUtils.transformFunctions(this).reverse var expr = base orderedFunctions.zipWithIndex.foreach { case (f, idx) => f match { @@ -784,6 +1035,7 @@ package object sql { case f: PainlessScript => expr = s"$expr${f.painless(context)}" case f => expr = f.toSQL(expr) // fallback } + currType = f.out } expr } @@ -843,7 +1095,9 @@ package object sql { fieldAlias: Option[String] = None, bucket: Option[Bucket] = None, nestedElement: Option[NestedElement] = None, - bucketPath: String = "" + bucketPath: String = "", + col: Option[Column] = None, + table: Option[String] = None ) extends Identifier { def withFunctions(functions: List[Function]): Identifier = this.copy(functions = functions) @@ -854,7 +1108,9 @@ package object sql { id } - def update(request: SQLSearchRequest): Identifier = { + override def baseType: SQLType = col.map(_.dataType).getOrElse(super.baseType) + + def update(request: SingleSearch): Identifier = { val bucketPath: String = request.groupBy match { case Some(gb) => @@ -870,7 +1126,8 @@ package object sql { } val parts: Seq[String] = name.split("\\.").toSeq val tableAlias = parts.head - if (request.tableAliases.values.toSeq.contains(tableAlias)) { + val table = request.tableAliases.find(t => t._2 == tableAlias).map(_._2) + if (table.nonEmpty) { request.unnestAliases.find(_._1 == tableAlias) match { case Some(tuple) if !nested => val nestedElement = @@ -878,27 +1135,33 @@ package object sql { case Some(unnest) => Some(request.toNestedElement(unnest)) case None => None } + val colName = parts.tail.mkString(".") this .copy( tableAlias = Some(tableAlias), - name = s"${tuple._2._1}.${parts.tail.mkString(".")}", + name = s"${tuple._2._1}.$colName", nested = true, limit = tuple._2._2, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), nestedElement = nestedElement, - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) .withFunctions(this.updateFunctions(request)) case Some(tuple) if nested => + val colName = parts.tail.mkString(".") this .copy( tableAlias = Some(tableAlias), - name = s"${tuple._2._1}.${parts.tail.mkString(".")}", + name = s"${tuple._2._1}.$colName", limit = tuple._2._2, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) .withFunctions(this.updateFunctions(request)) case None if nested => @@ -907,16 +1170,21 @@ package object sql { tableAlias = Some(tableAlias), fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(name)), + table = table ) .withFunctions(this.updateFunctions(request)) case _ => + val colName = parts.tail.mkString(".") this.copy( tableAlias = Some(tableAlias), - name = parts.tail.mkString("."), + name = colName, fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(colName)), + table = table ) } } else { @@ -924,7 +1192,8 @@ package object sql { .copy( fieldAlias = request.fieldAliases.get(identifierName).orElse(fieldAlias), bucket = request.bucketNames.get(identifierName).orElse(bucket), - bucketPath = bucketPath + bucketPath = bucketPath, + col = request.schema.flatMap(schema => schema.find(name)) ) .withFunctions(this.updateFunctions(request)) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala similarity index 54% rename from sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala rename to sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala index 29c1bb9c..4b0ac6ad 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLQuery.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/DmlParser.scala @@ -14,21 +14,8 @@ * limitations under the License. */ -package app.softnetwork.elastic.sql.query +package app.softnetwork.elastic.sql.parser -import app.softnetwork.elastic.sql.SQL +import app.softnetwork.elastic.sql.query.{Delete, Insert, Update} -/** SQL Query wrapper - * @param query - * - the SQL query - * @param score - * - optional minimum score for the elasticsearch query - */ -case class SQLQuery(query: SQL, score: Option[Double] = None) { - import app.softnetwork.elastic.sql.SQLImplicits._ - lazy val request: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { - query - } - - def minScore(score: Double): SQLQuery = this.copy(score = Some(score)) -} +trait DmlParser { self: Parser with WhereParser => } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala index ec798f2a..81ddacf5 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Parser.scala @@ -16,6 +16,7 @@ package app.softnetwork.elastic.sql.parser +import app.softnetwork.elastic.sql.PainlessContextType.Processor import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.function._ import app.softnetwork.elastic.sql.operator._ @@ -29,9 +30,19 @@ import app.softnetwork.elastic.sql.parser.function.string.StringParser import app.softnetwork.elastic.sql.parser.function.time.TemporalParser import app.softnetwork.elastic.sql.parser.operator.math.ArithmeticParser import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.{ + Column, + IngestPipelineType, + IngestProcessor, + IngestProcessorType, + PartitionDate, + ScriptProcessor +} +import app.softnetwork.elastic.sql.time.TimeUnit import scala.language.implicitConversions import scala.language.existentials +import scala.util.matching.Regex import scala.util.parsing.combinator.{PackratParsers, RegexParsers} import scala.util.parsing.input.CharSequenceReader @@ -49,25 +60,597 @@ object Parser with OrderByParser with LimitParser { - def request: PackratParser[SQLSearchRequest] = { - phrase(select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.?) ^^ { - case s ~ f ~ w ~ g ~ h ~ o ~ l => - val request = SQLSearchRequest(s, f, w, g, h, o, l).update() - request.validate() match { - case Left(error) => throw ValidationError(error) - case _ => - } - request + def single: PackratParser[SingleSearch] = { + select ~ from ~ where.? ~ groupBy.? ~ having.? ~ orderBy.? ~ limit.? ~ onConflict.? ^^ { + case s ~ f ~ w ~ g ~ h ~ o ~ l ~ oc => + SingleSearch(s, f, w, g, h, o, l, onConflict = oc).update() } } def union: PackratParser[UNION.type] = UNION.regex ^^ (_ => UNION) - def requests: PackratParser[List[SQLSearchRequest]] = rep1sep(request, union) ^^ (s => s) + def dqlStatement: PackratParser[DqlStatement] = rep1sep(single, union) ^^ { + case x :: Nil => x + case s => MultiSearch(s) + } + + def ident: Parser[String] = """[a-zA-Z_][a-zA-Z0-9_]*""".r + + private val lparen: Parser[String] = "(" + private val rparen: Parser[String] = ")" + private val comma: Parser[String] = "," + private val lbracket: Parser[String] = "[" + private val rbracket: Parser[String] = "]" + private val startStruct: Parser[String] = "{" + private val endStruct: Parser[String] = "}" + + def objectValue: PackratParser[ObjectValue] = + lparen ~> rep1sep(option, comma) <~ rparen ^^ { opts => + ObjectValue(opts.toMap) + } + + def objectValues: PackratParser[ObjectValues] = + lbracket ~> rep1sep(objectValue, comma) <~ rbracket ^^ { ovs => + ObjectValues(ovs) + } + + def option: PackratParser[(String, Value[_])] = + ident ~ "=" ~ (objectValues | objectValue | value) ^^ { case key ~ _ ~ value => + (key, value) + } + + def options: PackratParser[Map[String, Value[_]]] = + "OPTIONS" ~ lparen ~ repsep(option, comma) ~ rparen ^^ { case _ ~ _ ~ opts ~ _ => + opts.toMap + } + + def array_of_struct: PackratParser[ObjectValues] = + lbracket ~> repsep(struct, comma) <~ rbracket ^^ { ovs => + ObjectValues(ovs) + } + + def struct_entry: PackratParser[(String, Value[_])] = + ident ~ "=" ~ (array_of_struct | struct | value) ^^ { case key ~ _ ~ v => + key -> v + } + + def struct: PackratParser[ObjectValue] = + startStruct ~> repsep(struct_entry, comma) <~ endStruct ^^ { entries => + ObjectValue(entries.toMap) + } + + def row: PackratParser[List[Value[_]]] = + lparen ~> repsep(array_of_struct | struct | value, comma) <~ rparen + + def rows: PackratParser[List[List[Value[_]]]] = + repsep(row, comma) + + def processorType: PackratParser[IngestProcessorType] = + ident ^^ { name => + name.toLowerCase match { + case "set" => IngestProcessorType.Set + case "script" => IngestProcessorType.Script + case "rename" => IngestProcessorType.Rename + case "remove" => IngestProcessorType.Remove + case "date_index_name" => IngestProcessorType.DateIndexName + case other => IngestProcessorType(other) + } + } + + def processor: PackratParser[IngestProcessor] = + processorType ~ objectValue ^^ { case pt ~ opts => + IngestProcessor(pt, opts) + } + + def createOrReplacePipeline: PackratParser[CreatePipeline] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "PIPELINE") ~ ident ~ ("WITH" ~ "PROCESSORS") ~ start ~ repsep( + processor, + separator + ) ~ end ^^ { case _ ~ name ~ _ ~ _ ~ proc ~ _ => + CreatePipeline(name, IngestPipelineType.Custom, orReplace = true, processors = proc) + } + + def createPipeline: PackratParser[CreatePipeline] = + ("CREATE" ~ "PIPELINE") ~ ifNotExists ~ ident ~ ("WITH" ~ "PROCESSORS" ~ start) ~ repsep( + processor, + separator + ) <~ end ^^ { case _ ~ ine ~ name ~ _ ~ proc => + CreatePipeline(name, IngestPipelineType.Custom, ifNotExists = ine, processors = proc) + } + + def dropPipeline: PackratParser[DropPipeline] = + ("DROP" ~ "PIPELINE") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropPipeline(name, ifExists = ie) + } + + def showPipeline: PackratParser[ShowPipeline] = + ("SHOW" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + ShowPipeline(pipeline) + } + + def showCreatePipeline: PackratParser[ShowCreatePipeline] = + ("SHOW" ~ "CREATE" ~ "PIPELINE") ~ ident ^^ { case _ ~ _ ~ _ ~ pipeline => + ShowCreatePipeline(pipeline) + } + + def describePipeline: PackratParser[DescribePipeline] = + ("DESCRIBE" ~ "PIPELINE") ~ ident ^^ { case _ ~ pipeline => + DescribePipeline(pipeline) + } + + def addProcessor: PackratParser[AddPipelineProcessor] = + ("ADD" ~ "PROCESSOR") ~ processor ^^ { case _ ~ proc => + AddPipelineProcessor(proc) + } + + def dropProcessor: PackratParser[DropPipelineProcessor] = + ("DROP" ~ "PROCESSOR") ~ processorType ~ start ~ ident ~ end ^^ { case _ ~ pt ~ _ ~ name ~ _ => + DropPipelineProcessor(pt, name) + } + + def alterPipelineStatement: PackratParser[AlterPipelineStatement] = + addProcessor | dropProcessor + + def alterPipeline: PackratParser[AlterPipeline] = + ("ALTER" ~ "PIPELINE") ~ ifExists ~ ident ~ start.? ~ repsep( + alterPipelineStatement, + separator + ) ~ end.? ^^ { case _ ~ ie ~ pipeline ~ s ~ stmts ~ e => + if (s.isDefined && e.isEmpty) { + throw new Exception("Mismatched closing parentheses in ALTER PIPELINE statement") + } else if (s.isEmpty && e.isDefined) { + throw new Exception("Mismatched opening parentheses in ALTER PIPELINE statement") + } else if (s.isEmpty && e.isEmpty && stmts.size > 1) { + throw new Exception("Multiple ALTER PIPELINE statements require parentheses") + } else + AlterPipeline(pipeline, ie, stmts) + } + + def multiFields: PackratParser[List[Column]] = + "FIELDS" ~ start ~> repsep(column, separator) <~ end ^^ (cols => cols) | success(Nil) + + def ifExists: PackratParser[Boolean] = + opt("IF" ~ "EXISTS") ^^ { + case Some(_) => true + case None => false + } + + def ifNotExists: PackratParser[Boolean] = + opt("IF" ~ "NOT" ~ "EXISTS") ^^ { + case Some(_) => true + case None => false + } + + def notNull: PackratParser[Boolean] = + opt("NOT" ~ "NULL") ^^ { + case Some(_) => true + case None => false + } + + def ingest_id: PackratParser[Value[_]] = "_id" ^^ (_ => IdValue) + + def ingest_timestamp: PackratParser[Value[_]] = "_ingest.timestamp" ^^ (_ => IngestTimestampValue) + + def defaultVal: PackratParser[Option[Value[_]]] = + opt("DEFAULT" ~ (value | ingest_id | ingest_timestamp)) ^^ { + case Some(_ ~ v) => Some(v) + case None => None + } + + def comment: PackratParser[Option[String]] = + opt("COMMENT" ~ literal) ^^ { + case Some(_ ~ v) => Some(v.value) + case None => None + } + + def script: PackratParser[PainlessScript] = + ("SCRIPT" ~ "AS") ~ start ~ (identifierWithArithmeticExpression | + identifierWithTransformation | + identifierWithIntervalFunction | + identifierWithFunction) ~ end ^^ { case _ ~ _ ~ s ~ _ => s } + + def column: PackratParser[Column] = + ident ~ extension_type ~ (script | multiFields) ~ defaultVal ~ notNull ~ comment ~ (options | success( + Map.empty[String, Value[_]] + )) ^^ { case name ~ dt ~ mfs ~ dv ~ nn ~ ct ~ opts => + mfs match { + case script: PainlessScript => + val ctx = PainlessContext(Processor) + val scr = script.painless(Some(ctx)) + val temp = s"$ctx$scr" + val ret = + temp.split(";") match { + case Array(single) if single.trim.startsWith("return ") => + val stripReturn = single.trim.stripPrefix("return ").trim + s"ctx.$name = $stripReturn" + case multiple => + val last = multiple.last.trim + val temp = multiple.dropRight(1) :+ s" ctx.$name = $last" + temp.mkString(";") + } + Column( + name, + dt, + Some( + ScriptProcessor( + script = script.sql, + column = name, + dataType = dt, + source = ret + ) + ), + Nil, + dv, + nn, + ct, + opts + ) + case cols: List[Column] => + Column(name, dt, None, cols, dv, nn, ct, opts) + } + } + + def columns: PackratParser[List[Column]] = + start ~ repsep(column, separator) ~ end ^^ { case _ ~ cols ~ _ => cols } + + def primaryKey: PackratParser[List[String]] = + separator ~ "PRIMARY" ~ "KEY" ~ start ~ repsep(ident, separator) ~ end ^^ { + case _ ~ _ ~ _ ~ _ ~ keys ~ _ => + keys + } | success(Nil) + + def granularity: PackratParser[TimeUnit] = start ~ + (("YEAR" ^^^ TimeUnit.YEARS) | + ("MONTH" ^^^ TimeUnit.MONTHS) | + ("DAY" ^^^ TimeUnit.DAYS) | + ("HOUR" ^^^ TimeUnit.HOURS) | + ("MINUTE" ^^^ TimeUnit.MINUTES) | + ("SECOND" ^^^ TimeUnit.SECONDS)) ~ end ^^ { case _ ~ gf ~ _ => gf } + + def partitionBy: PackratParser[Option[PartitionDate]] = + opt("PARTITION" ~ "BY" ~ ident ~ opt(granularity)) ^^ { + case Some(_ ~ _ ~ pb ~ gf) => Some(PartitionDate(pb, gf.getOrElse(TimeUnit.DAYS))) + case None => None + } + + def columnsWithPartitionBy + : PackratParser[(List[Column], List[String], Option[PartitionDate], Map[String, Any])] = + start ~ repsep( + column, + separator + ) ~ primaryKey ~ end ~ partitionBy ~ ((separator.? ~> options) | success( + Map.empty[String, Value[_]] + )) ^^ { case _ ~ cols ~ pk ~ _ ~ pb ~ opts => + (cols, pk, pb, opts) + } + + def createOrReplaceTable: PackratParser[CreateTable] = + ("CREATE" ~ "OR" ~ "REPLACE" ~ "TABLE") ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { + case _ ~ name ~ lr => + lr match { + case ( + cols: List[Column], + pk: List[String], + p: Option[PartitionDate], + opts: Map[String, Value[_]] + ) => + CreateTable( + name, + Right(cols), + ifNotExists = false, + orReplace = true, + primaryKey = pk, + partitionBy = p, + options = opts + ) + case sel: DqlStatement => + CreateTable(name, Left(sel), ifNotExists = false, orReplace = true) + } + } + + def createTable: PackratParser[CreateTable] = + ("CREATE" ~ "TABLE") ~ ifNotExists ~ ident ~ (columnsWithPartitionBy | ("AS" ~> dqlStatement)) ^^ { + case _ ~ ine ~ name ~ lr => + lr match { + case ( + cols: List[Column], + pk: List[String], + p: Option[PartitionDate], + opts: Map[String, Value[_]] + ) => + CreateTable(name, Right(cols), ine, primaryKey = pk, partitionBy = p, options = opts) + case sel: DqlStatement => CreateTable(name, Left(sel), ine) + } + } + + def showTable: PackratParser[ShowTable] = + ("SHOW" ~ "TABLE") ~ ident ^^ { case _ ~ table => + ShowTable(table) + } + + def showCreateTable: PackratParser[ShowCreateTable] = + ("SHOW" ~ "CREATE" ~ "TABLE") ~ ident ^^ { case _ ~ _ ~ _ ~ table => + ShowCreateTable(table) + } + + def describeTable: PackratParser[DescribeTable] = + ("DESCRIBE" ~ "TABLE") ~ ident ^^ { case _ ~ table => + DescribeTable(table) + } + + def dropTable: PackratParser[DropTable] = + ("DROP" ~ "TABLE") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropTable(name, ifExists = ie) + } + + def truncateTable: PackratParser[TruncateTable] = + ("TRUNCATE" ~ "TABLE") ~ ident ^^ { case _ ~ name => + TruncateTable(name) + } + + def addColumn: PackratParser[AddColumn] = + ("ADD" ~ "COLUMN") ~ ifNotExists ~ column ^^ { case _ ~ ine ~ col => + AddColumn(col, ifNotExists = ine) + } + + def dropColumn: PackratParser[DropColumn] = + ("DROP" ~ "COLUMN") ~ ifExists ~ ident ^^ { case _ ~ ie ~ name => + DropColumn(name, ifExists = ie) + } + + def renameColumn: PackratParser[RenameColumn] = + ("RENAME" ~ "COLUMN") ~ ident ~ ("TO" ~> ident) ^^ { case _ ~ oldName ~ newName => + RenameColumn(oldName, newName) + } + + def alterColumnIfExists: PackratParser[Boolean] = + ("ALTER" ~ "COLUMN") ~ ifExists ^^ { case _ ~ ie => + ie + } + + def alterColumnOptions: PackratParser[AlterColumnOptions] = + alterColumnIfExists ~ ident ~ "SET" ~ options ^^ { case ie ~ col ~ _ ~ opts => + AlterColumnOptions(col, opts, ifExists = ie) + } + + def alterColumnOption: PackratParser[AlterColumnOption] = + alterColumnIfExists ~ ident ~ (("SET" | "ADD") ~ "OPTION") ~ start ~ option ~ end ^^ { + case ie ~ col ~ _ ~ _ ~ opt ~ _ => + AlterColumnOption(col, opt._1, opt._2, ifExists = ie) + } + + def dropColumnOption: PackratParser[DropColumnOption] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "OPTION") ~ ident ^^ { case ie ~ col ~ _ ~ optionName => + DropColumnOption(col, optionName, ifExists = ie) + } + + def alterColumnFields: PackratParser[AlterColumnFields] = + alterColumnIfExists ~ ident ~ "SET" ~ multiFields ^^ { case ie ~ col ~ _ ~ fields => + AlterColumnFields(col, fields, ifExists = ie) + } + + def alterColumnField: PackratParser[AlterColumnField] = + alterColumnIfExists ~ ident ~ (("SET" | "ADD") ~ "FIELD") ~ column ^^ { + case ie ~ col ~ _ ~ field => + AlterColumnField(col, field, ifExists = ie) + } + + def dropColumnField: PackratParser[DropColumnField] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "FIELD") ~ ident ^^ { case ie ~ col ~ _ ~ fieldName => + DropColumnField(col, fieldName, ifExists = ie) + } + + def alterColumnType: PackratParser[AlterColumnType] = + alterColumnIfExists ~ ident ~ ("SET" ~ "DATA" ~ "TYPE") ~ extension_type ^^ { + case ie ~ name ~ _ ~ newType => AlterColumnType(name, newType, ifExists = ie) + } + + def alterColumnScript: PackratParser[AlterColumnScript] = + alterColumnIfExists ~ ident ~ "SET" ~ script ^^ { case ie ~ name ~ _ ~ ns => + val ctx = PainlessContext(Processor) + val scr = ns.painless(Some(ctx)) + val temp = s"$ctx$scr" + val ret = + temp.split(";") match { + case Array(single) if single.trim.startsWith("return ") => + val stripReturn = single.trim.stripPrefix("return ").trim + s"ctx.$name = $stripReturn" + case multiple => + val last = multiple.last.trim + val temp = multiple.dropRight(1) :+ s" ctx.$name = $last" + temp.mkString(";") + } + AlterColumnScript( + name, + ScriptProcessor( + script = ns.sql, + column = name, + dataType = ns.out, + source = ret + ), + ifExists = ie + ) + } + + def dropColumnScript: PackratParser[DropColumnScript] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "SCRIPT") ^^ { case ie ~ name ~ _ => + DropColumnScript(name, ifExists = ie) + } + + def alterColumnDefault: PackratParser[AlterColumnDefault] = + alterColumnIfExists ~ ident ~ ("SET" ~ "DEFAULT") ~ value ^^ { case ie ~ name ~ _ ~ dv => + AlterColumnDefault(name, dv, ifExists = ie) + } + + def dropColumnDefault: PackratParser[DropColumnDefault] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "DEFAULT") ^^ { case ie ~ name ~ _ => + DropColumnDefault(name, ifExists = ie) + } + + def alterColumnNotNull: PackratParser[AlterColumnNotNull] = + alterColumnIfExists ~ ident ~ ("SET" ~ "NOT" ~ "NULL") ^^ { case ie ~ name ~ _ => + AlterColumnNotNull(name, ifExists = ie) + } + + def dropColumnNotNull: PackratParser[DropColumnNotNull] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "NOT" ~ "NULL") ^^ { case ie ~ name ~ _ => + DropColumnNotNull(name, ifExists = ie) + } + + def alterColumnComment: PackratParser[AlterColumnComment] = + alterColumnIfExists ~ ident ~ ("SET" ~ "COMMENT") ~ literal ^^ { case ie ~ name ~ _ ~ c => + AlterColumnComment(name, c.value, ifExists = ie) + } + + def dropColumnComment: PackratParser[DropColumnComment] = + alterColumnIfExists ~ ident ~ ("DROP" ~ "COMMENT") ^^ { case ie ~ name ~ _ => + DropColumnComment(name, ifExists = ie) + } + + def alterTableMapping: PackratParser[AlterTableMapping] = + (("SET" | "ADD") ~ "MAPPING") ~ start ~> option <~ end ^^ { opt => + AlterTableMapping(opt._1, opt._2) + } + + def dropTableMapping: PackratParser[DropTableMapping] = + ("DROP" ~ "MAPPING") ~> ident ^^ { m => DropTableMapping(m) } + + def alterTableSetting: PackratParser[AlterTableSetting] = + (("SET" | "ADD") ~ "SETTING") ~ start ~> option <~ end ^^ { opt => + AlterTableSetting(opt._1, opt._2) + } + + def dropTableSetting: PackratParser[DropTableSetting] = + ("DROP" ~ "SETTING") ~> ident ^^ { m => DropTableSetting(m) } + + def alterTableAlias: PackratParser[AlterTableAlias] = + (("SET" | "ADD") ~ "ALIAS") ~ start ~> option <~ end ^^ { opt => + AlterTableAlias(opt._1, opt._2) + } + + def dropTableAlias: PackratParser[DropTableAlias] = + ("DROP" ~ "ALIAS") ~> ident ^^ { m => DropTableAlias(m) } + + def alterTableStatement: PackratParser[AlterTableStatement] = + addColumn | + dropColumn | + renameColumn | + alterColumnOptions | + alterColumnOption | + dropColumnOption | + alterColumnType | + alterColumnScript | + dropColumnScript | + alterColumnDefault | + dropColumnDefault | + alterColumnNotNull | + dropColumnNotNull | + alterColumnComment | + dropColumnComment | + alterColumnFields | + alterColumnField | + dropColumnField | + alterTableMapping | + dropTableMapping | + alterTableSetting | + dropTableSetting | + alterTableAlias | + dropTableAlias + + def alterTable: PackratParser[AlterTable] = + ("ALTER" ~ "TABLE") ~ ifExists ~ ident ~ start.? ~ repsep( + alterTableStatement, + separator + ) ~ end.? ^^ { case _ ~ ie ~ table ~ s ~ stmts ~ e => + if (s.isDefined && e.isEmpty) { + throw new Exception("Mismatched closing parentheses in ALTER TABLE statement") + } else if (s.isEmpty && e.isDefined) { + throw new Exception("Mismatched opening parentheses in ALTER TABLE statement") + } else if (s.isEmpty && e.isEmpty && stmts.size > 1) { + throw new Exception("Multiple ALTER TABLE statements require parentheses") + } else + AlterTable(table, ie, stmts) + } + + def ddlStatement: PackratParser[DdlStatement] = + createTable | + createPipeline | + createOrReplaceTable | + createOrReplacePipeline | + alterTable | + alterPipeline | + dropTable | + truncateTable | + showTable | + showCreateTable | + describeTable | + dropPipeline | + showPipeline | + showCreatePipeline | + describePipeline + + def onConflict: PackratParser[OnConflict] = + ("ON" ~ "CONFLICT" ~> opt(conflictTarget) <~ "DO") ~ ("UPDATE" | "NOTHING") ^^ { + case target ~ action => + OnConflict(target, action == "UPDATE") + } + + def conflictTarget: PackratParser[List[String]] = + start ~> repsep(ident, separator) <~ end + + /** INSERT INTO table [(col1, col2, ...)] VALUES (v1, v2, ...) */ + def insert: PackratParser[Insert] = + ("INSERT" ~ "INTO") ~ ident ~ opt(lparen ~> repsep(ident, comma) <~ rparen) ~ + (("VALUES" ~> rows) ^^ { vs => Right(vs) } + | "AS".? ~> dqlStatement ^^ { q => Left(q) }) ~ opt(onConflict) ^^ { + case _ ~ table ~ colsOpt ~ vals ~ conflict => + conflict match { + case Some(c) => Insert(table, colsOpt.getOrElse(Nil), vals, Some(c)) + case _ => + vals match { + case Left(q: SingleSearch) => + Insert(table, colsOpt.getOrElse(Nil), vals, q.onConflict) + case _ => Insert(table, colsOpt.getOrElse(Nil), vals) + } + } + } + + def fileFormat: PackratParser[FileFormat] = + ("FILE_FORMAT" ~> ( + ("PARQUET" ^^^ Parquet) | + ("JSON" ^^^ Json) | + ("JSON_ARRAY" ^^^ JsonArray) | + ("DELTA_LAKE" ^^^ Delta) + )) ^^ { ff => ff } + + /** COPY INTO table FROM source */ + def copy: PackratParser[CopyInto] = + ("COPY" ~ "INTO") ~ ident ~ ("FROM" ~> literal) ~ opt(fileFormat) ~ opt(onConflict) ^^ { + case _ ~ table ~ source ~ format ~ conflict => + CopyInto(source.value, table, fileFormat = format, onConflict = conflict) + } + + /** UPDATE table SET col1 = v1, col2 = v2 [WHERE ...] */ + def update: PackratParser[Update] = + ("UPDATE" ~> ident) ~ ("SET" ~> repsep(ident ~ "=" ~ value, separator)) ~ where.? ^^ { + case table ~ assigns ~ w => + val values = assigns.map { case col ~ _ ~ v => col -> v }.toMap + Update(table, values, w) + } + + /** DELETE FROM table [WHERE ...] */ + def delete: PackratParser[Delete] = + ("DELETE" ~ "FROM") ~> ident ~ where.? ^^ { case table ~ w => + Delete(Table(table), w) + } + + def dmlStatement: PackratParser[DmlStatement] = insert | update | delete | copy + + def statement: PackratParser[Statement] = ddlStatement | dqlStatement | dmlStatement def apply( query: String - ): Either[ParserError, Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + ): Either[ParserError, Statement] = { val normalizedQuery = query .split("\n") @@ -75,14 +658,14 @@ object Parser .filterNot(w => w.isEmpty || w.startsWith("--")) .mkString(" ") val reader = new PackratReader(new CharSequenceReader(normalizedQuery)) - parse(requests, reader) match { + parse(statement, reader) match { case NoSuccess(msg, _) => Console.err.println(msg) Left(ParserError(msg)) case Success(result, _) => - result match { - case x :: Nil => Right(Left(x)) - case _ => Right(Right(SQLMultiSearchRequest(result))) + result.validate() match { + case Left(error) => Left(ParserError(error)) + case _ => Right(result) } } } @@ -134,6 +717,15 @@ trait Parser private val reservedKeywords = Seq( "select", + "insert", + "update", + "copy", + "delete", + "create", + "alter", + "drop", + "truncate", + "column", "from", "join", "where", @@ -255,13 +847,18 @@ trait Parser "last_value", "ltrim", "rtrim", - "replace" + "replace", + "on", + "conflict", + "do", + "show", + "describe" ) private val identifierRegexStr = s"""(?i)(?!(?:${reservedKeywords.mkString("|")})\\b)[\\*a-zA-Z_\\-][a-zA-Z0-9_\\-.\\[\\]\\*]*""" - val identifierRegex = identifierRegexStr.r // scala.util.matching.Regex + val identifierRegex: Regex = identifierRegexStr.r // scala.util.matching.Regex def identifier: PackratParser[Identifier] = (Distinct.regex.? ~ identifierRegex ^^ { case d ~ i => @@ -294,7 +891,7 @@ trait Parser fi.identifier.withFunctions(f ++ fi.identifier.functions) case _ => Identifier(f) } - case Some(id) => id.withFunctions(id.functions ++ f) + case Some(id) => id.withFunctions(f ++ id.functions) } }) >> cast diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala index d924e914..2268fb5e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/string/package.scala @@ -125,7 +125,7 @@ package object string { trim | ltrim | rtrim) ^^ { sf => - sf.identifier + Identifier(sf) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala index aead8076..7bb9e950 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/function/time/package.scala @@ -35,30 +35,30 @@ package object time { def parens: PackratParser[List[Delimiter]] = start ~ end ^^ { case s ~ e => s :: e :: Nil } - def current_date: PackratParser[CurrentFunction] = + def current_date: PackratParser[Identifier] = CurrentDate.regex ~ parens.? ^^ { case _ ~ p => - CurrentDate(p.isDefined) + Identifier(CurrentDate(p.isDefined)) } - def current_time: PackratParser[CurrentFunction] = + def current_time: PackratParser[Identifier] = CurrentTime.regex ~ parens.? ^^ { case _ ~ p => - CurrentTime(p.isDefined) + Identifier(CurrentTime(p.isDefined)) } - def current_timestamp: PackratParser[CurrentFunction] = + def current_timestamp: PackratParser[Identifier] = CurrentTimestamp.regex ~ parens.? ^^ { case _ ~ p => - CurrentTimestamp(p.isDefined) + Identifier(CurrentTimestamp(p.isDefined)) } - def now: PackratParser[CurrentFunction] = Now.regex ~ parens.? ^^ { case _ ~ p => - Now(p.isDefined) + def now: PackratParser[Identifier] = Now.regex ~ parens.? ^^ { case _ ~ p => + Identifier(Now(p.isDefined)) } - def today: PackratParser[CurrentFunction] = Today.regex ~ parens.? ^^ { case _ ~ p => - Today(p.isDefined) + def today: PackratParser[Identifier] = Today.regex ~ parens.? ^^ { case _ ~ p => + Identifier(Today(p.isDefined)) } - private[this] def current_function: PackratParser[CurrentFunction] = + private[this] def current_function: PackratParser[Identifier] = current_date | current_time | current_timestamp | now | today def currentFunctionWithIdentifier: PackratParser[Identifier] = diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala index 228673ea..d4b5380c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/type/package.scala @@ -18,11 +18,16 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.{ BooleanValue, + BooleanValues, DoubleValue, + DoubleValues, Identifier, LongValue, + LongValues, + ParamValue, PiValue, StringValue, + StringValues, Value } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} @@ -48,8 +53,29 @@ package object `type` { def boolean: PackratParser[BooleanValue] = """(?i)(true|false)\b""".r ^^ (bool => BooleanValue(bool.toBoolean)) + def param: PackratParser[ParamValue.type] = + "?" ^^ (_ => ParamValue) + + def literals: PackratParser[Value[_]] = "[" ~> repsep(literal, ",") <~ "]" ^^ { list => + StringValues(list) + } + + def longs: PackratParser[Value[_]] = "[" ~> repsep(long, ",") <~ "]" ^^ { list => + LongValues(list) + } + + def doubles: PackratParser[Value[_]] = "[" ~> repsep(double, ",") <~ "]" ^^ { list => + DoubleValues(list) + } + + def booleans: PackratParser[BooleanValues] = "[" ~> repsep(boolean, ",") <~ "]" ^^ { list => + BooleanValues(list) + } + + def array: PackratParser[Value[_]] = literals | longs | doubles | booleans + def value: PackratParser[Value[_]] = - literal | pi | double | long | boolean + literal | pi | double | long | boolean | param | array def identifierWithValue: Parser[Identifier] = (value ^^ functionAsIdentifier) >> cast @@ -87,8 +113,27 @@ package object `type` { def float_type: PackratParser[SQLTypes.Real.type] = "(?i)float|real".r ^^ (_ => SQLTypes.Real) + def struct_type: PackratParser[SQLTypes.Struct.type] = + "(?i)struct".r ^^ (_ => SQLTypes.Struct) + + def array_type: PackratParser[SQLTypes.Array] = + "(?i)array<".r ~> sql_type <~ ">" ^^ { elementType => + SQLTypes.Array(elementType) + } + def sql_type: PackratParser[SQLType] = - char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type + char_type | string_type | datetime_type | timestamp_type | date_type | time_type | boolean_type | long_type | double_type | float_type | int_type | short_type | byte_type | struct_type | array_type + + def text_type: PackratParser[SQLTypes.Text.type] = + "(?i)text".r ^^ (_ => SQLTypes.Text) + + def keyword_type: PackratParser[SQLTypes.Keyword.type] = + "(?i)keyword".r ^^ (_ => SQLTypes.Keyword) + + def geo_point_type: PackratParser[SQLTypes.GeoPoint.type] = + "(?i)(geo_point|geopoint)".r ^^ (_ => SQLTypes.GeoPoint) + def extension_type: PackratParser[SQLType] = + sql_type | text_type | keyword_type | geo_point_type } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala index 94edf995..3ad17781 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/From.scala @@ -46,7 +46,7 @@ case object On extends Expr("ON") with TokenRegex case class On(criteria: Criteria) extends Updateable { override def sql: String = s" $On $criteria" - def update(request: SQLSearchRequest): On = this.copy(criteria = criteria.update(request)) + def update(request: SingleSearch): On = this.copy(criteria = criteria.update(request)) } case object Join extends Expr("JOIN") with TokenRegex @@ -59,7 +59,7 @@ sealed trait Join extends Updateable { override def sql: String = s" ${asString(joinType)} $Join $source${asString(on)}${asString(alias)}" - override def update(request: SQLSearchRequest): Join + override def update(request: SingleSearch): Join override def validate(): Either[String, Unit] = for { @@ -71,8 +71,6 @@ sealed trait Join extends Updateable { _ <- this match { case j if joinType.isDefined && on.isEmpty && joinType.get != CrossJoin => Left(s"JOIN $j requires an ON clause") - case j if joinType.isEmpty && on.isDefined => - Left(s"JOIN $j requires a JOIN type") case j if alias.isEmpty => Left(s"JOIN $j requires an alias") case _ => Right(()) @@ -90,7 +88,7 @@ case class Unnest( ) extends Source with Join { override def sql: String = s"$Join $Unnest($identifier)${asString(alias)}" - def update(request: SQLSearchRequest): Unnest = { + def update(request: SingleSearch): Unnest = { val updated = this.copy( identifier = identifier.withNested(true).update(request), limit = limit.orElse(request.limit) @@ -141,7 +139,8 @@ case class Unnest( case class Table(name: String, tableAlias: Option[Alias] = None, joins: Seq[Join] = Nil) extends Source { override def sql: String = s"$name${asString(tableAlias)} ${joins.map(_.sql).mkString(" ")}".trim - def update(request: SQLSearchRequest): Table = this.copy(joins = joins.map(_.update(request))) + def update(request: SingleSearch): Table = + this.copy(joins = joins.map(_.update(request))) override def validate(): Either[String, Unit] = for { @@ -179,12 +178,14 @@ case class From(tables: Seq[Table]) extends Updateable { (u.alias.map(_.alias).getOrElse(u.name), (u.name, u.limit)) ) .toMap - def update(request: SQLSearchRequest): From = + def update(request: SingleSearch): From = this.copy(tables = tables.map(_.update(request))) override def validate(): Either[String, Unit] = { if (tables.isEmpty) { Left("At least one table is required in FROM clause") + } else if (tables.filter(_.joins.nonEmpty).size > 1) { + Left("Only one table with joins is supported in FROM clause") } else { for { _ <- tables.map(_.validate()).filter(_.isLeft) match { @@ -194,6 +195,8 @@ case class From(tables: Seq[Table]) extends Updateable { } yield () } } + + lazy val mainTable: Table = tables.head } case class NestedElement( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala index 8eb01687..4437309d 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/GroupBy.scala @@ -32,7 +32,7 @@ case object GroupBy extends Expr("GROUP BY") with TokenRegex case class GroupBy(buckets: Seq[Bucket]) extends Updateable { override def sql: String = s" $GroupBy ${buckets.mkString(", ")}" - def update(request: SQLSearchRequest): GroupBy = + def update(request: SingleSearch): GroupBy = this.copy(buckets = buckets.map(_.update(request))) lazy val bucketNames: Map[String, Bucket] = buckets.map { b => b.identifier.identifierName -> b @@ -55,8 +55,10 @@ case class Bucket( size: Option[Int] = None ) extends Updateable with PainlessScript { + def tableAlias: Option[String] = identifier.tableAlias + def table: Option[String] = identifier.table override def sql: String = s"$identifier" - def update(request: SQLSearchRequest): Bucket = { + def update(request: SingleSearch): Bucket = { identifier.functions.headOption match { case Some(func: LongValue) => if (func.value <= 0) { @@ -76,7 +78,7 @@ case class Bucket( lazy val sourceBucket: String = if (identifier.nested) { - identifier.tableAlias + tableAlias .map(a => s"$a.") .getOrElse("") + identifier.name.split("\\.").tail.mkString(".") } else { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala index 07ad6625..243116f7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Having.scala @@ -25,7 +25,7 @@ case class Having(criteria: Option[Criteria]) extends Updateable { case Some(c) => s" $Having $c" case _ => "" } - def update(request: SQLSearchRequest): Having = + def update(request: SingleSearch): Having = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria.map(_.validate()).getOrElse(Right(())) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala index 952a08ad..3f3212dd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/OrderBy.scala @@ -36,7 +36,7 @@ case class FieldSort( lazy val direction: SortOrder = order.getOrElse(Asc) lazy val name: String = field.identifierName override def sql: String = s"$name $direction" - override def update(request: SQLSearchRequest): FieldSort = this.copy( + override def update(request: SingleSearch): FieldSort = this.copy( field = field.update(request) ) def isScriptSort: Boolean = functions.nonEmpty && !hasAggregation && field.fieldAlias.isEmpty @@ -59,5 +59,6 @@ case class OrderBy(sorts: Seq[FieldSort]) extends Updateable { } } yield () - def update(request: SQLSearchRequest): OrderBy = this.copy(sorts = sorts.map(_.update(request))) + def update(request: SingleSearch): OrderBy = + this.copy(sorts = sorts.map(_.update(request))) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala deleted file mode 100644 index 459472e1..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLMultiSearchRequest.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.sql.query - -import app.softnetwork.elastic.sql.Token - -case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends Token { - override def sql: String = s"${requests.map(_.sql).mkString(" UNION ALL ")}" - - def update(): SQLMultiSearchRequest = this.copy(requests = requests.map(_.update())) - - override def validate(): Either[String, Unit] = { - requests.map(_.validate()).filter(_.isLeft) match { - case Nil => Right(()) - case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) - } - } - - lazy val sqlAggregations: Map[String, SQLAggregation] = - requests.flatMap(_.sqlAggregations).distinct.toMap - - lazy val fieldAliases: Map[String, String] = - requests.flatMap(_.fieldAliases).distinct.toMap -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala deleted file mode 100644 index 069aa5eb..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/SQLSearchRequest.scala +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2025 SOFTNETWORK - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package app.softnetwork.elastic.sql.query - -import app.softnetwork.elastic.sql.function.aggregate.WindowFunction -import app.softnetwork.elastic.sql.{asString, Token} - -case class SQLSearchRequest( - select: Select = Select(), - from: From, - where: Option[Where], - groupBy: Option[GroupBy] = None, - having: Option[Having] = None, - orderBy: Option[OrderBy] = None, - limit: Option[Limit] = None, - score: Option[Double] = None -) extends Token { - override def sql: String = - s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}" - - lazy val fieldAliases: Map[String, String] = select.fieldAliases - lazy val tableAliases: Map[String, String] = from.tableAliases - lazy val unnestAliases: Map[String, (String, Option[Limit])] = from.unnestAliases - lazy val bucketNames: Map[String, Bucket] = buckets.flatMap { b => - val name = b.identifier.identifierName - "\\d+".r.findFirstIn(name) match { - case Some(n) if name.trim.split(" ").length == 1 => - val identifier = select.fields(n.toInt - 1).identifier - val updated = b.copy(identifier = select.fields(n.toInt - 1).identifier) - Map( - n -> updated, // also map numeric bucket to field name - identifier.identifierName -> updated - ) - case _ => Map(name -> b) - } - }.toMap - - var unnests: scala.collection.mutable.Map[String, Unnest] = { - val map = from.unnests.map(u => u.alias.map(_.alias).getOrElse(u.name) -> u).toMap - scala.collection.mutable.Map(map.toSeq: _*) - } - - lazy val nestedFields: Map[String, Seq[Field]] = - select.fields - .filterNot(_.isAggregation) - .filter(_.nested) - .groupBy(_.identifier.innerHitsName.getOrElse("")) - lazy val nested: Seq[NestedElement] = - from.unnests.map(toNestedElement).groupBy(_.path).map(_._2.head).toList - private[this] lazy val nestedFieldsWithoutCriteria: Map[String, Seq[Field]] = { - val innerHitsWithCriteria = (where.map(_.nestedElements).getOrElse(Seq.empty) ++ - having.map(_.nestedElements).getOrElse(Seq.empty) ++ - groupBy.map(_.nestedElements).getOrElse(Seq.empty)) - .groupBy(_.path) - .map(_._2.head) - .toList - .map(_.innerHitsName) - val ret = nestedFields.filterNot { case (innerHitsName, _) => - innerHitsWithCriteria.contains(innerHitsName) - } - ret - } - // nested fields that are not part of where, having or group by clauses - lazy val nestedElementsWithoutCriteria: Seq[NestedElement] = - nested.filter(n => nestedFieldsWithoutCriteria.keys.toSeq.contains(n.innerHitsName)) - - def toNestedElement(u: Unnest): NestedElement = { - val updated = unnests.getOrElse(u.alias.map(_.alias).getOrElse(u.name), u) - val parent = updated.parent.map(toNestedElement) - NestedElement( - path = updated.path, - innerHitsName = updated.innerHitsName, - size = limit.map(_.limit), - children = Nil, - sources = nestedFields - .get(updated.innerHitsName) - .map(_.map(_.identifier.name.split('.').tail.mkString("."))) - .getOrElse(Nil), - parent = parent - ) - } - - lazy val sorts: Map[String, SortOrder] = - orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap - - def update(): SQLSearchRequest = { - (for { - from <- Option(this.copy(from = from.update(this))) - select <- Option( - from.copy( - select = select.update(from), - groupBy = groupBy.map(_.update(from)), - having = having.map(_.update(from)) - ) - ) - where <- Option(select.copy(where = where.map(_.update(select)))) - updated <- Option(where.copy(orderBy = orderBy.map(_.update(where)))) - } yield updated).getOrElse( - throw new IllegalStateException("Failed to update SQLSearchRequest") - ) - } - - lazy val scriptFields: Seq[Field] = { - if (aggregates.nonEmpty) - Seq.empty - else - select.fields.filter(_.isScriptField) - } - - lazy val fields: Seq[String] = { - if (groupBy.isEmpty && !windowFunctions.exists(_.isWindowing)) - select.fields - .filterNot(_.isScriptField) - .filterNot(_.nested) - .filterNot(_.isAggregation) - .map(_.sourceField) - .filterNot(f => excludes.contains(f)) - .distinct - else - Seq.empty - } - - lazy val windowFields: Seq[Field] = select.fields.filter(_.identifier.hasWindow) - - lazy val windowFunctions: Seq[WindowFunction] = windowFields.flatMap(_.identifier.windows) - - lazy val aggregates: Seq[Field] = - select.fields - .filter(f => f.isAggregation || f.isBucketScript) - .filterNot(_.identifier.hasWindow) ++ windowFields - - lazy val sqlAggregations: Map[String, SQLAggregation] = - aggregates.flatMap(f => SQLAggregation.fromField(f, this)).map(a => a.aggName -> a).toMap - - lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) - - lazy val sources: Seq[String] = from.tables.map(_.name) - - lazy val bucketTree: BucketTree = BucketTree.fromBuckets( - Seq(groupBy.map(_.buckets).getOrElse(Seq.empty)) ++ windowFunctions.map( - _.buckets - ) - ) - - lazy val buckets: Seq[Bucket] = bucketTree.allBuckets.flatten - - override def validate(): Either[String, Unit] = { - for { - _ <- from.validate() - _ <- select.validate() - _ <- where.map(_.validate()).getOrElse(Right(())) - _ <- groupBy.map(_.validate()).getOrElse(Right(())) - _ <- having.map(_.validate()).getOrElse(Right(())) - _ <- orderBy.map(_.validate()).getOrElse(Right(())) - _ <- limit.map(_.validate()).getOrElse(Right(())) - /*_ <- { - // validate that having clauses are only applied when group by is present - if (having.isDefined && groupBy.isEmpty) { - Left("HAVING clauses can only be applied when GROUP BY is present") - } else { - Right(()) - } - }*/ - _ <- { - // validate that non-aggregated fields are not present when group by is present - if (groupBy.isDefined) { - val nonAggregatedFields = - select.fields.filterNot(f => f.hasAggregation) - val invalidFields = nonAggregatedFields.filterNot(f => - buckets.exists(b => - b.name == f.fieldAlias.map(_.alias).getOrElse(f.sourceField.replace(".", "_")) - ) - ) - if (invalidFields.nonEmpty) { - Left( - s"Non-aggregated fields ${invalidFields.map(_.sql).mkString(", ")} cannot be selected when GROUP BY is present" - ) - } else { - Right(()) - } - } else { - Right(()) - } - } - } yield () - } - -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala index fa3cbd26..45d47194 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Select.scala @@ -45,12 +45,14 @@ case class Field( with FunctionChain with PainlessScript with DateMathScript { + def tableAlias: Option[String] = identifier.tableAlias + def table: Option[String] = identifier.table def isScriptField: Boolean = functions.nonEmpty && !hasAggregation && identifier.bucket.isEmpty override def sql: String = s"$identifier${asString(fieldAlias)}" lazy val sourceField: String = { if (identifier.nested) { - identifier.tableAlias + tableAlias .orElse(fieldAlias.map(_.alias)) .map(a => s"$a.") .getOrElse("") + identifier.name @@ -70,7 +72,7 @@ case class Field( override def functions: List[Function] = identifier.functions - def update(request: SQLSearchRequest): Field = { + def update(request: SingleSearch): Field = { identifier.windows match { case Some(th) => val windowFunction = th.update(request) @@ -93,7 +95,14 @@ case class Field( override def validate(): Either[String, Unit] = identifier.validate() - lazy val nested: Boolean = identifier.nested + def nestedElement: Option[NestedElement] = + identifier.nestedElement + .orElse( + identifier.functionNestedElement + ) + .orElse(this.functionNestedElement) + + lazy val nested: Boolean = nestedElement.isDefined lazy val path: String = identifier.path @@ -104,7 +113,7 @@ case object Except extends Expr("except") with TokenRegex case class Except(fields: Seq[Field]) extends Updateable { override def sql: String = s" $Except(${fields.mkString(",")})" - def update(request: SQLSearchRequest): Except = + def update(request: SingleSearch): Except = this.copy(fields = fields.map(_.update(request))) } @@ -117,7 +126,7 @@ case class Select( lazy val fieldAliases: Map[String, String] = fields.flatMap { field => field.fieldAlias.map(a => field.identifier.identifierName -> a.alias) }.toMap - def update(request: SQLSearchRequest): Select = + def update(request: SingleSearch): Select = this.copy(fields = fields.map(_.update(request)), except = except.map(_.update(request))) override def validate(): Either[String, Unit] = @@ -156,7 +165,7 @@ case class SQLAggregation( } object SQLAggregation { - def fromField(field: Field, request: SQLSearchRequest): Option[SQLAggregation] = { + def fromField(field: Field, request: SingleSearch): Option[SQLAggregation] = { import field._ val aggType = aggregateFunction match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index a983dfa6..58cffbfd 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -101,7 +101,7 @@ sealed trait Criteria extends Updateable with PainlessScript { def limit: Option[Limit] = None - def update(request: SQLSearchRequest): Criteria + def update(request: SingleSearch): Criteria def group: Boolean @@ -155,7 +155,7 @@ case class Predicate( else leftCriteria} $operator${not .map(_ => " NOT") .getOrElse("")} ${if (group) s"$rightCriteria)" else rightCriteria}" - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updatedPredicate = this.copy( leftCriteria = leftCriteria.update(request), rightCriteria = rightCriteria.update(request) @@ -400,7 +400,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { case Some(ctx) => ctx.addParam(identifier) match { case Some(_) => - identifier.baseType match { + identifier.originalType match { case SQLTypes.Any => // in painless context, Any is ZonedDateTime maybeValue.map(_.out).getOrElse(SQLTypes.Any) match { case SQLTypes.Date => @@ -527,7 +527,7 @@ case class GenericExpression( ) extends Expression { override def maybeValue: Option[Token] = Option(value) - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = value match { case id: Identifier => @@ -550,7 +550,7 @@ case class IsNullExpr(identifier: Identifier) extends Expression { override def maybeNot: Option[NOT.type] = None - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -568,7 +568,7 @@ case class IsNotNullExpr(identifier: Identifier) extends Expression { override def maybeNot: Option[NOT.type] = None - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -599,7 +599,7 @@ object ConditionalFunctionAsCriteria { case class IsNullCriteria(identifier: Identifier) extends CriteriaWithConditionalFunction[SQLAny] { override val conditionalFunction: ConditionalFunction[SQLAny] = IsNull(identifier) override val operator: Operator = IS_NULL - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -629,7 +629,7 @@ case class IsNotNullCriteria(identifier: Identifier) identifier ) override val operator: Operator = IS_NOT_NULL - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -666,7 +666,7 @@ case class InExpr[R, +T <: Value[R]]( override def sql = s"$id $notAsString$operator $values" override def operator: Operator = IN - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -706,7 +706,7 @@ case class BetweenExpr( ) extends Expression { override def sql = s"$identifier $notAsString$operator $fromTo" override def operator: Operator = BETWEEN - override def update(request: SQLSearchRequest): Criteria = { + override def update(request: SingleSearch): Criteria = { val updated = this.copy(identifier = identifier.update(request)) if (updated.nested) { ElasticNested(updated, limit) @@ -757,7 +757,7 @@ case class DistanceCriteria( override def sql = s"$distance $operator $geoDistance" - override def update(request: SQLSearchRequest): DistanceCriteria = + override def update(request: SingleSearch): DistanceCriteria = this.copy(distance = distance.update(request)) override def maybeValue: Option[Token] = Some(geoDistance) @@ -775,7 +775,7 @@ case class MultiMatchCriteria( override def sql: String = s"$operator (${identifiers.mkString(",")}) $AGAINST ($value)" override def operator: Operator = MATCH - override def update(request: SQLSearchRequest): Criteria = + override def update(request: SingleSearch): Criteria = this.copy(identifiers = identifiers.map(_.update(request))) override lazy val nested: Boolean = identifiers.forall(_.nested) @@ -816,7 +816,7 @@ case class MatchCriteria( override def sql: String = s"$operator($identifier,$value${options.map(o => s""","$o"""").getOrElse("")})" override def operator: Operator = MATCH - override def update(request: SQLSearchRequest): Criteria = + override def update(request: SingleSearch): Criteria = this.copy(identifier = identifier.update(request)) override def maybeValue: Option[Token] = Some(value) @@ -882,7 +882,7 @@ case class ElasticNested( def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticNested = + override def update(request: SingleSearch): ElasticNested = this.copy(criteria = criteria.update(request)) override def nested: Boolean = true @@ -902,7 +902,7 @@ case class ElasticChild( override val criteria: Criteria ) extends ElasticRelation(criteria, Child) { def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticChild = + override def update(request: SingleSearch): ElasticChild = this.copy(criteria = criteria.update(request)) } @@ -910,7 +910,7 @@ case class ElasticParent( override val criteria: Criteria ) extends ElasticRelation(criteria, Parent) { def nestedElement: Option[NestedElement] = None - override def update(request: SQLSearchRequest): ElasticParent = + override def update(request: SingleSearch): ElasticParent = this.copy(criteria = criteria.update(request)) } @@ -919,7 +919,7 @@ case class Where(criteria: Option[Criteria]) extends Updateable { case Some(c) => s" $Where $c" case _ => "" } - def update(request: SQLSearchRequest): Where = + def update(request: SingleSearch): Where = this.copy(criteria = criteria.map(_.update(request))) override def validate(): Either[String, Unit] = criteria match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala new file mode 100644 index 00000000..4b18a3d4 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -0,0 +1,900 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.schema.{ + sqlConfig, + Column, + IngestPipeline, + IngestPipelineType, + IngestProcessor, + IngestProcessorType, + PartitionDate, + RemoveProcessor, + RenameProcessor, + Schema, + ScriptProcessor, + SetProcessor, + Table => DdlTable +} +import app.softnetwork.elastic.sql.function.aggregate.WindowFunction +import app.softnetwork.elastic.sql.serialization._ +import com.fasterxml.jackson.databind.JsonNode + +import java.time.Instant + +package object query { + sealed trait Statement extends Token + + trait DqlStatement extends Statement + + /** Select Statement wrapper + * @param query + * - the SQL query + * @param score + * - optional minimum score for the elasticsearch query + */ + case class SelectStatement(query: SQL, score: Option[Double] = None) extends DqlStatement { + import app.softnetwork.elastic.sql.SQLImplicits._ + + lazy val statement: Option[DqlStatement] = { + queryToStatement(query) match { + case Some(s: DqlStatement) => Some(s) + case _ => None + } + } + + override def sql: SQL = + statement match { + case Some(value) => value.sql + case None => query + } + + def minScore(score: Double): SelectStatement = this.copy(score = Some(score)) + } + + case class SingleSearch( + select: Select = Select(), + from: From, + where: Option[Where], + groupBy: Option[GroupBy] = None, + having: Option[Having] = None, + orderBy: Option[OrderBy] = None, + limit: Option[Limit] = None, + score: Option[Double] = None, + deleteByQuery: Boolean = false, + updateByQuery: Boolean = false, + onConflict: Option[OnConflict] = None, + schema: Option[Schema] = None + ) extends DqlStatement { + override def sql: String = + s"$select$from${asString(where)}${asString(groupBy)}${asString(having)}${asString(orderBy)}${asString(limit)}${asString(onConflict)}" + + lazy val fieldAliases: Map[String, String] = select.fieldAliases + lazy val tableAliases: Map[String, String] = from.tableAliases + lazy val unnestAliases: Map[String, (String, Option[Limit])] = from.unnestAliases + lazy val bucketNames: Map[String, Bucket] = buckets.flatMap { b => + val name = b.identifier.identifierName + "\\d+".r.findFirstIn(name) match { + case Some(n) if name.trim.split(" ").length == 1 => + val identifier = select.fields(n.toInt - 1).identifier + val updated = b.copy(identifier = select.fields(n.toInt - 1).identifier) + Map( + n -> updated, // also map numeric bucket to field name + identifier.identifierName -> updated + ) + case _ => Map(name -> b) + } + }.toMap + + var unnests: scala.collection.mutable.Map[String, Unnest] = { + val map = from.unnests.map(u => u.alias.map(_.alias).getOrElse(u.name) -> u).toMap + scala.collection.mutable.Map(map.toSeq: _*) + } + + lazy val nestedFields: Map[String, Seq[Field]] = + select.fields + .filterNot(_.isAggregation) + .filter(_.nested) + .groupBy(_.identifier.innerHitsName.getOrElse("")) + lazy val nested: Seq[NestedElement] = + from.unnests.map(toNestedElement).groupBy(_.path).map(_._2.head).toList + private[this] lazy val nestedFieldsWithoutCriteria: Map[String, Seq[Field]] = { + val innerHitsWithCriteria = (where.map(_.nestedElements).getOrElse(Seq.empty) ++ + having.map(_.nestedElements).getOrElse(Seq.empty) ++ + groupBy.map(_.nestedElements).getOrElse(Seq.empty)) + .groupBy(_.path) + .map(_._2.head) + .toList + .map(_.innerHitsName) + val ret = nestedFields.filterNot { case (innerHitsName, _) => + innerHitsWithCriteria.contains(innerHitsName) + } + ret + } + // nested fields that are not part of where, having or group by clauses + lazy val nestedElementsWithoutCriteria: Seq[NestedElement] = + nested.filter(n => nestedFieldsWithoutCriteria.keys.toSeq.contains(n.innerHitsName)) + + def toNestedElement(u: Unnest): NestedElement = { + val updated = unnests.getOrElse(u.alias.map(_.alias).getOrElse(u.name), u) + val parent = updated.parent.map(toNestedElement) + NestedElement( + path = updated.path, + innerHitsName = updated.innerHitsName, + size = limit.map(_.limit), + children = Nil, + sources = nestedFields + .get(updated.innerHitsName) + .map(_.map(_.identifier.name.split('.').tail.mkString("."))) + .getOrElse(Nil), + parent = parent + ) + } + + lazy val sorts: Map[String, SortOrder] = + orderBy.map { _.sorts.map(s => s.name -> s.direction) }.getOrElse(Map.empty).toMap + + def update(schema: Option[Schema] = None): SingleSearch = { + schema match { + case Some(s) => return this.copy(schema = Some(s)).update() + case None => // continue + } + (for { + from <- Option(this.copy(from = from.update(this))) + select <- Option( + from.copy( + select = select.update(from), + groupBy = groupBy.map(_.update(from)), + having = having.map(_.update(from)) + ) + ) + where <- Option(select.copy(where = where.map(_.update(select)))) + updated <- Option(where.copy(orderBy = orderBy.map(_.update(where)))) + } yield updated).getOrElse( + throw new IllegalStateException("Failed to update SQLSearchRequest") + ) + } + + lazy val scriptFields: Seq[Field] = { + if (aggregates.nonEmpty) + Seq.empty + else + select.fields.filter(_.isScriptField) + } + + lazy val fields: Seq[String] = { + if (groupBy.isEmpty && !windowFunctions.exists(_.isWindowing)) + select.fields + .filterNot(_.isScriptField) + .filterNot(_.nested) + .filterNot(_.isAggregation) + .map(_.sourceField) + .filterNot(f => excludes.contains(f)) + .distinct + else + Seq.empty + } + + lazy val windowFields: Seq[Field] = select.fields.filter(_.identifier.hasWindow) + + lazy val windowFunctions: Seq[WindowFunction] = windowFields.flatMap(_.identifier.windows) + + lazy val aggregates: Seq[Field] = + select.fields + .filter(f => f.isAggregation || f.isBucketScript) + .filterNot(_.identifier.hasWindow) ++ windowFields + + lazy val sqlAggregations: Map[String, SQLAggregation] = + aggregates.flatMap(f => SQLAggregation.fromField(f, this)).map(a => a.aggName -> a).toMap + + lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) + + lazy val sources: Seq[String] = from.tables.map(_.name) + + lazy val bucketTree: BucketTree = BucketTree.fromBuckets( + Seq(groupBy.map(_.buckets).getOrElse(Seq.empty)) ++ windowFunctions.map( + _.buckets + ) + ) + + lazy val buckets: Seq[Bucket] = bucketTree.allBuckets.flatten + + override def validate(): Either[String, Unit] = { + for { + _ <- from.validate() + _ <- select.validate() + _ <- where.map(_.validate()).getOrElse(Right(())) + _ <- groupBy.map(_.validate()).getOrElse(Right(())) + _ <- having.map(_.validate()).getOrElse(Right(())) + _ <- orderBy.map(_.validate()).getOrElse(Right(())) + _ <- limit.map(_.validate()).getOrElse(Right(())) + /*_ <- { + // validate that having clauses are only applied when group by is present + if (having.isDefined && groupBy.isEmpty) { + Left("HAVING clauses can only be applied when GROUP BY is present") + } else { + Right(()) + } + }*/ + _ <- { + // validate that non-aggregated fields are not present when group by is present + if (groupBy.isDefined) { + val nonAggregatedFields = + select.fields.filterNot(f => f.hasAggregation) + val invalidFields = nonAggregatedFields.filterNot(f => + buckets.exists(b => + b.name == f.fieldAlias.map(_.alias).getOrElse(f.sourceField.replace(".", "_")) + ) + ) + if (invalidFields.nonEmpty) { + Left( + s"Non-aggregated fields ${invalidFields.map(_.sql).mkString(", ")} cannot be selected when GROUP BY is present" + ) + } else { + Right(()) + } + } else { + Right(()) + } + } + } yield () + } + + } + + case class MultiSearch(requests: Seq[SingleSearch]) extends DqlStatement { + override def sql: String = s"${requests.map(_.sql).mkString(" UNION ALL ")}" + + def update(): MultiSearch = this.copy(requests = requests.map(_.update())) + + override def validate(): Either[String, Unit] = { + requests.map(_.validate()).filter(_.isLeft) match { + case Nil => Right(()) // TODO validate that all requests have the same fields + case errors => Left(errors.map { case Left(err) => err }.mkString("\n")) + } + } + + lazy val sqlAggregations: Map[String, SQLAggregation] = + requests.flatMap(_.sqlAggregations).distinct.toMap + + lazy val fieldAliases: Map[String, String] = + requests.flatMap(_.fieldAliases).distinct.toMap + } + + sealed trait DmlStatement extends Statement + + case class OnConflict(target: Option[Seq[String]], doUpdate: Boolean) extends Token { + override def sql: String = { + val targetSql = + target match { + case Some(t) => if (t.isEmpty) " () " else s" (${t.mkString(", ")}) " + case None => " " + } + val actionSql = if (doUpdate) "DO UPDATE" else "DO NOTHING" + s" ON CONFLICT$targetSql$actionSql" + } + } + + case class Insert( + table: String, + cols: Seq[String], + values: Either[DqlStatement, Seq[Seq[Value[_]]]], + onConflict: Option[OnConflict] = None + ) extends DmlStatement { + lazy val conflictTarget: Option[Seq[String]] = onConflict.flatMap(_.target) + + lazy val doUpdate: Boolean = onConflict.exists(_.doUpdate) + + private def valueToSql(value: Value[_]): String = value match { + case v if v.isInstanceOf[ObjectValues] => + v.asInstanceOf[ObjectValues] + .values + .map(valueToSql) + .mkString("[", ", ", "]") + case v if v.isInstanceOf[ObjectValue] => + v.asInstanceOf[ObjectValue] + .value + .map { case (k, v) => + v match { + case IdValue | IngestTimestampValue => s"""$k = "${v.ddl}"""" + case _ => s"""$k = ${v.ddl}""" + } + } + .mkString("{", ", ", "}") + case v => s"${v.ddl}" + } + + private def rowToSql(row: Seq[Value[_]]): String = { + val rowSql = row + .map(valueToSql) + .mkString(", ") + s"($rowSql)" + } + + override def sql: String = { + values match { + case Left(query) if cols.isEmpty => + s"INSERT INTO $table ${query.sql}${asString(onConflict)}" + case Left(query) => + s"INSERT INTO $table (${cols.mkString(",")}) ${query.sql}${asString(onConflict)}" + case Right(rows) => + val valuesSql = rows + .map(rowToSql) + .mkString(", ") + s"INSERT INTO $table (${cols.mkString(",")}) VALUES $valuesSql${asString(onConflict)}" + } + } + + override def validate(): Either[String, Unit] = { + for { + _ <- values match { + case Left(query) => query.validate() + case Right(rows) => + val invalidRows = rows.filter(_.size != cols.size) + if (invalidRows.nonEmpty) + Left( + s"Some rows have invalid number of values: ${invalidRows + .map(r => s"(${r.map(_.value).mkString(",")})") + .mkString("; ")}" + ) + else Right(()) + case _ => + Right(()) + } + _ <- conflictTarget match { + case Some(target) => + values match { + case Left(query: SingleSearch) => + val queryFields = + query.select.fields.map(f => f.fieldAlias.map(_.alias).getOrElse(f.sourceField)) + if (!target.forall(queryFields.contains)) + Left( + s"Conflict target columns (${target.mkString(",")}) must be part of the inserted columns from SELECT (${queryFields + .mkString(",")})" + ) + else Right(()) + case _ => + if (!target.forall(cols.contains)) + Left( + s"Conflict target columns (${target.mkString(",")}) must be part of the inserted columns (${cols + .mkString(",")})" + ) + else Right(()) + + } + case _ => Right(()) + } + } yield () + } + + def toJson: Option[JsonNode] = { + values match { + case Right(rows) => + val maps: Seq[ObjectValue] = + for (row <- rows) yield { + val map: Map[String, Value[_]] = + cols + .zip(row) + .map { case (k, v) => + k -> v + } + .toMap + ObjectValue(map) + } + val json: JsonNode = ObjectValues(maps) + Some(json) + case _ => None + } + } + } + + case class Update(table: String, values: Map[String, Value[_]], where: Option[Where]) + extends DmlStatement { + override def sql: String = s"UPDATE $table SET ${values + .map { case (k, v) => s"$k = ${v.value}" } + .mkString(", ")}${where.map(w => s" ${w.sql}").getOrElse("")}" + + lazy val customPipeline: IngestPipeline = IngestPipeline( + s"update-$table-${Instant.now}", + IngestPipelineType.Custom, + values.map { case (k, v) => + SetProcessor( + column = k, + value = v + ) + }.toSeq + ) + + } + + case class Delete(table: Table, where: Option[Where]) extends DmlStatement { + override def sql: String = + s"DELETE FROM ${table.name}${asString(where)}" + } + + sealed trait FileFormat extends Token { + def name: String + + override def sql: SQL = s" FILE_FORMAT = $name" + } + + case object Parquet extends FileFormat { + override def name: String = "PARQUET" + } + + case object Json extends FileFormat { + override def name: String = "JSON" + } + + case object JsonArray extends FileFormat { + override def name: String = "JSON_ARRAY" + } + + case object Delta extends FileFormat { + override def name: String = "DELTA_LAKE" + } + + case object Unknown extends FileFormat { + override def name: String = "UNKNOWN" + } + + object FileFormat { + def apply(format: String): FileFormat = { + format.toUpperCase match { + case "PARQUET" => Parquet + case "JSON" => Json + case "JSON_ARRAY" => JsonArray + case "DELTA_LAKE" => Delta + case _ => Unknown + } + } + } + + case class CopyInto( + source: String, + targetTable: String, + fileFormat: Option[FileFormat] = None, + onConflict: Option[OnConflict] = None + ) extends DmlStatement { + override def sql: String = { + s"COPY INTO $targetTable FROM $source${asString(fileFormat)}${asString(onConflict)}" + } + } + + sealed trait DdlStatement extends Statement + + sealed trait PipelineStatement extends DdlStatement + + case class CreatePipeline( + name: String, + pipelineType: IngestPipelineType, + ifNotExists: Boolean = false, + orReplace: Boolean = false, + processors: Seq[IngestProcessor] + ) extends PipelineStatement { + override def sql: String = { + val processorsDdl = processors.map(_.ddl).mkString(", ") + val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" + s"CREATE$replaceClause PIPELINE$ineClause $name WITH PROCESSORS ($processorsDdl)" + } + + lazy val ddlPipeline: IngestPipeline = + IngestPipeline(name, pipelineType, processors) + } + + sealed trait AlterPipelineStatement extends AlterTableStatement + + case class AddPipelineProcessor(processor: IngestProcessor) extends AlterPipelineStatement { + override def sql: String = s"ADD PROCESSOR ${processor.ddl}" + override def ddlProcessor: Option[IngestProcessor] = Some(processor) + } + case class DropPipelineProcessor(processorType: IngestProcessorType, column: String) + extends AlterPipelineStatement { + override def sql: String = s"DROP PROCESSOR ${processorType.name.toUpperCase}($column)" + } + case class AlterPipelineProcessor(processor: IngestProcessor) extends AlterPipelineStatement { + override def sql: String = s"ALTER PROCESSOR ${processor.ddl}" + override def ddlProcessor: Option[IngestProcessor] = Some(processor) + } + + case class AlterPipeline( + name: String, + ifExists: Boolean, + statements: List[AlterPipelineStatement] + ) extends PipelineStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS " else "" + val parenthesesNeeded = statements.size > 1 + val statementsSql = if (parenthesesNeeded) { + statements.map(_.sql).mkString("(\n\t", ",\n\t", "\n)") + } else { + statements.map(_.sql).mkString("") + } + s"ALTER PIPELINE $name$ifExistsClause $statementsSql" + } + + lazy val ddlProcessors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) + + lazy val pipeline: IngestPipeline = + IngestPipeline( + s"alter-pipeline-$name-${Instant.now}", + IngestPipelineType.Custom, + ddlProcessors + ) + } + + case class DropPipeline(name: String, ifExists: Boolean = false) extends PipelineStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + s"DROP PIPELINE $ifExistsClause$name" + } + } + + case class ShowPipeline(name: String) extends PipelineStatement { + override def sql: String = s"SHOW PIPELINE $name" + } + + case class ShowCreatePipeline(name: String) extends PipelineStatement { + override def sql: String = s"SHOW CREATE PIPELINE $name" + } + + case class DescribePipeline(name: String) extends PipelineStatement { + override def sql: String = s"DESCRIBE PIPELINE $name" + } + + sealed trait TableStatement extends DdlStatement + + case class CreateTable( + table: String, + ddl: Either[DqlStatement, List[Column]], + ifNotExists: Boolean = false, + orReplace: Boolean = false, + primaryKey: List[String] = Nil, + partitionBy: Option[PartitionDate] = None, + options: Map[String, Value[_]] = Map.empty + ) extends TableStatement { + + lazy val partitioned: Boolean = partitionBy.isDefined + + override def sql: String = { + val replaceClause = if (orReplace) " OR REPLACE" else "" + val ineClause = if (!orReplace && ifNotExists) " IF NOT EXISTS" else "" + ddl match { + case Left(select) => + s"CREATE$replaceClause TABLE$ineClause $table AS ${select.sql}" + case Right(columns) => + val colsSql = columns.map(_.sql).mkString(", ") + s"CREATE$replaceClause TABLE$ineClause $table ($colsSql)" + } + } + + private val artificialPkColumnName: String = + s"${table}_${sqlConfig.artificialPrimaryKeyColumnName}" + + lazy val columns: Seq[Column] = { + val artificialPkColumn = if (primaryKey.isEmpty) { + Seq( + Column( + name = artificialPkColumnName, + dataType = SQLTypes.Keyword, + defaultValue = Some(IdValue), + comment = Some("Artificial primary key column") + ) + ) + } else { + Nil + } + (ddl match { + case Left(select) => + select match { + case s: SingleSearch => + s.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + case m: MultiSearch => + m.requests.headOption + .map { req => + req.select.fields.map(f => Column(f.identifier.aliasOrName, f.out)) + } + .getOrElse(Nil) + case _ => Nil + } + case Right(cols) => cols + }).filterNot(_.name == artificialPkColumnName) ++ artificialPkColumn + } + + lazy val mappings: Map[String, Value[_]] = options.get("mappings") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + + lazy val settings: Map[String, Value[_]] = options.get("settings") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + + lazy val aliases: Map[String, Value[_]] = options.get("aliases") match { + case Some(value) => + value match { + case o: ObjectValue => o.value + case _ => Map.empty + } + case None => Map.empty + } + + lazy val schema: Schema = DdlTable( + name = table, + columns = columns.toList, + primaryKey = primaryKey, + partitionBy = partitionBy, + mappings = mappings, + settings = settings, + aliases = aliases + ).update() + + lazy val defaultPipeline: IngestPipeline = schema.defaultPipeline + + } + + case class AlterTable(table: String, ifExists: Boolean, statements: List[AlterTableStatement]) + extends TableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS " else "" + val parenthesesNeeded = statements.size > 1 + val statementsSql = if (parenthesesNeeded) { + statements.map(_.sql).mkString("(\n\t", ",\n\t", "\n)") + } else { + statements.map(_.sql).mkString("") + } + s"ALTER TABLE $table$ifExistsClause $statementsSql" + } + + lazy val processors: Seq[IngestProcessor] = statements.flatMap(_.ddlProcessor) + + lazy val pipeline: IngestPipeline = + IngestPipeline(s"alter-$table-${Instant.now}", IngestPipelineType.Custom, processors) + } + + sealed trait AlterTableStatement extends Token { + def ddlProcessor: Option[IngestProcessor] = None + } + case class AddColumn(column: Column, ifNotExists: Boolean = false) extends AlterTableStatement { + override def sql: String = { + val ifNotExistsClause = if (ifNotExists) " IF NOT EXISTS" else "" + s"ADD COLUMN$ifNotExistsClause ${column.sql}" + } + } + case class DropColumn(columnName: String, ifExists: Boolean = false) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"DROP COLUMN$ifExistsClause $columnName" + } + override def ddlProcessor: Option[IngestProcessor] = Some( + RemoveProcessor(column = columnName) + ) + } + case class RenameColumn(oldName: String, newName: String) extends AlterTableStatement { + override def sql: String = s"RENAME COLUMN $oldName TO $newName" + override def ddlProcessor: Option[IngestProcessor] = Some( + RenameProcessor(column = oldName, newName = newName) + ) + } + case class AlterColumnOptions( + columnName: String, + options: Map[String, Value[_]], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET OPTIONS (${options + .map { case (k, v) => s"$k = $v" } + .mkString(", ")})" + } + } + case class AlterColumnOption( + columnName: String, + optionKey: String, + optionValue: Value[_], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET OPTION ($optionKey = $optionValue)" + } + } + case class DropColumnOption( + columnName: String, + optionKey: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP OPTION $optionKey" + } + } + case class AlterColumnType(columnName: String, newType: SQLType, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET TYPE $newType" + } + } + case class AlterColumnScript( + columnName: String, + newScript: ScriptProcessor, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET SCRIPT AS (${newScript.script})" + } + } + case class DropColumnScript(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP SCRIPT" + } + } + case class AlterColumnDefault( + columnName: String, + defaultValue: Value[_], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET DEFAULT $defaultValue" + } + override def ddlProcessor: Option[IngestProcessor] = + Some( + SetProcessor( + column = columnName, + value = defaultValue + ) + ) + } + case class DropColumnDefault(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP DEFAULT" + } + } + case class AlterColumnComment( + columnName: String, + comment: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET COMMENT '$comment'" + } + } + case class DropColumnComment(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP COMMENT" + } + } + case class AlterColumnNotNull(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET NOT NULL" + } + } + case class DropColumnNotNull(columnName: String, ifExists: Boolean = false) + extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP NOT NULL" + } + } + case class AlterColumnFields( + columnName: String, + fields: Seq[Column], + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + val fieldsSql = fields.map(_.sql).mkString("(\n\t\t", ",\n\t\t", "\n\t)") + s"ALTER COLUMN$ifExistsClause $columnName SET FIELDS $fieldsSql" + } + } + case class AlterColumnField( + columnName: String, + field: Column, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName SET FIELD $field" + } + } + case class DropColumnField( + columnName: String, + fieldName: String, + ifExists: Boolean = false + ) extends AlterTableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) " IF EXISTS" else "" + s"ALTER COLUMN$ifExistsClause $columnName DROP FIELD $fieldName" + } + } + case class AlterTableMapping(optionKey: String, optionValue: Value[_]) + extends AlterTableStatement { + override def sql: String = + s"SET MAPPING ($optionKey = $optionValue)" + } + case class DropTableMapping(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP MAPPING $optionKey" + } + case class AlterTableSetting(optionKey: String, optionValue: Value[_]) + extends AlterTableStatement { + override def sql: String = + s"SET SETTING ($optionKey = $optionValue)" + } + case class DropTableSetting(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP SETTING $optionKey" + } + case class AlterTableAlias(optionKey: String, optionValue: Value[_]) extends AlterTableStatement { + override def sql: String = + s"SET ALIAS ($optionKey = $optionValue)" + } + case class DropTableAlias(optionKey: String) extends AlterTableStatement { + override def sql: String = + s"DROP ALIAS $optionKey" + } + + case class DropTable(table: String, ifExists: Boolean = false, cascade: Boolean = false) + extends TableStatement { + override def sql: String = { + val ifExistsClause = if (ifExists) "IF EXISTS " else "" + val cascadeClause = if (cascade) " CASCADE" else "" + s"DROP TABLE $ifExistsClause$table$cascadeClause" + } + } + + case class TruncateTable(table: String) extends TableStatement { + override def sql: String = s"TRUNCATE TABLE $table" + } + + case class ShowTable(table: String) extends TableStatement { + override def sql: String = s"SHOW TABLE $table" + } + + case class ShowCreateTable(table: String) extends TableStatement { + override def sql: String = s"SHOW CREATE TABLE $table" + } + + case class DescribeTable(table: String) extends TableStatement { + override def sql: String = s"DESCRIBE TABLE $table" + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala new file mode 100644 index 00000000..c6f00f33 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/MappingsRules.scala @@ -0,0 +1,54 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +object MappingsRules { + + // Options can be modified without reindexing + private val safeOptions = Set( + "ignore_above", + "null_value", + "store", + "boost", + "coerce", + "copy_to", + "meta" + ) + + // Options requiring a reindex + private val unsafeOptions = Set( + "analyzer", + "search_analyzer", + "normalizer", + "index", + "doc_values", + "format", + "fields", + "similarity", + "eager_global_ordinals" + ) + + def isSafe(key: String): Boolean = { + + if (safeOptions.contains(key)) return true + + if (unsafeOptions.contains(key)) return false + + // Default: cautious β†’ UNSAFE + false + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala new file mode 100644 index 00000000..e990a344 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/SettingsRules.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +object SettingsRules { + + private val dynamicPrefixes = Seq( + "refresh_interval", + "number_of_replicas", + "max_result_window", + "max_inner_result_window", + "max_rescore_window", + "max_docvalue_fields_search", + "max_script_fields", + "max_ngram_diff", + "max_shingle_diff", + "blocks.read_only", + "blocks.read_only_allow_delete", + "blocks.write", + "routing.allocation" + ) + + private val staticPrefixes = Seq( + "number_of_shards", + "codec", + "routing_partition_size", + "analysis", + "similarity", + "sort", + "mapping.total_fields.limit", + "mapping.depth.limit", + "mapping.nested_fields.limit" + ) + + def isDynamic(key: String): Boolean = { + // If the setting is explicitly static β†’ false + if (staticPrefixes.exists(key.contains)) return false + + // If the setting is explicitly dynamic β†’ true + if (dynamicPrefixes.exists(key.contains)) return true + + // Default: cautious β†’ UNSAFE + false + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala new file mode 100644 index 00000000..bf5722b4 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/TableDiff.scala @@ -0,0 +1,273 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.sql.Value +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} +import app.softnetwork.elastic.sql.query._ + +sealed trait DiffSafety +case object Safe extends DiffSafety +case object UnsafeReindex extends DiffSafety +case object Impossible extends DiffSafety + +sealed trait AlterTableStatementDiff { + def stmt: AlterTableStatement + def safety: DiffSafety +} + +sealed trait ColumnDiff extends AlterTableStatementDiff + +case class ColumnAdded(column: Column) extends ColumnDiff { + override def stmt: AlterTableStatement = AddColumn(column) + override def safety: DiffSafety = Safe +} +case class ColumnRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumn(name) + override def safety: DiffSafety = UnsafeReindex +} +// case class ColumnRenamed(oldName: String, newName: String) extends ColumnDiff + +case class ColumnTypeChanged(name: String, from: SQLType, to: SQLType) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnType(name, to) + + override def safety: DiffSafety = + if (SQLTypeUtils.canConvert(from, to)) + UnsafeReindex + else + Impossible +} + +case class ColumnDefaultSet(name: String, value: Value[_]) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnDefault(name, value) + override def safety: DiffSafety = Safe +} +case class ColumnDefaultRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnDefault(name) + override def safety: DiffSafety = Safe +} + +case class ColumnScriptSet(name: String, script: ScriptProcessor) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnScript(name, script) + override def safety: DiffSafety = Safe +} +case class ColumnScriptRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnScript(name) + override def safety: DiffSafety = Safe +} + +case class ColumnCommentSet(name: String, comment: String) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnComment(name, comment) + override def safety: DiffSafety = Safe +} +case class ColumnCommentRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnComment(name) + override def safety: DiffSafety = Safe +} + +case class ColumnNotNullSet(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnNotNull(name) + override def safety: DiffSafety = Safe +} +case class ColumnNotNullRemoved(name: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnNotNull(name) + override def safety: DiffSafety = Safe +} + +case class ColumnOptionSet(name: String, key: String, value: Value[_]) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnOption(name, key, value) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex +} +case class ColumnOptionRemoved(name: String, key: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnOption(name, key) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex +} + +case class FieldAdded(column: String, field: Column) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnField(column, field) + override def safety: DiffSafety = Safe +} +case class FieldRemoved(column: String, fieldName: String) extends ColumnDiff { + override def stmt: AlterTableStatement = DropColumnField(column, fieldName) + override def safety: DiffSafety = Safe +} +case class FieldAltered(column: String, field: Column) extends ColumnDiff { + override def stmt: AlterTableStatement = AlterColumnField(column, field) + override def safety: DiffSafety = Safe +} + +sealed trait MappingDiff extends AlterTableStatementDiff + +case class MappingSet(key: String, value: Value[_]) extends MappingDiff { + override def stmt: AlterTableStatement = AlterTableMapping(key, value) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex +} +case class MappingRemoved(key: String) extends MappingDiff { + override def stmt: AlterTableStatement = DropTableMapping(key) + override def safety: DiffSafety = if (MappingsRules.isSafe(key)) Safe else UnsafeReindex +} + +sealed trait SettingDiff extends AlterTableStatementDiff + +case class SettingSet(key: String, value: Value[_]) extends SettingDiff { + override def stmt: AlterTableStatement = AlterTableSetting(key, value) + override def safety: DiffSafety = + if (SettingsRules.isDynamic(key)) Safe + else UnsafeReindex +} +case class SettingRemoved(key: String) extends SettingDiff { + override def stmt: AlterTableStatement = DropTableSetting(key) + override def safety: DiffSafety = + if (SettingsRules.isDynamic(key)) Safe + else UnsafeReindex +} + +sealed trait AliasDiff extends AlterTableStatementDiff + +case class AliasSet(key: String, value: Value[_]) extends AliasDiff { + override def stmt: AlterTableStatement = AlterTableAlias(key, value) + override def safety: DiffSafety = Safe +} +case class AliasRemoved(key: String) extends AliasDiff { + override def stmt: AlterTableStatement = DropTableAlias(key) + override def safety: DiffSafety = Safe +} + +sealed trait AlterPipelineStatementDiff { + def stmt: AlterPipelineStatement + def safety: DiffSafety = Safe + def processor: IngestProcessor + def pipelineType: IngestPipelineType = processor.pipelineType +} + +sealed trait PipelineDiff extends AlterPipelineStatementDiff + +case class ProcessorAdded(processor: IngestProcessor) extends PipelineDiff { + override def stmt: AlterPipelineStatement = AddPipelineProcessor(processor) +} +case class ProcessorRemoved(processor: IngestProcessor) extends PipelineDiff { + override def stmt: AlterPipelineStatement = + DropPipelineProcessor(processor.processorType, processor.column) +} +case class ProcessorTypeChanged( + actual: IngestProcessorType, + desired: IngestProcessorType +) +sealed trait ProcessorPropertyDiff +case class ProcessorPropertyAdded(key: String, value: Any) extends ProcessorPropertyDiff +case class ProcessorPropertyRemoved(key: String) extends ProcessorPropertyDiff +case class ProcessorPropertyChanged(key: String, from: Any, to: Any) extends ProcessorPropertyDiff +case class ProcessorDiff( + typeChanged: Option[ProcessorTypeChanged], + propertyDiffs: List[ProcessorPropertyDiff] +) +case class ProcessorChanged( + from: IngestProcessor, + to: IngestProcessor, + diff: ProcessorDiff +) extends PipelineDiff { + override def stmt: AlterPipelineStatement = AlterPipelineProcessor(to) + override def pipelineType: IngestPipelineType = from.pipelineType + override def processor: IngestProcessor = to +} + +case class TableDiff( + columns: List[ColumnDiff], + mappings: List[MappingDiff], + settings: List[SettingDiff], + pipeline: List[PipelineDiff], + aliases: List[AliasDiff] +) { + def isEmpty: Boolean = + columns.isEmpty && mappings.isEmpty && settings.isEmpty && pipeline.isEmpty + + def alterTable(tableName: String, ifExists: Boolean): Option[AlterTable] = { + if (isEmpty) { + None + } else { + val statements = columns.map(_.stmt) ++ + mappings.map(_.stmt) ++ + settings.map(_.stmt) ++ + aliases.map(_.stmt) + Some( + AlterTable( + tableName, + ifExists, + statements + ) + ) + } + } + + def defaultPipeline: List[PipelineDiff] = + pipeline.filter(_.pipelineType == IngestPipelineType.Default) + + def finalPipeline: List[PipelineDiff] = + pipeline.filter(_.pipelineType == IngestPipelineType.Final) + + def createPipeline(name: String, pipelineType: IngestPipelineType): Option[CreatePipeline] = { + val pipelineProcessors = pipeline.filter(_.pipelineType == pipelineType).map(_.processor) + if (pipelineProcessors.isEmpty) { + None + } else { + Some( + CreatePipeline( + name, + pipelineType, + ifNotExists = true, + orReplace = false, + pipelineProcessors + ) + ) + } + } + + def alterPipeline(name: String, pipelineType: IngestPipelineType): Option[AlterPipeline] = { + val pipelineStatements = pipeline.filter(_.pipelineType == pipelineType).map(_.stmt) + if (pipelineStatements.isEmpty) { + None + } else { + Some( + AlterPipeline( + name, + ifExists = true, + pipelineStatements + ) + ) + } + } + + def safety: DiffSafety = { + val safeties = + columns.map(_.safety) ++ + mappings.map(_.safety) ++ + settings.map(_.safety) ++ + pipeline.map(_.safety) ++ + aliases.map(_.safety) + + if (safeties.contains(Impossible)) Impossible + else if (safeties.contains(UnsafeReindex)) UnsafeReindex + else Safe + } + + def impossible: Boolean = safety == Impossible + + def requiresReindex: Boolean = safety == UnsafeReindex + + def safe: Boolean = safety == Safe + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala new file mode 100644 index 00000000..5b2ebf93 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -0,0 +1,1719 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.config.ElasticSqlConfig +import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.serialization._ +import app.softnetwork.elastic.sql.time.TimeUnit +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.ObjectNode + +import java.util.UUID +import scala.jdk.CollectionConverters._ +import scala.language.implicitConversions + +package object schema { + val mapper: ObjectMapper = JacksonConfig.objectMapper + + type Schema = Table + + lazy val sqlConfig: ElasticSqlConfig = ElasticSqlConfig() + + sealed trait IngestProcessorType { + def name: String + } + + object IngestProcessorType { + case object Script extends IngestProcessorType { + val name: String = "script" + } + case object Rename extends IngestProcessorType { + val name: String = "rename" + } + case object Remove extends IngestProcessorType { + val name: String = "remove" + } + case object Set extends IngestProcessorType { + val name: String = "set" + } + case object DateIndexName extends IngestProcessorType { + val name: String = "date_index_name" + } + case object Enrich extends IngestProcessorType { + val name: String = "enrich" + } + def apply(n: String): IngestProcessorType = new IngestProcessorType { + override val name: String = n + } + } + + sealed trait IngestProcessor extends DdlToken { + def column: String + def ignoreFailure: Boolean + final def node: ObjectNode = { + val node = mapper.createObjectNode() + node.set(processorType.name, properties) + node + } + def json: String = mapper.writeValueAsString(node) + def pipelineType: IngestPipelineType + def processorType: IngestProcessorType + def sql: String // = s"${processorType.name.toUpperCase}${Value(properties).ddl}" + def description: Option[String] + def name: String = processorType.name + def properties: Map[String, Any] + + private def normalizeValue(v: Any): Any = v match { + case s: String => s.trim + case b: Boolean => b + case i: Int => i + case l: Long => l + case d: Double => d + case o: ObjectNode => mapper.writeValueAsString(o) // canonical JSON + case seq: Seq[_] => seq.map(normalizeValue) // recursive + case other => other + } + + private def normalizeProperties(): Map[String, Any] = { + properties + .filterNot { case (k, _) => k == "description" } // we should not compare description + .toSeq + .sortBy(_._1) // order properties by key + .map { case (k, v) => (k, normalizeValue(v)) } + .toMap + } + + def diff(to: IngestProcessor): Option[ProcessorDiff] = { + + val from = this + + // 1. Diff of the type + val typeChanged: Option[ProcessorTypeChanged] = + if (from.processorType != to.processorType) + Some(ProcessorTypeChanged(from.processorType, to.processorType)) + else + None + + // 2. Diff of the properties + val fromProps = from.normalizeProperties() + val toProps = to.normalizeProperties() + + val allKeys = fromProps.keySet ++ toProps.keySet + + val propertyDiffs: List[ProcessorPropertyDiff] = + allKeys.toList.flatMap { key => + (fromProps.get(key), toProps.get(key)) match { + + case (None, Some(v)) => + Some(ProcessorPropertyAdded(key, v)) + + case (Some(_), None) => + Some(ProcessorPropertyRemoved(key)) + + case (Some(v1), Some(v2)) if v1 != v2 => + Some(ProcessorPropertyChanged(key, v1, v2)) + + case _ => + None + } + } + + // 3. If nothing has changed β†’ None + if (typeChanged.isEmpty && propertyDiffs.isEmpty) + None + else + Some(ProcessorDiff(typeChanged, propertyDiffs)) + } + + override def ddl: String = s"${processorType.name.toUpperCase}${Value(properties).ddl}" + } + + object IngestProcessor { + private val ScriptDescRegex = + """^\s*([a-zA-Z0-9_\\.]+)\s([a-zA-Z]+)\s+SCRIPT\s+AS\s*\((.*)\)\s*$""".r + + def apply(processorType: IngestProcessorType, properties: ObjectValue): IngestProcessor = { + val node = mapper.createObjectNode() + node.set(processorType.name, properties.toJson) + apply(IngestPipelineType.Default, node) + } + + def apply(pipelineType: IngestPipelineType, processor: JsonNode): IngestProcessor = { + val processorType = processor.fieldNames().next() // "set", "script", "date_index_name", etc. + val props = processor.get(processorType) + + processorType match { + case IngestProcessorType.Set.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val valueNode = Option(props.get("value")) + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + if (field == "_id" && valueNode.isDefined) { + val value = + valueNode.get + .asText() + .trim + .stripPrefix("{{") + .stripSuffix("}}") + .trim + // DdlPrimaryKeyProcessor + val cols = value.split(sqlConfig.compositeKeySeparator).toSet + PrimaryKeyProcessor( + pipelineType = pipelineType, + description = desc, + column = "_id", + value = cols, + ignoreFailure = ignoreFailure + ) + } else { + val copyFrom = Option(props.get("copy_from")).map(_.asText()) + val doOverride = Option(props.get("override")).map(_.asBoolean()) + val ignoreEmptyValue = Option(props.get("ignore_empty_value")).map(_.asBoolean()) + val doIf = + if (props.has("if")) { + val cond = props.get("if") + if (cond.has("source")) { + Option(cond.get("source")).map(_.asText()) + } else Option(cond).map(_.asText()) + } else None + SetProcessor( + pipelineType = pipelineType, + description = desc, + column = field, + value = valueNode.map(v => Value(v.asText())).getOrElse(Null), + copyFrom = copyFrom, + doOverride = doOverride, + ignoreEmptyValue = ignoreEmptyValue, + doIf = doIf match { + case Some(condition) if condition.nonEmpty => Some(condition) + case _ => None + }, + ignoreFailure = ignoreFailure + ) + } + + case IngestProcessorType.Script.name => + val desc = Option(props.get("description")).map(_.asText()) + val lang = props.get("lang").asText() + require(lang == "painless", s"Only painless supported, got $lang") + val source = props.get("source").asText() + val ignoreFailure = Option(props.get("ignore_failure")).exists(_.asBoolean()) + + desc match { + case Some(ScriptDescRegex(col, dataType, script)) => + ScriptProcessor( + pipelineType = pipelineType, + description = desc, + script = script, + column = col, + dataType = SQLTypes(dataType), + source = source, + ignoreFailure = ignoreFailure + ) + case _ => + GenericProcessor( + pipelineType = pipelineType, + processorType = IngestProcessorType.Script, + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) + } + + case IngestProcessorType.DateIndexName.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val rounding = props.get("date_rounding").asText() + val formats = Option(props.get("date_formats")) + .map(_.elements().asScala.toList.map(_.asText())) + .getOrElse(Nil) + val prefix = props.get("index_name_prefix").asText() + + DateIndexNameProcessor( + pipelineType = pipelineType, + description = desc, + column = field, + dateRounding = rounding, + dateFormats = formats, + prefix = prefix + ) + + case IngestProcessorType.Remove.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + + RemoveProcessor( + pipelineType = pipelineType, + description = desc, + column = field + ) + + case IngestProcessorType.Rename.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val targetField = props.get("target_field").asText() + + RenameProcessor( + pipelineType = pipelineType, + description = desc, + column = field, + newName = targetField + ) + + case IngestProcessorType.Enrich.name => + val field = props.get("field").asText() + val desc = Option(props.get("description")).map(_.asText()) + val policyName = props.get("policy_name").asText() + val targetField = props.get("target_field").asText() + val maxMatches = Option(props.get("max_matches")).map(_.asInt()).getOrElse(1) + EnrichProcessor( + pipelineType = pipelineType, + description = desc, + column = targetField, + policyName = policyName, + field = field, + maxMatches = maxMatches + ) + + case other => + GenericProcessor( + pipelineType = pipelineType, + processorType = IngestProcessorType(other), + properties = + mapper.convertValue(props, classOf[java.util.Map[String, Object]]).asScala.toMap + ) + + } + } + } + + case class GenericProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + processorType: IngestProcessorType, + properties: Map[String, Any] + ) extends IngestProcessor { + override def description: Option[String] = properties.get("description") match { + case Some(s: String) => Some(s) + case _ => None + } + + override def sql: String = ddl + override def column: String = properties.get("field") match { + case Some(s: String) => s + case _ => UUID.randomUUID().toString + } + override def ignoreFailure: Boolean = properties.get("ignore_failure") match { + case Some(b: Boolean) => b + case _ => false + } + } + + case class ScriptProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + script: String, + column: String, + dataType: SQLType, + source: String, + ignoreFailure: Boolean = true + ) extends IngestProcessor { + override def sql: String = s"$column $dataType SCRIPT AS ($script)" + + override def baseType: SQLType = dataType + + def processorType: IngestProcessorType = IngestProcessorType.Script + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "lang" -> "painless", + "source" -> source, + "ignore_failure" -> ignoreFailure + ) + + } + + case class RenameProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + newName: String, + ignoreFailure: Boolean = true, + ignoreMissing: Option[Boolean] = None + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Rename + + override def sql: String = s"$column RENAME TO $newName" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "target_field" -> newName, + "ignore_failure" -> ignoreFailure + ) ++ ignoreMissing + .map("ignore_missing" -> _) + .toMap + + } + + case class RemoveProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + ignoreFailure: Boolean = true, + ignoreMissing: Option[Boolean] = None + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Remove + + override def sql: String = s"REMOVE $column" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "ignore_failure" -> ignoreFailure + ) ++ ignoreMissing + .map("ignore_missing" -> _) + .toMap + + } + + case class PrimaryKeyProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + value: Set[String], + ignoreFailure: Boolean = false, + ignoreEmptyValue: Option[Boolean] = Some(false), + separator: String = "\\|\\|" + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Set + + override def sql: String = s"PRIMARY KEY (${value.mkString(", ")})" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "value" -> value.mkString("{{", separator, "}}"), + "ignore_failure" -> ignoreFailure + ) ++ ignoreEmptyValue + .map("ignore_empty_value" -> _) + .toMap + + } + + case class SetProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + value: Value[_], + copyFrom: Option[String] = None, + doOverride: Option[Boolean] = None, + ignoreEmptyValue: Option[Boolean] = None, + doIf: Option[String] = None, + ignoreFailure: Boolean = true + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Set + + override def sql: String = { + val base = copyFrom match { + case Some(source) => s"$column COPY FROM $source" + case None => s"$column SET VALUE ${value.sql}" + } + val withOverride = doOverride match { + case Some(overrideValue) => s"$base OVERRIDE $overrideValue" + case None => base + } + val withIgnoreEmpty = ignoreEmptyValue match { + case Some(ignoreEmpty) => s"$withOverride IGNORE EMPTY VALUE $ignoreEmpty" + case None => withOverride + } + val withIf = doIf match { + case Some(condition) => s"$withIgnoreEmpty IF $condition" + case None => withIgnoreEmpty + } + withIf + } + + lazy val defaultValue: Option[Any] = { + if (copyFrom.isDefined) None + else + value match { + case IdValue | IngestTimestampValue => Some(s"{{${value.value}}}") + case Null => None + case _ => Some(value.value) + } + } + + override def properties: Map[String, Any] = { + Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "ignore_failure" -> ignoreFailure + ) ++ defaultValue.map("value" -> _) ++ + copyFrom.map("copy_from" -> _) ++ + doOverride.map("override" -> _) ++ + doIf.map("if" -> _) ++ + ignoreEmptyValue.map("ignore_empty_value" -> _) + } + + override def validate(): Either[String, Unit] = { + for { + _ <- + if (value != Null && copyFrom.isDefined) + Left( + s"Only one of value or copy_from should be defined for SET processor on column $column" + ) + else Right(()) + } yield () + } + } + + case class DateIndexNameProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + dateRounding: String, + dateFormats: List[String], + prefix: String, + ignoreFailure: Boolean = true + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.DateIndexName + + override def sql: String = s"PARTITION BY $column (${TimeUnit(dateRounding).sql})" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "field" -> column, + "date_rounding" -> dateRounding, + "date_formats" -> dateFormats, + "index_name_prefix" -> prefix, + "ignore_failure" -> ignoreFailure + ) + + } + + implicit def primaryKeyToDdlProcessor( + primaryKey: List[String] + ): Seq[IngestProcessor] = { + if (primaryKey.nonEmpty) { + Seq( + PrimaryKeyProcessor( + column = "_id", + value = primaryKey.toSet, + separator = sqlConfig.compositeKeySeparator + ) + ) + } else { + Nil + } + } + + case class EnrichProcessor( + pipelineType: IngestPipelineType = IngestPipelineType.Default, + description: Option[String] = None, + column: String, + policyName: String, + field: String, + maxMatches: Int = 1, + ignoreFailure: Boolean = true + ) extends IngestProcessor { + def processorType: IngestProcessorType = IngestProcessorType.Enrich + + def targetField: String = column + + override def sql: String = s"ENRICH USING POLICY $policyName FROM $field INTO $targetField" + + override def properties: Map[String, Any] = Map( + "description" -> description.getOrElse(sql), + "policy_name" -> policyName, + "field" -> field, + "target_field" -> targetField, + "max_matches" -> maxMatches, + "ignore_failure" -> ignoreFailure + ) + + } + + sealed trait IngestPipelineType { + def name: String + } + + object IngestPipelineType { + case object Default extends IngestPipelineType { + val name: String = "DEFAULT" + } + case object Final extends IngestPipelineType { + val name: String = "FINAL" + } + case object Custom extends IngestPipelineType { + val name: String = "CUSTOM" + } + } + + case class IngestPipeline( + name: String, + pipelineType: IngestPipelineType, + processors: Seq[IngestProcessor] + ) extends DdlToken { + def sql: String = + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${processors.map(_.sql.trim).mkString(", ")})" + + def node: ObjectNode = { + val node = mapper.createObjectNode() + val processorsNode = mapper.createArrayNode() + processors.foreach { processor => + processorsNode.add(processor.node) + } + node.put("description", sql) + node.set("processors", processorsNode) + node + } + + override def ddl: String = + s"CREATE OR REPLACE PIPELINE $name WITH PROCESSORS (${processors.map(_.ddl.trim).mkString(", ")})" + + def json: String = mapper.writeValueAsString(node) + + def diff(pipeline: IngestPipeline): List[PipelineDiff] = { + + val actual = this.processors + + val desired = pipeline.processors + + // 1. Index processors by logical key + def key(p: IngestProcessor) = s"${p.pipelineType.name}-${p.processorType.name}-${p.column}" + + val desiredMap = desired.map(p => key(p) -> p).toMap + val actualMap = actual.map(p => key(p) -> p).toMap + + val diffs = scala.collection.mutable.ListBuffer[PipelineDiff]() + + // 2. Added processors + for ((k, p) <- desiredMap if !actualMap.contains(k)) { + diffs += ProcessorAdded(p) + } + + // 3. Removed processors + for ((k, p) <- actualMap if !desiredMap.contains(k)) { + diffs += ProcessorRemoved(p) + } + + // 4. Modified processors + for ((k, from) <- actualMap if desiredMap.contains(k)) { + val to = desiredMap(k) + from.diff(to) match { + case Some(d) => diffs += ProcessorChanged(from, to, d) + case None => // identical β†’ nothing + } + } + + // 5. Optional: detect reordering for processors where order matters + diffs.toList + } + + def merge(statements: Seq[AlterPipelineStatement]): IngestPipeline = { + statements.foldLeft(this) { (current, alter) => + alter match { + case AddPipelineProcessor(processor) => + current.copy(processors = + current.processors.filterNot(p => + p.processorType == processor.processorType && p.column == processor.column + ) :+ processor + ) + case DropPipelineProcessor(processorType, column) => + current.copy(processors = + current.processors.filterNot(p => + p.processorType == processorType && p.column == column + ) + ) + case AlterPipelineProcessor(processor) => + current.copy(processors = current.processors.map { p => + if (p.processorType == processor.processorType && p.column == processor.column) { + processor + } else { + p + } + }) + } + } + } + + def merge(other: IngestPipeline): IngestPipeline = { + other.processors.foldLeft(this) { (current, processor) => + current.copy(processors = + current.processors.filterNot(p => + p.processorType == processor.processorType && p.column == processor.column + ) :+ processor + ) + } + } + } + + object IngestPipeline { + def apply( + name: String, + json: String, + pipelineType: Option[IngestPipelineType] = None + ): IngestPipeline = { + val ddlPipelineType = pipelineType.getOrElse( + if (name.startsWith("default-")) IngestPipelineType.Default + else if (name.startsWith("final-")) IngestPipelineType.Final + else IngestPipelineType.Custom + ) + val node = mapper.readTree(json) + val processorsNode = node.get("processors") + val processors = processorsNode.elements().asScala.toSeq.map { p => + IngestProcessor(ddlPipelineType, p) + } + IngestPipeline( + name = name, + pipelineType = ddlPipelineType, + processors = processors + ) + } + } + + case class Column( + name: String, + dataType: SQLType, + script: Option[ScriptProcessor] = None, + multiFields: List[Column] = Nil, + defaultValue: Option[Value[_]] = None, + notNull: Boolean = false, + comment: Option[String] = None, + options: Map[String, Value[_]] = Map.empty, + struct: Option[Column] = None + ) extends DdlToken { + def path: String = struct.map(st => s"${st.name}.$name").getOrElse(name) + private def level: Int = struct.map(_.level + 1).getOrElse(0) + + private def cols: Map[String, Column] = multiFields.map(field => field.name -> field).toMap + + /* Recursive find */ + def find(path: String): Option[Column] = { + if (path.contains(".")) { + val parts = path.split("\\.") + cols.get(parts.head).flatMap(col => col.find(parts.tail.mkString("."))) + } else { + cols.get(path) + } + } + + def _meta: Map[String, Value[_]] = { + Map( + "data_type" -> StringValue(dataType.typeId), + "not_null" -> StringValue(s"$notNull") + ) ++ defaultValue.map(d => "default_value" -> d) ++ comment.map(ct => + "comment" -> StringValue(ct) + ) ++ script + .map { sc => + ObjectValue( + Map( + "sql" -> StringValue(sc.script), + "column" -> StringValue(path), + "painless" -> StringValue(sc.source) + ) + ) + } + .map("script" -> _) ++ Map( + "multi_fields" -> ObjectValue( + multiFields.map(field => field.name -> ObjectValue(field._meta)).toMap + ) + ) + } + + def updateStruct(): Column = { + struct + .map { st => + val updated = st.copy(multiFields = st.multiFields.filterNot(_.name == this.name) :+ this) + updated.updateStruct() + } + .getOrElse(this) + .update() + } + + def update(struct: Option[Column] = None): Column = { + val updated = this.copy(struct = struct) + // update script accordingly with new path + val updated_script = + script.map { sc => + sc.copy( + column = updated.path, + source = sc.source.replace(s"ctx.$name", s"ctx.${updated.path}") + ) + } + updated.copy( + multiFields = multiFields.map { field => + field.update(Some(updated)) + }, + script = updated_script, + options = options + ) + } + + def sql: String = { + val opts = if (options.nonEmpty) { + s" OPTIONS ${ObjectValue(options).ddl}" + } else { + "" + } + val defaultOpt = defaultValue.map(v => s" DEFAULT ${v.sql}").getOrElse("") + val notNullOpt = if (notNull) " NOT NULL" else "" + val commentOpt = comment.map(c => s" COMMENT '$c'").getOrElse("") + val fieldsOpt = if (multiFields.nonEmpty) { + s" FIELDS (\n\t${multiFields.mkString(s",\n\t")}\n\t)" + } else { + "" + } + val scriptOpt = script.map(s => s" SCRIPT AS (${s.script})").getOrElse("") + val tabs = "\t" * level + s"$tabs$name $dataType$fieldsOpt$scriptOpt$defaultOpt$notNullOpt$commentOpt$opts" + } + + def asMap: Seq[Map[String, Any]] = Seq( + Map( + "name" -> path, + "type" -> dataType.typeId, + "script" -> script.map(_.script), + "default" -> defaultValue.map(_.value).getOrElse(""), + "notNull" -> notNull, + "comment" -> comment.getOrElse(""), + "options" -> ObjectValue(options).ddl + ) + ) ++ multiFields.flatMap(_.asMap) + + def processors: Seq[IngestProcessor] = script.map(st => st.copy(column = path)).toSeq ++ + defaultValue.map { dv => + SetProcessor( + column = path, + value = dv, + doIf = Some { + if (path.contains(".")) + s"""ctx.${path.split("\\.").mkString("?.")} == null""" + else + s"""ctx.$path == null""" + } + ) + }.toSeq ++ multiFields.flatMap(_.processors) + + def node: ObjectNode = { + val root = mapper.createObjectNode() + val esType = SQLTypeUtils.elasticType(dataType) + root.put("type", esType) + dataType match { + case SQLTypes.Varchar | SQLTypes.Text => // do not set null_value for text types + case _ => + defaultValue.foreach { + case IngestTimestampValue => () // do not set null_value for ingest timestamp + case dv => // set null_value for other types + updateNode(root, Map("null_value" -> dv)) + } + } + if (multiFields.nonEmpty) { + val name = + esType match { + case "object" | "nested" => "properties" + case _ => "fields" + } + val fieldsNode = mapper.createObjectNode() + multiFields.foreach { field => + fieldsNode.replace(field.name, field.node) + } + root.set(name, fieldsNode) + } + updateNode(root, options) + root + } + + def diff(desired: Column, parent: Option[Column] = None): List[ColumnDiff] = { + val actual = this + val diffs = scala.collection.mutable.ListBuffer[ColumnDiff]() + + // 1. Type + if (SQLTypeUtils.elasticType(actual.dataType) != SQLTypeUtils.elasticType(desired.dataType)) { + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnTypeChanged(path, actual.dataType, desired.dataType) + } + } + + // 2. Default + (actual.defaultValue, desired.defaultValue) match { + case (None, Some(v)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + diffs += ColumnDefaultSet(path, v) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnDefaultRemoved(path) + } + case (Some(a), Some(b)) if a != b => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnDefaultSet(path, b) + } + case _ => + } + + // 3. Script + (actual.script, desired.script) match { + case (None, Some(s)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + diffs += ColumnScriptSet(path, s) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnScriptRemoved(path) + } + case (Some(a), Some(b)) if a.sql != b.sql => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnScriptSet(path, b) + } + case _ => + } + + // 4. Comment + (actual.comment, desired.comment) match { + case (None, Some(c)) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentSet(path, c) + } + case (Some(_), None) => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentRemoved(path) + } + case (Some(a), Some(b)) if a != b => + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => diffs += ColumnCommentSet(path, b) + } + case _ => + } + + // 5. Not Null + if (actual.notNull != desired.notNull) { + parent match { + case Some(p) => + diffs += FieldAltered( + p.path, + desired + ) + case None => + if (desired.notNull) diffs += ColumnNotNullSet(path) + else diffs += ColumnNotNullRemoved(path) + } + } + + // 6. Options + val optionDiffs = ObjectValue(actual.options).diff(ObjectValue(desired.options)) + parent match { + case Some(p) => + if (optionDiffs.nonEmpty) { + diffs += FieldAltered( + p.path, + desired + ) + } + case None => + diffs ++= optionDiffs.map { + case Altered(name, value) => ColumnOptionSet(path, name, value) + case Removed(name) => ColumnOptionRemoved(path, name) + } + } + + // 7. STRUCT / multi-fields + val actualFields = actual.multiFields.map(f => f.name -> f).toMap + val desiredFields = desired.multiFields.map(f => f.name -> f).toMap + + // 7.1. Fields added + for ((name, f) <- desiredFields if !actualFields.contains(name)) { + diffs += FieldAdded(path, f) + } + + // 7.2. Fields removed + for ((name, f) <- actualFields if !desiredFields.contains(name)) { + diffs += FieldRemoved(path, name) + } + + // 7.3. Fields modified + for ((name, a) <- actualFields if desiredFields.contains(name)) { + val d = desiredFields(name) + diffs ++= a.diff(d, Some(desired)) + } + + diffs.toList + } + } + + case class PartitionDate(column: String, granularity: TimeUnit = TimeUnit.DAYS) extends DdlToken { + def sql: String = s" PARTITION BY $column ($granularity)" + + val dateRounding: String = granularity.script.get + + val dateFormats: List[String] = granularity match { + case TimeUnit.YEARS => List("yyyy") + case TimeUnit.MONTHS => List("yyyy-MM") + case TimeUnit.DAYS => List("yyyy-MM-dd") + case TimeUnit.HOURS => List("yyyy-MM-dd'T'HH", "yyyy-MM-dd HH") + case TimeUnit.MINUTES => + List("yyyy-MM-dd'T'HH:mm", "yyyy-MM-dd HH:mm") + case TimeUnit.SECONDS => + List("yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd HH:mm:ss") + case _ => List.empty + } + + def processor(table: Table): DateIndexNameProcessor = + DateIndexNameProcessor( + column = column, + dateRounding = dateRounding, + dateFormats = dateFormats, + prefix = s"${table.name}-" + ) + } + + case class ColumnNotFound(column: String, table: String) + extends Exception(s"Column $column does not exist in table $table") + + case class TableAlias( + table: String, + alias: String, + filter: Map[String, Any] = Map.empty, + routing: Option[String] = None, + indexRouting: Option[String] = None, + searchRouting: Option[String] = None, + isWriteIndex: Boolean = false, + isHidden: Boolean = false + ) { + + def node: ObjectNode = { + val node = mapper.createObjectNode() + if (filter.nonEmpty) { + val filterNode = mapper.valueToTree[JsonNode](filter.asJava) + node.set("filter", filterNode) + } + routing.foreach(r => node.put("routing", r)) + indexRouting.foreach(r => node.put("index_routing", r)) + searchRouting.foreach(r => node.put("search_routing", r)) + if (isWriteIndex) { + node.put("is_write_index", true) + } + if (isHidden) { + node.put("is_hidden", true) + } + node + } + + def json: String = node + + def ddl: String = { + val opts = scala.collection.mutable.ListBuffer[String]() + if (filter.nonEmpty) { + opts += s"filter = ${Value(filter).ddl}" + } + routing.foreach(r => opts += s"routing = '$r'") + indexRouting.foreach(r => opts += s"index_routing = '$r'") + searchRouting.foreach(r => opts += s"search_routing = '$r'") + if (isWriteIndex) { + opts += s"is_write_index = true" + } + if (isHidden) { + opts += s"is_hidden = true" + } + s"ALTER TABLE $table SET ALIAS ($alias = ${if (opts.nonEmpty) s"(${opts.mkString(", ")})" + else "()"})" + } + } + + object TableAlias { + def apply(table: String, alias: String, value: Value[_]): TableAlias = { + val obj = value.asInstanceOf[ObjectValue] + val filter = obj.value.get("filter") match { + case Some(ObjectValue(f)) => + f.map { case (k, v) => k -> v.value } + case _ => Map.empty[String, Any] + } + val routing = obj.value.get("routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val indexRouting = obj.value.get("index_routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val searchRouting = obj.value.get("search_routing") match { + case Some(StringValue(r)) => Some(r) + case _ => None + } + val isWriteIndex = obj.value.get("is_write_index") match { + case Some(BooleanValue(b)) => b + case _ => false + } + val isHidden = obj.value.get("is_hidden") match { + case Some(BooleanValue(b)) => b + case _ => false + } + TableAlias( + table = table, + alias = alias, + filter = filter, + routing = routing, + indexRouting = indexRouting, + searchRouting = searchRouting, + isWriteIndex = isWriteIndex, + isHidden = isHidden + ) + } + } + + /** Definition of a table within the schema + * + * @param name + * the table name + * @param columns + * the list of columns within the table + * @param primaryKey + * optional list of columns composing the primary key + * @param partitionBy + * optional partition by date definition + * @param mappings + * optional index mappings + * @param settings + * optional index settings + * @param processors + * optional list of ingest processors associated to this table (apart from column processors) + * @param aliases + * optional map of aliases associated to this table + */ + case class Table( + name: String, + columns: List[Column], + primaryKey: List[String] = Nil, + partitionBy: Option[PartitionDate] = None, + mappings: Map[String, Value[_]] = Map.empty, + settings: Map[String, Value[_]] = Map.empty, + processors: Seq[IngestProcessor] = Seq.empty, + aliases: Map[String, Value[_]] = Map.empty + ) extends DdlToken { + lazy val indexName: String = name.toLowerCase + private[schema] lazy val cols: Map[String, Column] = columns.map(c => c.name -> c).toMap + + def find(path: String): Option[Column] = { + if (path.contains(".")) { + val parts = path.split("\\.") + cols.get(parts.head).flatMap(col => col.find(parts.tail.mkString("."))) + } else { + cols.get(path) + } + } + + private lazy val _meta: Map[String, Value[_]] = Map.empty ++ { + if (primaryKey.nonEmpty) + Option("primary_key" -> StringValues(primaryKey.map(StringValue))) + else + None + }.toMap ++ partitionBy + .map(pb => + Map( + "partition_by" -> ObjectValue( + Map( + "column" -> StringValue(pb.column), + "granularity" -> StringValue(pb.granularity.script.get) + ) + ) + ) + ) + .getOrElse(Map.empty) ++ + Map("columns" -> ObjectValue(cols.map { case (name, col) => name -> ObjectValue(col._meta) })) + + def update(): Table = { + val updated = + this.copy(columns = columns.map(_.update())) // update columns first with struct info + updated.copy( + mappings = updated.mappings ++ Map( + "_meta" -> + ObjectValue(updated.mappings.get("_meta") match { + case Some(ObjectValue(value)) => + (value - "primary_key" - "partition_by" - "columns") ++ updated._meta + case _ => updated._meta + }) + ) + ) + } + + def sql: String = { + val opts = + if (mappings.nonEmpty || settings.nonEmpty) { + val mappingOpts = + if (mappings.nonEmpty) { + s"mappings = ${ObjectValue(mappings).ddl}" + } else { + "" + } + val settingsOpts = + if (settings.nonEmpty) { + s"settings = ${ObjectValue(settings).ddl}" + } else { + "" + } + val aliasesOpts = + if (aliases.nonEmpty) { + s"aliases = ${ObjectValue(aliases).ddl}" + } else { + "" + } + val separator = if (partitionBy.nonEmpty) "," else "" + s"$separator OPTIONS = (${Seq(mappingOpts, settingsOpts, aliasesOpts).filter(_.nonEmpty).mkString(", ")})" + } else { + "" + } + val cols = columns.map(_.sql).mkString(",\n\t") + val pkStr = if (primaryKey.nonEmpty) { + s",\n\tPRIMARY KEY (${primaryKey.mkString(", ")})\n" + } else { + "" + } + s"CREATE OR REPLACE TABLE $name (\n\t$cols$pkStr)${partitionBy.getOrElse("")}$opts" + } + + def tableProcessors: Seq[IngestProcessor] = + columns.flatMap(_.processors) ++ partitionBy + .map(_.processor(this)) + .toSeq ++ implicitly[Seq[IngestProcessor]](primaryKey) + + def merge(statements: Seq[AlterTableStatement]): Table = { + statements + .foldLeft(this) { (table, statement) => + statement match { + // table columns + case AddColumn(column, ifNotExists) => + if (ifNotExists && table.cols.contains(column.name)) table + else if (!table.cols.contains(column.name)) + table.copy(columns = table.columns :+ column) + else throw ColumnNotFound(column.name, table.name) + case DropColumn(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy(columns = table.columns.filterNot(_.name == columnName)) + else throw ColumnNotFound(columnName, table.name) + case RenameColumn(oldName, newName) => + if (cols.contains(oldName)) + table.copy( + columns = table.columns.map { col => + if (col.name == oldName) col.copy(name = newName) else col + } + ) + else throw ColumnNotFound(oldName, table.name) + // column type + case AlterColumnType(columnName, newType, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(dataType = newType) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column script + case AlterColumnScript(columnName, newScript, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(script = Some(newScript.copy(dataType = col.dataType))) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnScript(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(script = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column default value + case AlterColumnDefault(columnName, newDefault, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = Some(newDefault)) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnDefault(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(defaultValue = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column not null + case AlterColumnNotNull(columnName, ifExists) => + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = true) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnNotNull(columnName, ifExists) => + if (!table.cols.contains(columnName) && ifExists) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) col.copy(notNull = false) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column options + case AlterColumnOptions(columnName, newOptions, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(options = col.options ++ newOptions) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case AlterColumnOption( + columnName, + optionKey, + optionValue, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + options = ObjectValue(col.options).set(optionKey, optionValue).value + ) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnOption( + columnName, + optionKey, + ifExists + ) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy( + options = ObjectValue(col.options).remove(optionKey).value + ) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // column comments + case AlterColumnComment(columnName, newComment, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = Some(newComment)) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + case DropColumnComment(columnName, ifExists) => + if (ifExists && !table.cols.contains(columnName)) table + else if (table.cols.contains(columnName)) + table.copy( + columns = table.columns.map { col => + if (col.name == columnName) + col.copy(comment = None) + else col + } + ) + else throw ColumnNotFound(columnName, table.name) + // multi-fields + case AlterColumnFields(columnName, newFields, ifExists) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + val updated = c + .copy( + multiFields = newFields.toList.map(_.update(Some(c))) + ) + .updateStruct() + table.copy( + columns = table.columns.map { col => + if (col.name == updated.name) updated else col + } + ) + case _ => throw ColumnNotFound(columnName, table.name) + } + } + case AlterColumnField( + columnName, + field, + ifExists + ) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + val updatedFields = c.multiFields.filterNot(_.name == field.name) :+ field + c.copy(multiFields = updatedFields) + table + case _ => throw ColumnNotFound(columnName, table.name) + } + } + case DropColumnField( + columnName, + fieldName, + ifExists + ) => + val col = find(columnName) + val exists = col.isDefined + if (ifExists && !exists) table + else { + col match { + case Some(c) => + c.copy( + multiFields = c.multiFields.filterNot(_.name == fieldName) + ) + table + case _ => throw ColumnNotFound(columnName, table.name) + } + } + // mappings / settings + case AlterTableMapping(optionKey, optionValue) => + table.copy( + mappings = ObjectValue(table.mappings).set(optionKey, optionValue).value + ) + case DropTableMapping(optionKey) => + table.copy( + mappings = ObjectValue(table.mappings).remove(optionKey).value + ) + case AlterTableSetting(optionKey, optionValue) => + table.copy( + settings = ObjectValue(table.settings).set(optionKey, optionValue).value + ) + case DropTableSetting(optionKey) => + table.copy( + settings = ObjectValue(table.settings).remove(optionKey).value + ) + case AlterTableAlias(aliasName, aliasValue) => + table.copy( + aliases = table.aliases + (aliasName -> aliasValue) + ) + case DropTableAlias(aliasName) => + table.copy( + aliases = table.aliases - aliasName + ) + case _ => table + } + } + .update() + + } + + def mergeWithSearch(search: SingleSearch): Table = { + val fields = search.update(Some(this)).select.fields + val cols = + fields.foldLeft(List.empty[Column]) { (cols, field) => + val colName = field.fieldAlias.map(_.alias).getOrElse(field.sourceField) + val col = this.find(colName) match { + case Some(c) if field.functions.isEmpty => c + case None => + Column( + name = colName, + dataType = field.out + ) + } + cols :+ col + } + val primaryKey = + this.primaryKey.filter(pk => cols.exists(_.name == pk)) + val partitionBy = + this.partitionBy match { + case Some(partition) => + if (cols.exists(_.name == partition.column)) + Some(partition) + else + None + case _ => None + } + this.copy(columns = cols, primaryKey = primaryKey, partitionBy = partitionBy).update() + } + + override def validate(): Either[String, Unit] = { + var errors = Seq[String]() + // check that primary key columns exist + primaryKey.foreach { pk => + if (!cols.contains(pk)) { + errors = errors :+ s"Primary key column $pk does not exist in table $name" + } + } + // check that partition column exists + partitionBy.foreach { partition => + if (!cols.contains(partition.column)) { + errors = errors :+ s"Partition column ${partition.column} does not exist in table $name" + } + } + if (errors.isEmpty) Right(()) else Left(errors.mkString("\n")) + } + + private[schema] lazy val diffPipeline: IngestPipeline = { + val processorsFromColumns = tableProcessors.map(p => p.column -> p).toMap + IngestPipeline( + name = s"${name}_ddl_diff_pipeline", + pipelineType = IngestPipelineType.Custom, + processors = + tableProcessors ++ processors.filterNot(p => processorsFromColumns.contains(p.column)) + ) + } + + lazy val defaultPipeline: IngestPipeline = { + val processorsFromColumns = tableProcessors.map(p => p.column -> p).toMap + IngestPipeline( + name = defaultPipelineName.getOrElse(s"${name}_ddl_default_pipeline"), + pipelineType = IngestPipelineType.Default, + processors = tableProcessors ++ processors + .filter(p => p.pipelineType == IngestPipelineType.Default) + .filterNot(p => processorsFromColumns.contains(p.column)) + ) + } + + lazy val finalPipeline: IngestPipeline = { + IngestPipeline( + name = finalPipelineName.getOrElse(s"${name}_ddl_final_pipeline"), + pipelineType = IngestPipelineType.Final, + processors = processors.filter(p => p.pipelineType == IngestPipelineType.Final) + ) + } + + def setDefaultPipelineName(pipelineName: String): Table = { + this.copy( + settings = this.settings + ("default_pipeline" -> StringValue(pipelineName)) + ) + } + + lazy val defaultPipelineName: Option[String] = + settings + .get("default_pipeline") + .map(_.value) + .flatMap { + case v: String if v != "_none" => Some(v) + case _ => None + } + + def setFinalPipelineName(pipelineName: String): Table = { + this.copy( + settings = this.settings + ("final_pipeline" -> StringValue(pipelineName)) + ) + } + + lazy val finalPipelineName: Option[String] = + settings + .get("final_pipeline") + .map(_.value) + .flatMap { + case v: String if v != "_none" => Some(v) + case _ => None + } + + lazy val indexMappings: ObjectNode = { + val node = mapper.createObjectNode() + val fields = mapper.createObjectNode() + columns.foreach { column => + fields.replace(column.name, column.node) + } + node.set("properties", fields) + updateNode(node, mappings) + node + } + + lazy val indexSettings: ObjectNode = { + val node = mapper.createObjectNode() + val index = mapper.createObjectNode() + updateNode(index, settings) + node.set("index", index) + node + } + + lazy val indexAliases: Seq[TableAlias] = aliases.map { case (aliasName, value) => + TableAlias(name, aliasName, value) + }.toSeq + + lazy val indexTemplate: ObjectNode = { + val node = mapper.createObjectNode() + val patterns = mapper.createArrayNode() + patterns.add(s"$name-*") + node.set("index_patterns", patterns) + node.put("priority", 1) + val template = mapper.createObjectNode() + template.set("mappings", indexMappings) + template.set("settings", indexSettings) + if (aliases.nonEmpty) { + val aliasesNode = mapper.createObjectNode() + indexAliases.foreach { alias => + aliasesNode.set(alias.alias, alias.node) + } + template.set("aliases", aliasesNode) + } + node.set("template", template) + node + } + + lazy val defaultPipelineNode: ObjectNode = defaultPipeline.node + + lazy val finalPipelineNode: ObjectNode = finalPipeline.node + + def diff(desired: Table): TableDiff = { + val actual = this.update() + val desiredUpdated = desired.update() + + val columnDiffs = scala.collection.mutable.ListBuffer[ColumnDiff]() + + // 1. Columns added + val actualCols = actual.cols + val desiredCols = desiredUpdated.cols + + for ((name, col) <- desiredCols if !actualCols.contains(name)) { + columnDiffs += ColumnAdded(col) + } + + // 2. Columns removed + for ((name, col) <- actualCols if !desiredCols.contains(name)) { + columnDiffs += ColumnRemoved(col.name) + } + + // 3. Columns modified + for ((name, a) <- actualCols if desiredCols.contains(name)) { + val d = desiredCols(name) + columnDiffs ++= a.diff(d) + } + + // 4. Mappings + val mappingDiffs = scala.collection.mutable.ListBuffer[MappingDiff]() + mappingDiffs ++= ObjectValue(actual.mappings).diff(ObjectValue(desired.mappings)).map { + case Altered(name, value) => MappingSet(name, value) + case Removed(name) => MappingRemoved(name) + } + + // 5. Settings + val settingDiffs = scala.collection.mutable.ListBuffer[SettingDiff]() + settingDiffs ++= ObjectValue(actual.settings).diff(ObjectValue(desired.settings)).map { + case Altered(name, value) => SettingSet(name, value) + case Removed(name) => SettingRemoved(name) + } + + // 6. Aliases + val aliasDiffs = scala.collection.mutable.ListBuffer[AliasDiff]() + aliasDiffs ++= ObjectValue(actual.aliases).diff(ObjectValue(desired.aliases)).map { + case Altered(name, value) => AliasSet(name, value) + case Removed(name) => AliasRemoved(name) + } + + // 7. Default Pipeline + val pipelineDiffs = scala.collection.mutable.ListBuffer[PipelineDiff]() + val actualPipeline = actual.diffPipeline + val desiredPipeline = desiredUpdated.diffPipeline + actualPipeline.diff(desiredPipeline).foreach { d => + pipelineDiffs += d + } + + TableDiff( + columns = columnDiffs.toList, + mappings = mappingDiffs.toList, + settings = settingDiffs.toList, + pipeline = pipelineDiffs.toList, + aliases = aliasDiffs.toList + ) + } + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala new file mode 100644 index 00000000..24a1df33 --- /dev/null +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/serialization/package.scala @@ -0,0 +1,182 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.fasterxml.jackson.databind.{ + DeserializationFeature, + JsonNode, + ObjectMapper, + SerializationFeature +} +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} + +import scala.language.implicitConversions +import scala.jdk.CollectionConverters._ + +package object serialization { + + /** Jackson ObjectMapper configuration */ + object JacksonConfig { + lazy val objectMapper: ObjectMapper = { + val mapper = new ObjectMapper() with ClassTagExtensions + + // Scala module for native support of Scala types + mapper.registerModule(DefaultScalaModule) + + // Java Time module for java.time.Instant, LocalDateTime, etc. + mapper.registerModule(new JavaTimeModule()) + + // Setup for performance + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false) + + // Ignores null values in serialization + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + + // Optimizations + mapper.configure(SerializationFeature.INDENT_OUTPUT, false) // No pretty print + mapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, false) + + mapper + } + } + + implicit def mapToJsonNode(value: Map[String, Any]): JsonNode = { + import JacksonConfig.{objectMapper => mapper} + mapper.valueToTree[JsonNode](value.asJava) + } + + implicit def jsonNodeToString(node: JsonNode): String = { + import JacksonConfig.{objectMapper => mapper} + mapper.writeValueAsString(node) + } + + implicit def jsonNodeToMap(value: JsonNode): Map[String, Any] = { + import JacksonConfig.{objectMapper => mapper} + mapper.convertValue(value, classOf[Map[String, Any]]) + } + + implicit def objectValueToObjectNode(value: ObjectValue): ObjectNode = { + import JacksonConfig.{objectMapper => mapper} + val node = mapper.createObjectNode() + updateNode(node, value.value) + node + } + + implicit def objectValuesToObjectNode(values: ObjectValues): ArrayNode = { + import JacksonConfig.{objectMapper => mapper} + val node = mapper.createArrayNode() + values.values.foreach { value => + node.add(value) + } + node + } + + private[sql] def updateNode(node: ObjectNode, updates: Map[String, Value[_]]): ObjectNode = { + import JacksonConfig.{objectMapper => mapper} + + updates.foreach { case (k, v) => + v match { + case Null => node.putNull(k) + case BooleanValue(b) => node.put(k, b) + case StringValue(s) => node.put(k, s) + case ByteValue(b) => node.put(k, b) + case ShortValue(s) => node.put(k, s) + case IntValue(i) => node.put(k, i) + case LongValue(l) => node.put(k, l) + case DoubleValue(d) => node.put(k, d) + case FloatValue(f) => node.put(k, f) + case IdValue => node.put(k, s"${v.value}") + case IngestTimestampValue => node.put(k, s"${v.value}") + case v: Values[_, _] => + val arrayNode = mapper.createArrayNode() + v.values.foreach { + case Null => arrayNode.addNull() + case BooleanValue(b) => arrayNode.add(b) + case StringValue(s) => arrayNode.add(s) + case ByteValue(b) => arrayNode.add(b) + case ShortValue(s) => arrayNode.add(s) + case IntValue(i) => arrayNode.add(i) + case LongValue(l) => arrayNode.add(l) + case DoubleValue(d) => arrayNode.add(d) + case FloatValue(f) => arrayNode.add(f) + case ObjectValue(o) => arrayNode.add(updateNode(mapper.createObjectNode(), o)) + case _ => // do nothing + } + node.set(k, arrayNode) + case ObjectValue(value) => + if (value.nonEmpty) + node.set(k, updateNode(mapper.createObjectNode(), value)) + case _ => // do nothing + } + } + node + } + + implicit def jsonNodeToObjectValue( + node: JsonNode + )(implicit ignoredKeys: Set[String] = Set.empty): ObjectValue = { + ObjectValue(extractObject(node, ignoredKeys)) + } + + private[elastic] def extractObject( + node: JsonNode, + ignoredKeys: Set[String] = Set.empty + ): Map[String, Value[_]] = { + node + .properties() + .asScala + .flatMap { entry => + val key = entry.getKey + val value = entry.getValue + + if (ignoredKeys.contains(key)) { + None + } else { + Value(value).map(key -> Value(_)) + } + } + .toMap + } + + private[elastic] def extractArray( + node: JsonNode + ): Seq[Value[_]] = { + node + .elements() + .asScala + .flatMap { element => + Value(element).map(Value(_)) + } + .toSeq + } + + implicit def toJson(json: String): JsonNode = { + import JacksonConfig.{objectMapper => mapper} + mapper.readTree(json) + } + + implicit def stringToObjectValue(json: String): ObjectValue = { + import JacksonConfig.{objectMapper => mapper} + mapper.readTree(json) + } + +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala index ec373753..1efa5099 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/time/package.scala @@ -113,6 +113,17 @@ package object time { sealed trait FixedUnit extends TimeUnit object TimeUnit { + def apply(script: String): TimeUnit = script match { + case "y" => YEARS + case "M" => MONTHS + case "w" => WEEKS + case "d" => DAYS + case "H" => HOURS + case "m" => MINUTES + case "s" => SECONDS + case _ => throw new IllegalArgumentException(s"Invalid time unit script: $script") + } + case object YEARS extends Expr("YEAR") with CalendarUnit { override def script: Option[String] = Some("y") } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala index a61d4f57..89681c5a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLType.scala @@ -50,3 +50,11 @@ trait SQLBool extends SQLType trait SQLArray extends SQLType { def elementType: SQLType } trait SQLStruct extends SQLType + +sealed trait EsqlType extends SQLType + +trait EsqlText extends EsqlType with SQLVarchar + +trait EsqlKeyword extends EsqlType with SQLVarchar + +trait EsqlGeoPoint extends EsqlType diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index eafb23b3..d127b457 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -21,6 +21,61 @@ import app.softnetwork.elastic.sql.`type`.SQLTypes._ object SQLTypeUtils { + def painlessType(sqlType: SQLType): String = sqlType match { + case TinyInt => "byte" + case SmallInt => "short" + case Int => "int" + case BigInt => "long" + case Double => "double" + case Real => "float" + case Numeric => "java.math.BigDecimal" + case Varchar => "String" + case Boolean => "boolean" + case Date => "LocalDate" + case Time => "LocalTime" + case DateTime => "LocalDateTime" + case Timestamp => "ZonedDateTime" + case Temporal => "ZonedDateTime" + case Array(inner) => + inner match { + case TinyInt => "byte[]" + case SmallInt => "short[]" + case Int => "int[]" + case BigInt => "long[]" + case Double => "double[]" + case Real => "float[]" + case Boolean => "boolean[]" + case _ => s"java.util.List<${painlessType(inner)}>" + } + case Struct => "Map" + case Any => "Object" + case Null => "Object" + case _ => "Object" + } + + def elasticType(sqlType: SQLType): String = sqlType match { + case Null => "null" + case TinyInt => "byte" + case SmallInt => "short" + case Int => "integer" + case BigInt => "long" + case Double => "double" + case Real => "float" + case Numeric => "scaled_float" + case Varchar | Text => "text" + case Keyword => "keyword" + case Boolean => "boolean" + case Date => "date" + case Time => "date" + case DateTime => "date" + case Timestamp => "date" + case Temporal => "date" + case GeoPoint => "geo_point" + case Array(Struct) => "nested" + case Struct => "object" + case _ => "object" + } + def matches(out: SQLType, in: SQLType): Boolean = out.typeId == in.typeId || (out.typeId == Temporal.typeId && Set( @@ -39,6 +94,10 @@ object SQLTypeUtils { ).contains( out.typeId )) || + (Set(DateTime.typeId, Timestamp.typeId) + .contains(out.typeId) && Set(DateTime.typeId, Timestamp.typeId).contains( + in.typeId + )) || (out.typeId == Numeric.typeId && Set( TinyInt.typeId, SmallInt.typeId, @@ -62,7 +121,10 @@ object SQLTypeUtils { out.typeId )) || (out.isInstanceOf[SQLNumeric] && in.isInstanceOf[SQLNumeric]) || - (out.typeId == Varchar.typeId && in.typeId == Varchar.typeId) || + (Set(Varchar.typeId, Text.typeId, Keyword.typeId) + .contains(out.typeId) && Set(Varchar.typeId, Text.typeId, Keyword.typeId).contains( + in.typeId + )) || (out.typeId == Boolean.typeId && in.typeId == Boolean.typeId) || out.typeId == Any.typeId || in.typeId == Any.typeId || out.typeId == Null.typeId || in.typeId == Null.typeId @@ -105,11 +167,11 @@ object SQLTypeUtils { def coerce(in: PainlessScript, to: SQLType, context: Option[PainlessContext]): String = { context match { - case Some(_) => + case Some(ctx) => in match { case identifier: Identifier => - identifier.baseType match { - case SQLTypes.Any => // in painless context, Any is ZonedDateTime + identifier.originalType match { + case SQLTypes.Any if !ctx.isProcessor => // in painless context, Any is ZonedDateTime to match { case SQLTypes.Date => identifier.addPainlessMethod(".toLocalDate()") @@ -117,6 +179,15 @@ object SQLTypeUtils { identifier.addPainlessMethod(".toLocalTime()") case _ => // do nothing } + case SQLTypes.Any if ctx.isProcessor => + to match { + case SQLTypes.DateTime | SQLTypes.Timestamp => + val expr = identifier.painless(context) + val from = SQLTypes.BigInt + val ret = coerce(expr, from, to, identifier.nullable, context) + return ret + case _ => // do nothing + } case _ => // do nothing } case _ => // do nothing @@ -260,4 +331,65 @@ object SQLTypeUtils { s"($expr != null ? $ret : null)" } + private val numericRank: Map[Class[_], Int] = Map( + classOf[SQLTinyInt] -> 1, + classOf[SQLSmallInt] -> 2, + classOf[SQLInt] -> 3, + classOf[SQLBigInt] -> 4, + classOf[SQLReal] -> 5, + classOf[SQLDouble] -> 6 + ) + + private def numericRankOf(t: SQLType): Option[Int] = + numericRank.collectFirst { case (cls, rank) if cls.isInstance(t) => rank } + + def canConvert(from: SQLType, to: SQLType): Boolean = { + + // 1. Identity + if (from.typeId == to.typeId) return true + + // 2. ANY / NULL + if (to.isInstanceOf[SQLAny]) return true + if (from.isInstanceOf[SQLNull]) return true + + // 3. Numerics + (numericRankOf(from), numericRankOf(to)) match { + case (Some(r1), Some(r2)) => + return r1 <= r2 // expansion only + case _ => + } + + // 4. Literals (VARCHAR, CHAR, TEXT, KEYWORD) + if (from.isInstanceOf[SQLLiteral] && to.isInstanceOf[SQLLiteral]) + return true + + // 5. Booleans + if (from.isInstanceOf[SQLBool] && to.isInstanceOf[SQLBool]) + return true + + // 6. Temporals + if (from.isInstanceOf[SQLTemporal] && to.isInstanceOf[SQLTemporal]) + return true + + // 7. Arrays + (from, to) match { + case (f: SQLArray, t: SQLArray) => + canConvert(f.elementType, t.elementType) + case _ => + } + + // 8. Structs + (from, to) match { + case (f: SQLStruct, t: SQLStruct) => + structConvertible(f, t) + case _ => + } + + false + } + + private def structConvertible(from: SQLStruct, to: SQLStruct): Boolean = { + // TODO To be refined according to our definition of SQLStruct + true + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala index 52cca678..da1fc9c4 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypes.scala @@ -16,6 +16,8 @@ package app.softnetwork.elastic.sql.`type` +import app.softnetwork.elastic.schema.IndexField + object SQLTypes { case object Any extends SQLAny { val typeId = "ANY" } @@ -41,12 +43,41 @@ object SQLTypes { case object Char extends SQLChar { val typeId = "CHAR" } case object Varchar extends SQLVarchar { val typeId = "VARCHAR" } + case object Text extends EsqlText { val typeId = "TEXT" } + case object Keyword extends EsqlKeyword { val typeId = "KEYWORD" } case object Boolean extends SQLBool { val typeId = "BOOLEAN" } case class Array(elementType: SQLType) extends SQLArray { - val typeId = s"array<${elementType.typeId}>" + val typeId = s"ARRAY<${elementType.typeId}>" } case object Struct extends SQLStruct { val typeId = "STRUCT" } + + case object GeoPoint extends EsqlGeoPoint { val typeId = "GEO_POINT" } + + def apply(typeName: String): SQLType = typeName.toLowerCase match { + case "null" => Null + case "boolean" => Boolean + case "int" | "integer" => Int + case "long" | "bigint" => BigInt + case "short" | "smallint" => SmallInt + case "byte" | "tinyint" => TinyInt + case "keyword" => Keyword + case "text" => Text + case "varchar" => Varchar + case "timestamp" => Timestamp + case "datetime" => DateTime + case "date" => Date + case "time" => Time + case "double" => Double + case "float" | "real" => Real + case "object" | "struct" => Struct + case "nested" | "array" => Array(Struct) + case "geo_point" => GeoPoint + case _ => Any + } + + def apply(field: IndexField): SQLType = apply(field.`type`) + } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala new file mode 100644 index 00000000..a6a4043c --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/ObjectValueSpec.scala @@ -0,0 +1,144 @@ +package app.softnetwork.elastic.sql + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class ObjectValueSpec extends AnyFlatSpec with Matchers { + + "ObjectValue" should "correctly store and retrieve nested objects" in { + val nestedObject = Map( + "level1" -> ObjectValue( + Map( + "level2" -> ObjectValue( + Map( + "level3" -> StringValue("value") + ) + ) + ) + ) + ) + + val objectValue = ObjectValue(nestedObject) + + val retrievedObject = objectValue.value + val level1 = retrievedObject("level1").asInstanceOf[ObjectValue].value + val level2 = level1("level2").asInstanceOf[ObjectValue].value + val level3Value = level2("level3").asInstanceOf[StringValue] + + level3Value.value shouldEqual "value" + + objectValue.find("level1.level2.level3").getOrElse(Null).value shouldBe "value" + } + + it should "return Null for non-existing paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + objectValue.find("key2.key4") shouldBe None + objectValue.find("key5") shouldBe None + } + + it should "set values at specified paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.set("key2.key4", StringValue("newValue")) + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + val key4Value = key2("key4").asInstanceOf[StringValue] + + key4Value.value shouldEqual "newValue" + } + + it should "overwrite existing values when setting at a path" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.set("key2.key3", StringValue("updatedValue")) + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + val key3Value = key2("key3").asInstanceOf[StringValue] + + key3Value.value shouldEqual "updatedValue" + } + + it should "remove values at specified paths" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3"), + "key4" -> StringValue("value4") + ) + ) + ) + ) + + val updatedObject = objectValue.remove("key2.key3") + + val key2 = updatedObject.value("key2").asInstanceOf[ObjectValue].value + + key2.contains("key3") shouldBe false + key2.contains("key4") shouldBe true + } + + it should "handle removal of non-existing paths gracefully" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val updatedObject = objectValue.remove("key2.key4") + + updatedObject shouldEqual objectValue + } + + it should "correctly serialize and deserialize to/from JSON" in { + val objectValue = ObjectValue( + Map( + "key1" -> StringValue("value1"), + "key2" -> ObjectValue( + Map( + "key3" -> StringValue("value3") + ) + ) + ) + ) + + val jsonString = objectValue.toJson + val deserializedObject = ObjectValue.fromJson(jsonString) + + deserializedObject shouldEqual objectValue + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala deleted file mode 100644 index 9bbf49cb..00000000 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/StringValueSpec.scala +++ /dev/null @@ -1,18 +0,0 @@ -package app.softnetwork.elastic.sql - -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -/** Created by smanciot on 17/02/17. - */ -class StringValueSpec extends AnyFlatSpec with Matchers { - - "SQLLiteral" should "perform sql like" in { - val l = StringValue("%dummy%") - l.like(Seq("dummy")) should ===(true) - l.like(Seq("aa dummy")) should ===(true) - l.like(Seq("dummy bbb")) should ===(true) - l.like(Seq("aaa dummy bbb")) should ===(true) - l.like(Seq("dummY")) should ===(false) - } -} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala similarity index 95% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala rename to sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala index a9998da9..151f339b 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLDateTimeFunctionSuite.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/function/time/DateTimeFunctionSuite.scala @@ -1,12 +1,12 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.function.time -import org.scalatest.funsuite.AnyFunSuite +import app.softnetwork.elastic.sql.Identifier +import app.softnetwork.elastic.sql.`type`.SQLType import app.softnetwork.elastic.sql.function._ -import app.softnetwork.elastic.sql.function.time._ import app.softnetwork.elastic.sql.time._ -import app.softnetwork.elastic.sql.`type`.SQLType +import org.scalatest.funsuite.AnyFunSuite -class SQLDateTimeFunctionSuite extends AnyFunSuite { +class DateTimeFunctionSuite extends AnyFunSuite { // Base d'exemple val baseDate = "doc['createdAt'].value" diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala similarity index 50% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala rename to sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index 186833e3..1178abda 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1,7 +1,20 @@ -package app.softnetwork.elastic.sql - -import app.softnetwork.elastic.sql.parser._ - +package app.softnetwork.elastic.sql.parser + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.{IngestTimestampValue, StringValue} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.query._ +import app.softnetwork.elastic.sql.schema.{ + mapper, + DateIndexNameProcessor, + IngestPipelineType, + IngestProcessorType, + PartitionDate, + PrimaryKeyProcessor, + ScriptProcessor, + SetProcessor +} +import app.softnetwork.elastic.sql.time.TimeUnit import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -222,14 +235,14 @@ object Queries { /** Created by smanciot on 15/02/17. */ -class SQLParserSpec extends AnyFlatSpec with Matchers { +class ParserSpec extends AnyFlatSpec with Matchers { import Queries._ - "SQLParser" should "parse numerical EQ" in { + "Parser" should "parse numerical EQ" in { val result = Parser(numericalEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalEq) shouldBe true } @@ -237,7 +250,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical NE" in { val result = Parser(numericalNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalNe) shouldBe true } @@ -245,7 +258,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical LT" in { val result = Parser(numericalLt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalLt) shouldBe true } @@ -253,7 +266,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical LE" in { val result = Parser(numericalLe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalLe) shouldBe true } @@ -261,7 +274,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical GT" in { val result = Parser(numericalGt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalGt) shouldBe true } @@ -269,7 +282,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse numerical GE" in { val result = Parser(numericalGe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(numericalGe) shouldBe true } @@ -277,7 +290,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal EQ" in { val result = Parser(literalEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalEq) shouldBe true } @@ -285,7 +298,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LIKE" in { val result = Parser(literalLike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLike) shouldBe true } @@ -293,7 +306,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal RLIKE" in { val result = Parser(literalRlike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalRlike) shouldBe true } @@ -301,7 +314,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal NOT LIKE" in { val result = Parser(literalNotLike) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalNotLike) shouldBe true } @@ -309,7 +322,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal NE" in { val result = Parser(literalNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalNe) shouldBe true } @@ -317,7 +330,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LT" in { val result = Parser(literalLt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLt) shouldBe true } @@ -325,7 +338,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal LE" in { val result = Parser(literalLe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalLe) shouldBe true } @@ -333,20 +346,20 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse literal GT" in { val result = Parser(literalGt) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(literalGt) shouldBe true } it should "parse literal GE" in { val result = Parser(literalGe) - result.toOption.flatMap(_.left.toOption.map(_.sql)).getOrElse("") equalsIgnoreCase literalGe + result.toOption.map(_.sql).getOrElse("") equalsIgnoreCase literalGe } it should "parse boolean EQ" in { val result = Parser(boolEq) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(boolEq) shouldBe true } @@ -354,7 +367,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse boolean NE" in { val result = Parser(boolNe) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(boolNe) shouldBe true } @@ -362,7 +375,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse BETWEEN" in { val result = Parser(betweenExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(betweenExpression) shouldBe true } @@ -370,7 +383,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse AND predicate" in { val result = Parser(andPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(andPredicate) shouldBe true } @@ -378,7 +391,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse OR predicate" in { val result = Parser(orPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(orPredicate) shouldBe true } @@ -386,7 +399,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse left predicate with criteria" in { val result = Parser(leftPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(leftPredicate) shouldBe true } @@ -394,7 +407,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse right predicate with criteria" in { val result = Parser(rightPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(rightPredicate) shouldBe true } @@ -402,7 +415,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse multiple predicates" in { val result = Parser(predicates) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(predicates) shouldBe true } @@ -410,14 +423,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse nested predicate" in { val result = Parser(nestedPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedPredicate } it should "parse nested criteria" in { val result = Parser(nestedCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nestedCriteria) shouldBe true } @@ -425,7 +438,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse child predicate" in { val result = Parser(childPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(childPredicate) shouldBe true } @@ -433,7 +446,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse child criteria" in { val result = Parser(childCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(childCriteria) shouldBe true } @@ -441,7 +454,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse parent predicate" in { val result = Parser(parentPredicate) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(parentPredicate) shouldBe true } @@ -449,7 +462,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse parent criteria" in { val result = Parser(parentCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(parentCriteria) shouldBe true } @@ -457,7 +470,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN literal expression" in { val result = Parser(inLiteralExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inLiteralExpression) shouldBe true } @@ -465,7 +478,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN numerical expression with Int values" in { val result = Parser(inNumericalExpressionWithIntValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inNumericalExpressionWithIntValues) shouldBe true } @@ -473,7 +486,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IN numerical expression with Double values" in { val result = Parser(inNumericalExpressionWithDoubleValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(inNumericalExpressionWithDoubleValues) shouldBe true } @@ -481,7 +494,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN literal expression" in { val result = Parser(notInLiteralExpression) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInLiteralExpression) shouldBe true } @@ -489,7 +502,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN numerical expression with Int values" in { val result = Parser(notInNumericalExpressionWithIntValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInNumericalExpressionWithIntValues) shouldBe true } @@ -497,7 +510,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NOT IN numerical expression with Double values" in { val result = Parser(notInNumericalExpressionWithDoubleValues) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(notInNumericalExpressionWithDoubleValues) shouldBe true } @@ -505,7 +518,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse nested with BETWEEN" in { val result = Parser(nestedWithBetween) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nestedWithBetween) shouldBe true } @@ -513,7 +526,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COUNT" in { val result = Parser(COUNT) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(COUNT) shouldBe true } @@ -521,7 +534,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse DISTINCT COUNT" in { val result = Parser(countDistinct) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(countDistinct) shouldBe true } @@ -529,7 +542,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COUNT with nested criteria" in { val result = Parser(countNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(countNested) shouldBe true } @@ -537,7 +550,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IS NULL" in { val result = Parser(isNull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNull) shouldBe true } @@ -545,7 +558,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse IS NOT NULL" in { val result = Parser(isNotNull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNotNull) shouldBe true } @@ -553,14 +566,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse geo distance criteria" in { val result = Parser(geoDistanceCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe geoDistanceCriteria } it should "parse EXCEPT fields" in { val result = Parser(except) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(except) shouldBe true } @@ -568,7 +581,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse MATCH criteria" in { val result = Parser(matchCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(matchCriteria) shouldBe true } @@ -576,7 +589,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY" in { val result = Parser(groupBy) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupBy) shouldBe true } @@ -584,7 +597,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ORDER BY" in { val result = Parser(orderBy) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(orderBy) shouldBe true } @@ -592,14 +605,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse LIMIT" in { val result = Parser(limit) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe limit } it should "parse GROUP BY with ORDER BY and LIMIT" in { val result = Parser(groupByWithOrderByAndLimit) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupByWithOrderByAndLimit) shouldBe true } @@ -607,7 +620,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY with HAVING" in { val result = Parser(groupByWithHaving) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(groupByWithHaving) shouldBe true } @@ -615,7 +628,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date time fields" in { val result = Parser(dateTimeWithIntervalFields) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeWithIntervalFields) shouldBe true } @@ -623,7 +636,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse fields with INTERVAL" in { val result = Parser(fieldsWithInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(fieldsWithInterval) shouldBe true } @@ -631,7 +644,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with date time and INTERVAL" in { val result = Parser(filterWithDateTimeAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithDateTimeAndInterval) shouldBe true } @@ -639,7 +652,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with date and interval" in { val result = Parser(filterWithDateAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithDateAndInterval) shouldBe true } @@ -647,7 +660,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse filter with time and interval" in { val result = Parser(filterWithTimeAndInterval) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(filterWithTimeAndInterval) shouldBe true } @@ -655,14 +668,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse GROUP BY with HAVING and date time functions" in { val result = Parser(groupByWithHavingAndDateTimeFunctions) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe groupByWithHavingAndDateTimeFunctions } it should "parse date_parse function" in { val result = Parser(dateParse) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateParse) shouldBe true } @@ -670,7 +683,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_parse_time function" in { val result = Parser(dateTimeParse) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeParse) shouldBe true } @@ -678,7 +691,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_diff function" in { val result = Parser(dateDiff) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateDiff) shouldBe true } @@ -686,7 +699,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_diff function with aggregation" in { val result = Parser(aggregationWithDateDiff) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(aggregationWithDateDiff) shouldBe true } @@ -694,7 +707,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse format_date function" in { val result = Parser(dateFormat) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateFormat) shouldBe true } @@ -702,7 +715,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse format_datetime function" in { val result = Parser(dateTimeFormat) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeFormat) shouldBe true } @@ -710,7 +723,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_add function" in { val result = Parser(dateAdd) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateAdd) shouldBe true } @@ -718,7 +731,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse date_sub function" in { val result = Parser(dateSub) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateSub) shouldBe true } @@ -726,7 +739,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse datetime_add function" in { val result = Parser(dateTimeAdd) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeAdd) shouldBe true } @@ -734,7 +747,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse datetime_sub function" in { val result = Parser(dateTimeSub) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(dateTimeSub) shouldBe true } @@ -742,7 +755,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNULL function" in { val result = Parser(isnull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isnull) shouldBe true } @@ -750,7 +763,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNOTNULL function" in { val result = Parser(isnotnull) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isnotnull) shouldBe true } @@ -758,7 +771,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNULL criteria" in { val result = Parser(isNullCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNullCriteria) shouldBe true } @@ -766,7 +779,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse ISNOTNULL criteria" in { val result = Parser(isNotNullCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(isNotNullCriteria) shouldBe true } @@ -774,7 +787,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse COALESCE function" in { val result = Parser(coalesce) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(coalesce) shouldBe true } @@ -782,7 +795,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse NULLIF function" in { val result = Parser(nullif) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(nullif) shouldBe true } @@ -790,14 +803,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse conversion function" in { val result = Parser(conversion) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe conversion } it should "parse all casts function" in { val result = Parser(allCasts) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(allCasts) shouldBe true } @@ -805,7 +818,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse CASE WHEN expression" in { val result = Parser(caseWhen) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(caseWhen) shouldBe true } @@ -813,7 +826,7 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse CASE WHEN with expression" in { val result = Parser(caseWhenExpr) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(caseWhenExpr) shouldBe true } @@ -821,14 +834,14 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse EXTRACT function" in { val result = Parser(extract) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe extract } it should "parse arithmetic expressions" in { val result = Parser(arithmetic) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") .equalsIgnoreCase(arithmetic) shouldBe true } @@ -836,91 +849,744 @@ class SQLParserSpec extends AnyFlatSpec with Matchers { it should "parse mathematical functions" in { val result = Parser(mathematical) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe mathematical } it should "parse string functions" in { val result = Parser(string) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe string } it should "parse top hits functions" in { val result = Parser(topHits) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe topHits } it should "parse last_day function" in { val result = Parser(lastDay) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe lastDay } it should "parse all date extractors" in { val result = Parser(extractors) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe extractors } it should "parse geo distance field" in { val result = Parser(geoDistance) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe geoDistance } it should "parse BETWEEN with temporal fields" in { val result = Parser(betweenTemporal) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe betweenTemporal } it should "parse nested of nested" in { val result = Parser(nestedOfNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedOfNested } it should "parse predicate with distinct nested" in { val result = Parser(predicateWithDistinctNested) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe predicateWithDistinctNested } it should "parse nested without criteria" in { val result = Parser(nestedWithoutCriteria) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe nestedWithoutCriteria } it should "determine the aggregation context" in { val result = Parser(determinationOfTheAggregationContext) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe determinationOfTheAggregationContext } it should "parse aggregation with nested of nested context" in { val result = Parser(aggregationWithNestedOfNestedContext) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe aggregationWithNestedOfNestedContext } it should "parse where filters according to scope" in { val result = Parser(whereFiltersAccordingToScope) result.toOption - .flatMap(_.left.toOption.map(_.sql)) + .map(_.sql) .getOrElse("") shouldBe whereFiltersAccordingToScope } + + // --- DDL --- + + it should "parse CREATE TABLE if not exists" in { + val sql = + """CREATE TABLE IF NOT EXISTS users ( + | id INT NOT NULL COMMENT 'user identifier', + | name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), + | birthdate DATE, + | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), + | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) COMMENT 'user profile', + | PRIMARY KEY (id) + |) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false))""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case ct @ CreateTable( + "users", + Right(cols), + true, + false, + List("id"), + Some(PartitionDate("birthdate", TimeUnit.MONTHS)), + _ + ) => + cols.map(_.name) should contain allOf ("id", "name") + cols.find(_.name == "id").get.notNull shouldBe true + cols.find(_.name == "name").get.defaultValue.map(_.value) shouldBe Some("anonymous") + cols.find(_.name == "birthdate").get.dataType.typeId should include("DATE") + cols.find(_.name == "age").get.script.nonEmpty shouldBe true + cols + .find(_.name == "age") + .get + .script + .map(p => p.source) + .getOrElse("") should include( + """def param1 = ctx.birthdate; + |def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); + |def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); + |ctx.age = (param1 == null) ? null : param3""".stripMargin.replaceAll("\n", " ") + ) + cols.find(_.name == "ingested_at").get.defaultValue.map(_.value) shouldBe Some( + "_ingest.timestamp" + ) + ct.mappings.get("dynamic").map(_.value) shouldBe Some(false) + val schema = ct.schema + val sql = schema.sql + println(sql) + println(schema.defaultPipeline.ddl) + val json = schema.defaultPipeline.json + println(json) + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + val indexMappings = schema.indexMappings + println(indexMappings) + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"age":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(birthdate, CURRENT_DATE, YEAR)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3"}},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3"}}}},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"birthdate":{"data_type":"DATE","not_null":"false"},"id":{"data_type":"INT","not_null":"true","comment":"user identifier"}}}}""".stripMargin + val indexSettings = schema.indexSettings + println(indexSettings) + indexSettings.toString shouldBe """{"index":{}}""" + val pipeline = schema.defaultPipelineNode + println(pipeline) + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET VALUE 'anonymous' IF ctx.name == null, age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR)), ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null, profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"field":"name","if":"ctx.name == null","description":"name SET VALUE 'anonymous' IF ctx.name == null","ignore_failure":true,"value":"anonymous"}},{"script":{"description":"age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.YEARS.between(param1, param2)); ctx.age = (param1 == null) ? null : param3","ignore_failure":true}},{"set":{"field":"ingested_at","if":"ctx.ingested_at == null","description":"ingested_at SET VALUE _ingest.timestamp IF ctx.ingested_at == null","ignore_failure":true,"value":"{{_ingest.timestamp}}"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); def param3 = Long.valueOf(ChronoUnit.DAYS.between(param1, param2)); ctx.profile.seniority = (param1 == null) ? null : param3","ignore_failure":true}},{"date_index_name":{"field":"birthdate","description":"PARTITION BY birthdate (MONTH)","index_name_prefix":"users-","date_formats":["yyyy-MM"],"ignore_failure":true,"date_rounding":"M"}},{"set":{"field":"_id","description":"PRIMARY KEY (id)","ignore_failure":false,"ignore_empty_value":false,"value":"{{id}}"}}]}""" + // Reconstruct EsIndex + val mappings = mapper.createObjectNode() + mappings.set("mappings", indexMappings) + val settings = mapper.createObjectNode() + settings.set("settings", indexSettings) + val esIndex = Index( + name = "users", + mappings = mappings, + settings = settings, + defaultPipeline = Some(pipeline) + ) + val ddlTable = esIndex.schema + println(s"""esIndex ddl -> ${ddlTable.sql}""") + println(s"""esIndex mappings -> ${ddlTable.indexMappings.toString}""") + println(s"""esIndex settings -> ${ddlTable.indexSettings.toString}""") + println(s"""esIndex pipeline -> ${ddlTable.defaultPipelineNode.toString}""") + println(s"""esIndex ddl pipeline -> ${ddlTable.defaultPipeline.ddl}""") + val ddlTableDiff = ddlTable.diff(ct.schema) + ddlTableDiff.columns.isEmpty shouldBe true + ddlTableDiff.mappings.isEmpty shouldBe true + ddlTableDiff.settings.isEmpty shouldBe true + ddlTableDiff.pipeline.isEmpty shouldBe true + case _ => fail("Expected CreateTable") + } + } + + it should "parse CREATE OR REPLACE TABLE as select" in { + val sql = "CREATE OR REPLACE TABLE users AS SELECT id, name FROM accounts WHERE active = true" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case table: CreateTable => + println(table.sql) + table.table shouldBe "users" + table.ifNotExists shouldBe false + table.orReplace shouldBe true + table.columns.map(_.name) should contain allOf ("id", "name") + case _ => fail("Expected CreateTable") + } + } + + it should "parse DROP TABLE if exists" in { + val sql = "DROP TABLE IF EXISTS users" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case DropTable("users", ie, _) if ie => + case _ => fail("Expected DropTable") + } + } + + it should "parse TRUNCATE TABLE" in { + val sql = "TRUNCATE TABLE users" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case TruncateTable("users") => + case _ => fail("Expected TruncateTable") + } + } + + it should "parse ALTER TABLE add column if not exists" in { + val sql = + """ALTER TABLE users + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0 + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(AddColumn(c, ine)) if ine => + c.name shouldBe "age" + c.dataType.typeId should include("INT") + c.defaultValue.map(_.value) shouldBe Some(0L) + case _ => fail("Expected AddColumn") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE rename column" in { + val sql = + """ALTER TABLE users + | RENAME COLUMN name TO full_name + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(RenameColumn(o, n)) => + o shouldBe "name" + n shouldBe "full_name" + case _ => fail("Expected RenameColumn") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column options if exists" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN IF EXISTS status SET OPTIONS (description = 'a description') + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnOptions(c, d, ie)) if ie => + c shouldBe "status" + d.get("description").map(_.value) shouldBe Some("a description") + case _ => fail("Expected AlterColumnDefault") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column default" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET DEFAULT 'active' + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnDefault(c, d, _)) => + c shouldBe "status" + d.value shouldBe "active" + case other => fail(s"Expected AlterColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE drop column default" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status DROP DEFAULT + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(DropColumnDefault(c, _)) => + c shouldBe "status" + case other => fail(s"Expected DropColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column not null" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET NOT NULL + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnNotNull(c, _)) => + c shouldBe "status" + case other => fail(s"Expected AlterColumnNotNull, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE drop column not null" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status DROP NOT NULL + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(DropColumnNotNull(c, _)) => + c shouldBe "status" + case other => fail(s"Expected DropColumnNotNull, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE set column type" in { + val sql = + """ALTER TABLE users + | ALTER COLUMN status SET DATA TYPE BIGINT + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", _, stmts) => + stmts match { + case List(AlterColumnType(c, d, _)) => + c shouldBe "status" + d shouldBe SQLTypes.BigInt + case other => fail(s"Expected AlterColumnType, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE if exists" in { + val sql = + """ALTER TABLE IF EXISTS users + | ALTER COLUMN status SET DEFAULT 'active' + |""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterTable("users", ifExists, stmts) if ifExists => + stmts match { + case List(AlterColumnDefault(c, d, _)) => + c shouldBe "status" + d.value shouldBe "active" + case other => fail(s"Expected AlterColumnDefault, got $other") + } + case _ => fail("Expected AlterTable") + } + } + + it should "parse ALTER TABLE with multiple statements" in { + val sql = + """ALTER TABLE users ( + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + | RENAME COLUMN name TO full_name, + | ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + | ALTER COLUMN IF EXISTS profile SET FIELDS ( + | description VARCHAR DEFAULT 'N/A', + | visibility BOOLEAN DEFAULT true + | ) + |)""".stripMargin + + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + println(stmt.sql) + stmt.sql.replaceAll("\t", " ") shouldBe sql + stmt match { + case AlterTable("users", _, stmts) => + stmts.length shouldBe 4 + stmts.collect { case AddColumn(c, true) => c.name } should contain("age") + stmts.collect { case RenameColumn(o, n) => (o, n) } should contain(("name", "full_name")) + stmts.collect { case AlterColumnDefault(c, d, true) => + List(c, d.value) + }.flatten should contain allOf ("status", "active") + stmts.collect { case AlterColumnFields("profile", fields, true) => + fields.map(f => (f.name, f.dataType.typeId, f.defaultValue.map(_.value))) + }.flatten should contain allOf (("description", "VARCHAR", Some("N/A")), ( + "visibility", + "BOOLEAN", + Some(true) + )) + case _ => fail("Expected AlterTable") + } + } + + it should "parse CREATE OR REPLACE PIPELINE" in { + val sql = + """CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ), + | SCRIPT ( + | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + | lang = "painless", + | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))", + | ignore_failure = true + | ), + | SET ( + | field = "ingested_at", + | if = "ctx.ingested_at == null", + | description = "DEFAULT _ingest.timestamp", + | ignore_failure = true, + | value = "_ingest.timestamp" + | ), + | SCRIPT ( + | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", + | lang = "painless", + | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))", + | ignore_failure = true + | ), + | DATE_INDEX_NAME ( + | field = "birthdate", + | index_name_prefix = "users-", + | date_formats = ["yyyy-MM"], + | ignore_failure = true, + | date_rounding = "M", + | description = "PARTITION BY birthdate (MONTH)", + | separator = "-" + | ), + | SET ( + | field = "_id", + | description = "PRIMARY KEY (id)", + | ignore_failure = false, + | ignore_empty_value = false, + | value = "{{id}}" + | ) + |)""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case CreatePipeline( + "user_pipeline", + _, + false, + true, + processors + ) => + processors.size shouldBe 6 + processors.find(_.column == "name") match { + case Some( + SetProcessor( + IngestPipelineType.Default, + Some("DEFAULT 'anonymous'"), + "name", + StringValue("anonymous"), + None, + None, + None, + Some("ctx.name == null"), + true + ) + ) => + case other => fail(s"Expected SetProcessor for name, got $other") + } + processors.find(_.column == "age") match { + case Some( + ScriptProcessor( + IngestPipelineType.Default, + Some("age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))"), + "DATE_DIFF(birthdate, CURRENT_DATE, YEAR)", + "age", + SQLTypes.Int, + source, + true + ) + ) => + source should include( + "def param1 = ctx.birthdate; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))" + ) + case other => fail(s"Expected DdlScriptProcessor for age, got $other") + } + processors.find(_.column == "ingested_at") match { + case Some( + SetProcessor( + IngestPipelineType.Default, + Some("DEFAULT _ingest.timestamp"), + "ingested_at", + IngestTimestampValue, + None, + None, + None, + _, + true + ) + ) => + case other => fail(s"Expected DdlDefaultValueProcessor for ingested_at, got $other") + } + processors.find(_.column == "profile.seniority") match { + case Some( + ScriptProcessor( + IngestPipelineType.Default, + Some( + "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))" + ), + "DATE_DIFF(profile.join_date, CURRENT_DATE, DAY)", + "profile.seniority", + SQLTypes.Int, + source, + true + ) + ) => + source should include( + "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.now(ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))" + ) + case other => fail(s"Expected DdlScriptProcessor for profile.seniority, got $other") + } + processors.find(_.column == "birthdate") match { + case Some( + DateIndexNameProcessor( + IngestPipelineType.Default, + Some("PARTITION BY birthdate (MONTH)"), + "birthdate", + "M", + List("yyyy-MM"), + "users-", + true + ) + ) => + case other => fail(s"Expected DdlDateIndexNameProcessor for birthdate, got $other") + } + processors.find(_.column == "_id") match { + case Some( + PrimaryKeyProcessor( + IngestPipelineType.Default, + Some("PRIMARY KEY (id)"), + "_id", + cols, + false, + Some(false), + "\\|\\|" + ) + ) => + cols should contain("id") + case other => fail(s"Expected DdlPrimaryKeyProcessor for _id, got $other") + } + + case _ => fail("Expected CreatePipeline") + } + } + + it should "parse ALTER PIPELINE IF EXISTS" in { + val sql = + """ALTER PIPELINE IF EXISTS user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | if = "ctx.status == null", + | description = "status DEFAULT 'active'", + | ignore_failure = true, + | value = "active" + | ), + | DROP PROCESSOR SET (_id) + |)""".stripMargin + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case AlterPipeline( + "user_pipeline", + ie, + statements + ) if ie => + statements.size shouldBe 2 + statements.collect { case AddPipelineProcessor(p) => p } match { + case SetProcessor( + IngestPipelineType.Default, + Some("status DEFAULT 'active'"), + "status", + StringValue("active"), + None, + None, + None, + Some("ctx.status == null"), + true + ) :: Nil => + case other => fail(s"Expected AddPipelineProcessor with SetProcessor, got $other") + } + statements.collect { case DropPipelineProcessor(IngestProcessorType.Set, f) => + f + } should contain("_id") + case _ => fail("Expected AlterPipeline") + } + } + + it should "parse DROP PIPELINE if exists" in { + val sql = "DROP PIPELINE IF EXISTS user_pipeline" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case DropPipeline("user_pipeline", ie) if ie => + case _ => fail("Expected DropPipeline") + } + } + + // --- DML --- + + it should "parse INSERT INTO ... VALUES" in { + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice') ON CONFLICT DO NOTHING" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", cols, Right(values), Some(OnConflict(None, false))) => + cols should contain inOrder ("id", "name") + values.head.map(_.value) should contain inOrder (1, "Alice") + case _ => fail("Expected Insert with values") + } + } + + it should "parse INSERT INTO ... VALUES with DO UPDATE" in { + val sql = "INSERT INTO users (id, name) VALUES (1, 'Alice'), (2, 'BOB') ON CONFLICT DO UPDATE" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", cols, Right(values), Some(OnConflict(None, true))) => + cols should contain inOrder ("id", "name") + values.head.map(_.value) should contain inOrder (1, "Alice") + values.last.map(_.value) should contain inOrder (2, "BOB") + case _ => fail("Expected Insert with values") + } + } + + it should "parse INSERT INTO ... SELECT" in { + val sql = "INSERT INTO users AS SELECT id, name FROM old_users ON CONFLICT DO NOTHING" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", Nil, Left(sel: DqlStatement), Some(OnConflict(None, false))) => + sel.sql should include("SELECT id, name FROM old_users ON CONFLICT DO NOTHING") + case _ => fail("Expected Insert with select") + } + } + + it should "parse INSERT INTO ... SELECT with ON CONFLICT" in { + val sql = "INSERT INTO users SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Insert("users", Nil, Left(sel: DqlStatement), Some(OnConflict(Some(Seq("id")), true))) => + sel.sql should include("SELECT id, name FROM old_users ON CONFLICT (id) DO UPDATE") + case _ => fail("Expected Insert with select") + } + } + + it should "parse UPDATE" in { + val sql = "UPDATE users SET name = 'Bob', age = 42 WHERE id = 1" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Update("users", values, Some(where)) => + values("name").value shouldBe "Bob" + values("age").value shouldBe 42 + where.sql should include("id = 1") + case _ => fail("Expected Update") + } + } + + it should "parse DELETE" in { + val sql = "DELETE FROM users WHERE age > 30" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case Delete(table, Some(where)) => + table.name shouldBe "users" + where.sql should include("age > 30") + case _ => fail("Expected Delete") + } + } + + it should "parse UNION ALL" in { + val sql = + "SELECT id, name FROM dql_users WHERE age > 30 UNION ALL SELECT id, name FROM dql_users WHERE age <= 30" + val result = Parser(sql) + result.isRight shouldBe true + val stmt = result.toOption.get + stmt match { + case MultiSearch(Seq(left: DqlStatement, right: DqlStatement)) => + left.sql should include("SELECT id, name FROM dql_users WHERE age > 30") + right.sql should include("SELECT id, name FROM dql_users WHERE age <= 30") + case _ => fail("Expected Union") + } + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala index 7ba465d4..6c3c423d 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/BulkApiSpec.scala @@ -18,8 +18,8 @@ package app.softnetwork.elastic.client import akka.actor.ActorSystem import app.softnetwork.elastic.client.bulk.BulkOptions -import app.softnetwork.elastic.client.file._ import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.query.JsonArray import app.softnetwork.persistence.generateUUID import org.scalatest.concurrent.ScalaFutures import org.scalatest.flatspec.AnyFlatSpecLike @@ -74,7 +74,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid") + idKey = Some(Set("uuid")) ) .futureValue @@ -100,7 +100,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -119,7 +119,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -141,7 +141,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -162,7 +162,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -199,7 +199,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val result = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue @@ -224,7 +224,7 @@ trait BulkApiSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { val response = client .bulkFromFile( tempFile.getAbsolutePath, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), format = JsonArray ) .futureValue diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala index 3e8d7ce1..a2efa13c 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -25,7 +25,9 @@ import app.softnetwork.elastic.client.scroll._ import app.softnetwork.elastic.model.{Binary, Child, Parent, Sample} import app.softnetwork.elastic.persistence.query.ElasticProvider import app.softnetwork.elastic.scalatest.ElasticDockerTestKit -import app.softnetwork.elastic.sql.query.SQLQuery +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.elastic.sql.schema.{Table, TableAlias} import app.softnetwork.persistence._ import app.softnetwork.persistence.person.model.Person import com.fasterxml.jackson.core.JsonParseException @@ -41,7 +43,7 @@ import _root_.java.time.format.DateTimeFormatter import _root_.java.util.UUID import _root_.java.util.concurrent.TimeUnit import java.time.temporal.Temporal -import java.time.{LocalDate, LocalDateTime, ZoneOffset} +import java.time.{LocalDate, LocalDateTime, ZoneOffset, ZonedDateTime} import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContextExecutor, Future} import scala.util.{Failure, Success, Try} @@ -58,6 +60,8 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M import ElasticProviders._ + implicit def timestamp: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + lazy val pClient: ElasticProvider[Person] = new PersonProvider( elasticConfig ) @@ -73,14 +77,14 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M import scala.language.implicitConversions - implicit def toSQLQuery(sqlQuery: String): SQLQuery = SQLQuery(sqlQuery) + implicit def toSQLQuery(sqlQuery: String): SelectStatement = SelectStatement(sqlQuery) implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = Source.fromIterator(() => list.iterator) override def beforeAll(): Unit = { super.beforeAll() - pClient.createIndex("person") + pClient.createIndex("person", mappings = None, aliases = Nil) } override def afterAll(): Unit = { @@ -103,7 +107,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M ) "Creating an index and then delete it" should "work fine" in { - pClient.createIndex("create_delete") + pClient.createIndex("create_delete", mappings = None, aliases = Nil) blockUntilIndexExists("create_delete") "create_delete" should beCreated @@ -112,6 +116,83 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "create_delete" should not(beCreated()) } + "Creating an index with mappings and aliases" should "work fine" in { + val mappings = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | } + | }, + | "_meta": { + | "primary_key": ["uuid"], + | "partition_by": { + | "column": "birthDate", + | "granularity": "M" + | } + | } + |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + val aliases: Seq[TableAlias] = Seq( + TableAlias(table = "create_mappings_aliases", alias = "create_mappings_aliases_alias1"), + TableAlias( + table = "create_mappings_aliases", + alias = "create_mappings_aliases_alias2", + filter = Map("term" -> Map("name.raw" -> "Homer Simpson")) + ) + ) + + pClient + .createIndex("create_mappings_aliases", mappings = Some(mappings), aliases = aliases) + .get shouldBe true + "create_mappings_aliases" should beCreated() + + val index: Option[Index] = pClient.getIndex("create_mappings_aliases").get + + index match { + case Some(idx) => + val table: Table = idx.schema + log.info(table.sql) + table.columns.size shouldBe 3 + table.columns.map(_.name) should contain allOf ("uuid", "name", "birthDate") + table.primaryKey should contain("uuid") + table.partitionBy match { + case Some(partitionBy) => + partitionBy.column shouldBe "birthDate" + partitionBy.granularity.sql shouldBe "MONTH" + case None => fail("Partition by not found") + } + val tableAliases = table.aliases.keys + tableAliases should contain("create_mappings_aliases_alias1") + tableAliases should contain("create_mappings_aliases_alias2") + case None => fail("Index not found") + } + + pClient.deleteIndex("create_mappings_aliases").get shouldBe true + "create_mappings_aliases" should not(beCreated()) + } + + "Checking for an alias that does not exist" should "return false" in { + pClient.aliasExists("person_alias").get shouldBe false + } + "Adding an alias and then removing it" should "work" in { pClient.addAlias("person", "person_alias") @@ -119,7 +200,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M pClient.getAliases("person") match { case ElasticSuccess(aliases) => - aliases should contain("person_alias") + aliases.map(_.alias) should contain("person_alias") case ElasticFailure(elasticError) => fail(elasticError.fullMessage) } @@ -174,7 +255,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Setting a mapping" should "work" in { - pClient.createIndex("person_mapping") + pClient.createIndex("person_mapping", mappings = None, aliases = Nil) blockUntilIndexExists("person_mapping") "person_mapping" should beCreated() @@ -213,7 +294,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M ) shouldBe false implicit val bulkOptions: BulkOptions = BulkOptions("person_mapping") - val result = pClient.bulk[String](persons, identity, idKey = Some("uuid")).get + val result = pClient.bulk[String](persons, identity, idKey = Some(Set("uuid"))).get result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices @@ -270,7 +351,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Updating a mapping" should "work" in { - pClient.createIndex("person_migration").get shouldBe true + pClient.createIndex("person_migration", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -295,7 +376,11 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M implicit val bulkOptions: BulkOptions = BulkOptions("person_migration") val result = pClient - .bulk[String](Source.fromIterator(() => persons.iterator), identity, idKey = Some("uuid")) + .bulk[String]( + Source.fromIterator(() => persons.iterator), + identity, + idKey = Some(Set("uuid")) + ) .get result.failedCount shouldBe 0 result.successCount shouldBe persons.size @@ -401,7 +486,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Bulk index valid json with an id key but no suffix key" should "work" in { implicit val bulkOptions: BulkOptions = BulkOptions("person2") - val result = pClient.bulk[String](persons, identity, idKey = Some("uuid")).get + val result = pClient.bulk[String](persons, identity, idKey = Some(Set("uuid"))).get result.failedCount shouldBe 0 result.successCount shouldBe persons.size val indices = result.indices @@ -440,7 +525,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( persons, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), suffixDateKey = Some("birthDate") ) .get @@ -493,7 +578,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -528,13 +613,13 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Bulk upsert valid json with an id key and a suffix key" should "work" in { - pClient.createIndex("person5").get shouldBe true + pClient.createIndex("person5", mappings = None, aliases = Nil).get shouldBe true implicit val bulkOptions: BulkOptions = BulkOptions("person5") val result = pClient .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), suffixDateKey = Some("birthDate"), update = Some(true) ) @@ -580,7 +665,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -616,7 +701,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -673,7 +758,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -703,7 +788,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -735,7 +820,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Index" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -758,7 +843,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Update" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.updateAs(sample, uuid, Some(index)).get result shouldBe true @@ -781,7 +866,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Delete" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -799,7 +884,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M "Delete asynchronously" should "work" in { val uuid = UUID.randomUUID().toString val index = s"sample-$uuid" - sClient.createIndex(index).get shouldBe true + sClient.createIndex(index, mappings = None, aliases = Nil).get shouldBe true val sample = Sample(uuid) val result = sClient.indexAs(sample, uuid, Some(index)).get result shouldBe true @@ -819,7 +904,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Index binary data" should "work" in { - bClient.createIndex("binaries").get shouldBe true + bClient.createIndex("binaries", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -875,7 +960,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Aggregations" should "work" in { - pClient.createIndex("person10").get shouldBe true + pClient.createIndex("person10", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -915,7 +1000,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -1136,7 +1221,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M } "Nested queries" should "work" in { - parentClient.createIndex("parent").get shouldBe true + parentClient.createIndex("parent", mappings = None, aliases = Nil).get shouldBe true val mapping = """{ | "properties": { @@ -1166,6 +1251,9 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M | }, | "birthDate": { | "type": "date" + | }, + | "parentId": { + | "type": "keyword" | } | } | }, @@ -1184,7 +1272,7 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M .bulk[String]( personsWithUpsert, identity, - idKey = Some("uuid"), + idKey = Some(Set("uuid")), update = Some(true) ) .get @@ -1316,4 +1404,533 @@ trait ElasticClientSpec extends AnyFlatSpecLike with ElasticDockerTestKit with M scrollResult.children.map(_.parentId) should contain only "A16" } } + + "Truncate index" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_truncate") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => refresh(index).getStatusLine.getStatusCode < 400) shouldBe true + + indices should contain only "person_to_truncate" + + blockUntilCount(3, "person_to_truncate") + + "person_to_truncate" should haveCount(3) + + val truncateResult = pClient.truncateIndex("person_to_truncate").get + truncateResult shouldBe 3L + + "person_to_truncate" should haveCount(0) + } + + "Delete by query on closed index" should "work depending on ES version" in { + pClient.createIndex("closed_index_test").get shouldBe true + + pClient.closeIndex("closed_index_test").get shouldBe true + + val deleted = pClient + .deleteByQuery( + "closed_index_test", + """{"query": {"match_all": {}}}""" + ) + .get + + deleted shouldBe 0L // index is empty anyway + } + + "Delete by query with invalid JSON" should "fail" in { + pClient.createIndex("delete_by_query_invalid_json").get shouldBe true + val result = pClient.deleteByQuery("idx", "{not valid json}") + result.isFailure shouldBe true + } + + "Delete by query with invalid SQL" should "fail" in { + pClient.createIndex("delete_by_query_invalid_sql").get shouldBe true + val result = pClient.deleteByQuery("delete_by_query_invalid_sql", "DELETE FROM") + result.isFailure shouldBe true + } + + "Delete by query with SQL referencing wrong index" should "fail" in { + pClient.createIndex("index_a").get shouldBe true + pClient.createIndex("index_b").get shouldBe true + val result = pClient.deleteByQuery( + "index_a", + "DELETE FROM index_b WHERE x = 1" + ) + result.isFailure shouldBe true + } + + "Delete by query with valid JSON" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_json_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => refresh(index).getStatusLine.getStatusCode < 400) shouldBe true + + indices should contain only "person_to_delete_by_json_query" + + blockUntilCount(3, "person_to_delete_by_json_query") + + "person_to_delete_by_json_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery("person_to_delete_by_json_query", """{"query": {"match_all": {}}}""") + .get + deleteByQuery shouldBe 3L + + "person_to_delete_by_json_query" should haveCount(0) + } + + "Delete by query with SQL DELETE" should "work" in { + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + log.info(s"mapping: $mapping") + pClient + .createIndex("person_to_delete_by_sql_delete_query", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_sql_delete_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => pClient.refresh(index).get) shouldBe true + + indices should contain only "person_to_delete_by_sql_delete_query" + + blockUntilCount(3, "person_to_delete_by_sql_delete_query") + + "person_to_delete_by_sql_delete_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery( + "person_to_delete_by_sql_delete_query", + """DELETE FROM person_to_delete_by_sql_delete_query WHERE uuid = 'A16'""" + ) + .get + deleteByQuery shouldBe 1L + + "person_to_delete_by_sql_delete_query" should haveCount(2) + + pClient + .deleteByQuery( + "person_to_delete_by_sql_delete_query", + "DELETE FROM person_to_delete_by_sql_delete_query" + ) + .get shouldBe 2L + + "person_to_delete_by_sql_delete_query" should haveCount(0) + } + + "Delete by query with SQL SELECT" should "work" in { + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + log.info(s"mapping: $mapping") + pClient + .createIndex("person_to_delete_by_sql_select_query", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_to_delete_by_sql_select_query") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + val indices = result.indices + indices.forall(index => pClient.refresh(index).get) shouldBe true + + indices should contain only "person_to_delete_by_sql_select_query" + + blockUntilCount(3, "person_to_delete_by_sql_select_query") + + "person_to_delete_by_sql_select_query" should haveCount(3) + + val deleteByQuery = pClient + .deleteByQuery( + "person_to_delete_by_sql_select_query", + """SELECT * FROM person_to_delete_by_sql_select_query WHERE uuid = 'A16'""" + ) + .get + deleteByQuery shouldBe 1L + + "person_to_delete_by_sql_select_query" should haveCount(2) + + pClient + .deleteByQuery( + "person_to_delete_by_sql_select_query", + "SELECT * FROM person_to_delete_by_sql_select_query" + ) + .get shouldBe 2L + + "person_to_delete_by_sql_select_query" should haveCount(0) + } + + "Update by query with missing index" should "fail with 404" in { + val result = pClient.updateByQuery( + "person_update_with_missing_index", + """{"query": {"match_all": {}}}""" + ) + + result.isFailure shouldBe true + result.toEither.left.get.statusCode shouldBe Some(404) + } + + "Update by query with missing pipeline" should "fail with 404" in { + pClient.createIndex("person_update_with_missing_pipeline").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_missing_pipeline", + """{"query": {"match_all": {}}}""", + pipelineId = Some("does-not-exist") + ) + + result.isFailure shouldBe true + result.toEither.left.get.statusCode shouldBe Some(404) + } + + "Update by query with invalid SQL" should "fail" in { + pClient.createIndex("person_update_with_invalid_sql").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_invalid_sql", + """UPDATE person_update_sql_where WHERE uuid = 'A16'""" + ) + + result.isFailure shouldBe true + } + + "Update by query with invalid JSON" should "fail" in { + pClient.createIndex("person_update_with_invalid_json").get shouldBe true + val result = pClient.updateByQuery( + "person_update_with_invalid_json", + """{ invalid json }""" + ) + + result.isFailure shouldBe true + } + + "Update by query with SQL UPDATE and WHERE" should "update only matching documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_sql_where", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_sql_where") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + + blockUntilCount(3, "person_update_sql_where") + "person_update_sql_where" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_sql_where", + """UPDATE person_update_sql_where SET name = 'Another Name' WHERE uuid = 'A16'""" + ) + .get + + updated shouldBe 1L + + pClient.refresh("person_update_sql_where").get shouldBe true + + pClient.getAs[Person]("A16", Some("person_update_sql_where")).get match { + case Some(doc) => doc.name shouldBe "Another Name" + case None => fail("Document A16 not found") + } + } + + "Update by query with SQL UPDATE without WHERE" should "update all documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_all_sql", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_all_sql") + val result = + pClient + .bulk[String]( + persons, + identity, + idKey = Some(Set("uuid")) + ) + .get + + result.failedCount shouldBe 0 + result.successCount shouldBe persons.size + + blockUntilCount(3, "person_update_all_sql") + "person_update_all_sql" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_all_sql", + """UPDATE person_update_all_sql SET birthDate = '1972-12-26'""" + ) + .get + + updated shouldBe 3L + + pClient.refresh("person_update_all_sql").get shouldBe true + + val updatedPersons = pClient.searchDocuments("SELECT * FROM person_update_all_sql") + updatedPersons.size shouldBe 3 + updatedPersons.forall(person => person.birthDate == "1972-12-26") shouldBe true + } + + "Update by query with JSON query and user pipeline" should "apply the pipeline to all documents" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_json", mappings = Some(mapping)) + .get shouldBe true + + // User Pipeline + val userPipelineJson = + """{ + | "processors": [ + | { "set": { "field": "birthDate", "value": "1972-12-26" } } + | ] + |} + """.stripMargin + + pClient.createPipeline("set-birthdate-1972-12-26", userPipelineJson).get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_json") + pClient + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) + .get + + blockUntilCount(3, "person_update_json") + "person_update_json" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_json", + """{"query": {"match_all": {}}}""", + pipelineId = Some("set-birthdate-1972-12-26") + ) + .get + + updated shouldBe 3L + + pClient.refresh("person_update_json").get shouldBe true + + val updatedPersons = pClient.searchDocuments("SELECT * FROM person_update_json") + updatedPersons.size shouldBe 3 + updatedPersons.forall(person => person.birthDate == "1972-12-26") shouldBe true + } + + "Update by query with SQL UPDATE and user pipeline" should "merge user and SQL pipelines" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_merge_pipeline", mappings = Some(mapping)) + .get shouldBe true + + val userPipelineJson = + """{ + | "processors": [ + | { "set": { "field": "name", "value": "UPDATED_NAME" } } + | ] + |} + """.stripMargin + + pClient.createPipeline("user-update-name", userPipelineJson).get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_merge_pipeline") + pClient + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) + .get + + blockUntilCount(3, "person_update_merge_pipeline") + "person_update_merge_pipeline" should haveCount(3) + + val updated = pClient + .updateByQuery( + "person_update_merge_pipeline", + """UPDATE person_update_merge_pipeline SET birthDate = '1972-12-26' WHERE uuid = 'A16'""", + pipelineId = Some("user-update-name") + ) + .get + + updated shouldBe 1L + + pClient.refresh("person_update_merge_pipeline").get shouldBe true + + pClient.getAs[Person]("A16", Some("person_update_merge_pipeline")).get match { + case Some(doc) => + doc.name shouldBe "UPDATED_NAME" + doc.birthDate shouldBe "1972-12-26" + case None => fail("Document A16 not found") + } + } + + "Update by query on a closed index" should "open, update, and restore closed state" in { + val mapping = + """{ + | "properties": { + | "uuid": { "type": "keyword" }, + | "name": { "type": "keyword" }, + | "birthDate": { "type": "date" }, + | "childrenCount": { "type": "integer" } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + + pClient + .createIndex("person_update_closed_index", mappings = Some(mapping)) + .get shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person_update_closed_index") + pClient + .bulk[String](persons, identity, idKey = Some(Set("uuid"))) + .get + + blockUntilCount(3, "person_update_closed_index") + "person_update_closed_index" should haveCount(3) + + pClient.closeIndex("person_update_closed_index").get shouldBe true + pClient.isIndexClosed("person_update_closed_index").get shouldBe true + + val updated = pClient + .updateByQuery( + "person_update_closed_index", + """UPDATE person_update_closed_index SET birthDate = '1972-12-26' WHERE uuid = 'A16'""" + ) + .get + + updated shouldBe 1L + + pClient.isIndexClosed("person_update_closed_index").get shouldBe true + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala index a98d3cba..e4ce7e38 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/EmployeeData.scala @@ -52,7 +52,7 @@ trait EmployeeData { _: Suite => implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = Source.fromIterator(() => list.iterator) - client.bulk[String](employees, identity, idKey = Some("id")) match { + client.bulk[String](employees, identity, idKey = Some(Set("id"))) match { case ElasticSuccess(response) => println(s"βœ… Bulk indexing completed:") println(s" - Total items: ${response.metrics.totalDocuments}") diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala new file mode 100644 index 00000000..ce8fa93b --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/GatewayApiIntegrationSpec.scala @@ -0,0 +1,1537 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import akka.stream.scaladsl.Sink +import app.softnetwork.elastic.client.result.{ + DdlResult, + DmlResult, + ElasticResult, + PipelineResult, + QueryResult, + QueryRows, + QueryStream, + QueryStructured, + SQLResult, + TableResult +} +import app.softnetwork.elastic.client.scroll.ScrollMetrics +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.{DoubleValue, IdValue} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.schema.{IngestPipeline, Table} +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import java.time.LocalDate +import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContextExecutor} +import java.util.concurrent.TimeUnit + +// --------------------------------------------------------------------------- +// Base test trait β€” to be mixed with ElasticDockerTestKit +// --------------------------------------------------------------------------- + +trait GatewayApiIntegrationSpec extends AnyFlatSpecLike with Matchers with ScalaFutures { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(30, Seconds)) + + // Provided by concrete test class + def client: GatewayApi + + override def beforeAll(): Unit = { + self.beforeAll() + } + + override def afterAll(): Unit = { + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + self.afterAll() + } + + // ------------------------------------------------------------------------- + // Helper: assert SELECT result type + // ------------------------------------------------------------------------- + + private def normalizeRow(row: Map[String, Any]): Map[String, Any] = { + val updated = row - "_id" - "_index" - "_score" - "_version" - "_sort" + updated.map(entry => + entry._2 match { + case m: Map[_, _] => + entry._1 -> normalizeRow(m.asInstanceOf[Map[String, Any]]) + case seq: Seq[_] if seq.nonEmpty && seq.head.isInstanceOf[Map[_, _]] => + entry._1 -> seq + .asInstanceOf[Seq[Map[String, Any]]] + .map(m => normalizeRow(m)) + case other => entry._1 -> other + } + ) + } + + def assertSelectResult( + res: ElasticResult[QueryResult], + rows: Seq[Map[String, Any]] = Seq.empty + ): Unit = { + res.isSuccess shouldBe true + res.toOption.get match { + case QueryStream(stream) => + val sink = Sink.fold[Seq[Map[String, Any]], (Map[String, Any], ScrollMetrics)](Seq.empty) { + case (acc, (row, _)) => + acc :+ normalizeRow(row) + } + val results = stream.runWith(sink).futureValue + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case QueryStructured(response) => + val results = + response.results.map(normalizeRow) + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case q: QueryRows => + val results = q.rows.map(normalizeRow) + if (rows.nonEmpty) { + results.size shouldBe rows.size + results should contain theSameElementsAs rows + } else { + log.info(s"Rows: $results") + } + case other => fail(s"Unexpected QueryResult type for SELECT: $other") + } + } + + // ------------------------------------------------------------------------- + // Helper: assert DDL result type + // ------------------------------------------------------------------------- + + def assertDdl(res: ElasticResult[QueryResult]): Unit = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[DdlResult] + } + + // ------------------------------------------------------------------------- + // Helper: assert DML result type + // ------------------------------------------------------------------------- + + def assertDml(res: ElasticResult[QueryResult], result: Option[DmlResult] = None): Unit = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[DmlResult] + result match { + case Some(expected) => + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.inserted shouldBe expected.inserted + dml.updated shouldBe expected.updated + dml.deleted shouldBe expected.deleted + case None => // do nothing + } + } + + // ------------------------------------------------------------------------- + // Helper: assert SHOW TABLE result type + // ------------------------------------------------------------------------- + + def assertShowTable(res: ElasticResult[QueryResult]): Table = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[TableResult] + res.toOption.get.asInstanceOf[TableResult].table + } + + // ------------------------------------------------------------------------- + // Helper: assert SHOW PIPELINE result type + // ------------------------------------------------------------------------- + + def assertShowPipeline(res: ElasticResult[QueryResult]): IngestPipeline = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[PipelineResult] + res.toOption.get.asInstanceOf[PipelineResult].pipeline + } + + // ------------------------------------------------------------------------- + // Helper: assert SHOW CREATE result type + // ------------------------------------------------------------------------- + + def assertShowCreate(res: ElasticResult[QueryResult]): String = { + res.isSuccess shouldBe true + res.toOption.get shouldBe a[SQLResult] + res.toOption.get.asInstanceOf[SQLResult].sql + } + + // ------------------------------------------------------------------------- + // SHOW / DESCRIBE TABLE tests + // ------------------------------------------------------------------------- + + behavior of "SHOW / DESCRIBE TABLE" + + it should "return the table schema using SHOW TABLE" in { + val create = + """CREATE TABLE IF NOT EXISTS show_users ( + | id INT NOT NULL, + | name VARCHAR, + | age INT DEFAULT 0 + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val show = client.run("SHOW TABLE show_users").futureValue + val table = assertShowTable(show) + + val showCreate = client.run("SHOW CREATE TABLE show_users").futureValue + val sql = assertShowCreate(showCreate) + sql should include("CREATE OR REPLACE TABLE show_users") + + val ddl = table.ddl + ddl should include("CREATE OR REPLACE TABLE show_users") + ddl should include("id INT NOT NULL") + ddl should include("name VARCHAR") + ddl should include("age INT DEFAULT 0") + } + + it should "describe a table using DESCRIBE TABLE" in { + val create = + """CREATE TABLE IF NOT EXISTS desc_users ( + | id INT NOT NULL, + | name VARCHAR DEFAULT 'anonymous', + | age INT, + | profile STRUCT FIELDS( + | city VARCHAR, + | followers INT + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val res = client.run("DESCRIBE TABLE desc_users").futureValue + res.isSuccess shouldBe true + res.toOption.get shouldBe a[QueryRows] + + val rows = res.toOption.get.asInstanceOf[QueryRows].rows + + rows.exists(_("name") == "id") shouldBe true + rows.exists(_("name") == "name") shouldBe true + rows.exists(_("name") == "profile.city") shouldBe true + } + + // =========================================================================== + // 2. DDL β€” CREATE TABLE, ALTER TABLE, DROP TABLE, TRUNCATE TABLE + // =========================================================================== + + behavior of "DDL statements" + + // --------------------------------------------------------------------------- + // CREATE TABLE β€” full complex schema + // --------------------------------------------------------------------------- + + it should "create a table with complex columns, defaults, scripts, struct, PK, partitioning and options" in { + val sql = + """CREATE TABLE IF NOT EXISTS users ( + | id INT NOT NULL COMMENT 'user identifier', + | name VARCHAR FIELDS(raw Keyword COMMENT 'sortable') DEFAULT 'anonymous' OPTIONS (analyzer = 'french', search_analyzer = 'french'), + | birthdate DATE, + | age INT SCRIPT AS (DATEDIFF(birthdate, CURRENT_DATE, YEAR)), + | ingested_at TIMESTAMP DEFAULT _ingest.timestamp, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) COMMENT 'user profile', + | PRIMARY KEY (id) + |) PARTITION BY birthdate (MONTH), OPTIONS (mappings = (dynamic = false));""".stripMargin + + assertDdl(client.run(sql).futureValue) + + // VΓ©rification via SHOW TABLE + val table = assertShowTable(client.run("SHOW TABLE users").futureValue) + val ddl = table.ddl.replaceAll("\\s+", " ") + + ddl should include("CREATE OR REPLACE TABLE users") + ddl should include("id INT NOT NULL COMMENT 'user identifier'") + ddl should include( + """name VARCHAR FIELDS ( raw KEYWORD COMMENT 'sortable' ) DEFAULT 'anonymous' OPTIONS (analyzer = "french", search_analyzer = "french")""" + ) + ddl should include("birthdate DATE") + ddl should include("age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))") + ddl should include("ingested_at TIMESTAMP DEFAULT _ingest.timestamp") + ddl should include("profile STRUCT FIELDS (") + ddl should include("bio VARCHAR") + ddl should include("followers INT") + ddl should include("join_date DATE") + ddl should include("seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))") + ddl should include("PRIMARY KEY (id)") + ddl should include("PARTITION BY birthdate (MONTH)") + } + + // --------------------------------------------------------------------------- + // CREATE TABLE IF NOT EXISTS + // --------------------------------------------------------------------------- + + it should "create a simple table and allow re-creation with IF NOT EXISTS" in { + val sql = + """CREATE TABLE IF NOT EXISTS accounts ( + | id INT NOT NULL, + | owner VARCHAR, + | balance DOUBLE + |) OPTIONS ( + | settings = ( + | number_of_shards = 1, + | number_of_replicas = 0 + | ) + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + assertDdl(client.run(sql).futureValue) // second call should succeed + } + + // --------------------------------------------------------------------------- + // CREATE OR REPLACE TABLE AS SELECT + // --------------------------------------------------------------------------- + + it should "create or replace a table from a SELECT query" in { + val createSource = + """CREATE TABLE IF NOT EXISTS accounts_src ( + | id INT NOT NULL, + | name VARCHAR, + | active BOOLEAN, + | PRIMARY KEY (id) + |);""".stripMargin + + assertDdl(client.run(createSource).futureValue) + + val insertSource = + """INSERT INTO accounts_src (id, name, active) VALUES + | (1, 'Alice', true), + | (2, 'Bob', false), + | (3, 'Chloe', true);""".stripMargin + + assertDml(client.run(insertSource).futureValue) + + val createOrReplace = + "CREATE OR REPLACE TABLE users_cr AS SELECT id, name FROM accounts_src WHERE active = true;" + + assertDml(client.run(createOrReplace).futureValue) + + // VΓ©rification via SHOW TABLE + val table = assertShowTable(client.run("SHOW TABLE users_cr").futureValue) + table.ddl should include("CREATE OR REPLACE TABLE users_cr") + table.ddl should include("id INT") + table.ddl should include("name VARCHAR") + } + + // --------------------------------------------------------------------------- + // DROP TABLE + // --------------------------------------------------------------------------- + + it should "drop a table if it exists" in { + val create = + """CREATE TABLE IF NOT EXISTS tmp_drop ( + | id INT NOT NULL, + | value VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val drop = + """DROP TABLE IF EXISTS tmp_drop;""".stripMargin + + assertDdl(client.run(drop).futureValue) + } + + // --------------------------------------------------------------------------- + // TRUNCATE TABLE + // --------------------------------------------------------------------------- + + it should "truncate a table" in { + val create = + """CREATE TABLE IF NOT EXISTS tmp_truncate ( + | id INT NOT NULL, + | value VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO tmp_truncate (id, value) VALUES + | (1, 'a'), + | (2, 'b'), + | (3, 'c');""".stripMargin + + assertDml(client.run(insert).futureValue) + + val truncate = "TRUNCATE TABLE tmp_truncate;" + assertDdl(client.run(truncate).futureValue) + + // VΓ©rification : SELECT doit renvoyer 0 lignes + val select = client.run("SELECT * FROM tmp_truncate").futureValue + assertSelectResult(select) + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” ADD COLUMN + // --------------------------------------------------------------------------- + + it should "add a column IF NOT EXISTS with DEFAULT" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter1 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter1 + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter1").futureValue) + table.ddl should include("age INT DEFAULT 0") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” RENAME COLUMN + // --------------------------------------------------------------------------- + + it should "rename a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter2 ( + | id INT NOT NULL, + | name VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter2 + | RENAME COLUMN name TO full_name;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter2").futureValue) + table.ddl should include("full_name VARCHAR") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” COMMENT + // --------------------------------------------------------------------------- + + it should "set and drop a column COMMENT" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter3 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setComment = + """ALTER TABLE users_alter3 + | ALTER COLUMN IF EXISTS status SET COMMENT 'a description';""".stripMargin + + val dropComment = + """ALTER TABLE users_alter3 + | ALTER COLUMN IF EXISTS status DROP COMMENT;""".stripMargin + + assertDdl(client.run(setComment).futureValue) + assertDdl(client.run(dropComment).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter3").futureValue) + table.ddl should not include "COMMENT 'a description'" + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” DEFAULT + // --------------------------------------------------------------------------- + + it should "set and drop a DEFAULT value on a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter4 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setDefault = + """ALTER TABLE users_alter4 + | ALTER COLUMN status SET DEFAULT 'active';""".stripMargin + + val dropDefault = + """ALTER TABLE users_alter4 + | ALTER COLUMN status DROP DEFAULT;""".stripMargin + + assertDdl(client.run(setDefault).futureValue) + assertDdl(client.run(dropDefault).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter4").futureValue) + table.ddl should not include "DEFAULT 'active'" + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” STRUCT SET FIELDS + // --------------------------------------------------------------------------- + + it should "alter a STRUCT column with SET FIELDS" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter5 ( + | id INT NOT NULL, + | profile STRUCT FIELDS( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)) + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter5 + | ALTER COLUMN profile SET FIELDS ( + | bio VARCHAR, + | followers INT, + | join_date DATE, + | seniority INT SCRIPT AS (DATEDIFF(profile.join_date, CURRENT_DATE, DAY)), + | reputation DOUBLE DEFAULT 0.0 + | );""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter5").futureValue) + table.find("profile.reputation") match { + case Some(col) => + col.dataType shouldBe SQLTypes.Double + col.defaultValue shouldBe Some(DoubleValue(0.0)) + case _ => fail("Column 'profile.reputation' not found") + } + table.defaultPipeline.processors.size shouldBe 3 // added processor to create an artificial primary key + table.find("users_alter5_id") match { + case Some(col) => + col.dataType shouldBe SQLTypes.Keyword + col.defaultValue shouldBe Some(IdValue) + col.processors.size shouldBe 1 + case _ => fail("Column 'users_alter5_id' not found") + } + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” NOT NULL + // --------------------------------------------------------------------------- + + it should "set and drop NOT NULL on a column" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter6 ( + | id INT, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val setNotNull = + """ALTER TABLE users_alter6 + | ALTER COLUMN status SET NOT NULL;""".stripMargin + + val dropNotNull = + """ALTER TABLE users_alter6 + | ALTER COLUMN status DROP NOT NULL;""".stripMargin + + assertDdl(client.run(setNotNull).futureValue) + assertDdl(client.run(dropNotNull).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter6").futureValue) + table.find("status") match { + case Some(col) => col.nullable shouldBe true + case _ => fail("Column 'status' not found") + } + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” TYPE CHANGE (UNSAFE β†’ migration) + // --------------------------------------------------------------------------- + + it should "change a column data type (UNSAFE β†’ migration)" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter7 ( + | id INT NOT NULL, + | status VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter7 + | ALTER COLUMN id SET DATA TYPE BIGINT;""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter7").futureValue) + table.ddl should include("id BIGINT NOT NULL") + } + + // --------------------------------------------------------------------------- + // ALTER TABLE β€” multi-alteration + // --------------------------------------------------------------------------- + + it should "apply multiple alterations in a single ALTER TABLE statement" in { + val create = + """CREATE TABLE IF NOT EXISTS users_alter8 ( + | id INT NOT NULL, + | name VARCHAR, + | status VARCHAR, + | profile STRUCT FIELDS( + | description VARCHAR, + | visibility BOOLEAN + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val alter = + """ALTER TABLE users_alter8 ( + | ADD COLUMN IF NOT EXISTS age INT DEFAULT 0, + | RENAME COLUMN name TO full_name, + | ALTER COLUMN IF EXISTS status SET DEFAULT 'active', + | ALTER COLUMN IF EXISTS profile SET FIELDS ( + | description VARCHAR DEFAULT 'N/A', + | visibility BOOLEAN DEFAULT true + | ) + |);""".stripMargin + + assertDdl(client.run(alter).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE users_alter8").futureValue) + + table.ddl should include("age INT DEFAULT 0") + table.ddl should include("full_name VARCHAR") + table.ddl should include("status VARCHAR DEFAULT 'active'") + table.ddl should include("description VARCHAR DEFAULT 'N/A'") + table.ddl should include("visibility BOOLEAN DEFAULT true") + } + + // =========================================================================== + // 3. DML β€” INSERT / UPDATE / DELETE + // =========================================================================== + + behavior of "DML statements" + + // --------------------------------------------------------------------------- + // INSERT + // --------------------------------------------------------------------------- + + it should "insert rows into a table and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_users ( + | id INT NOT NULL, + | name VARCHAR, + | age INT + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_users (id, name, age) VALUES + | (1, 'Alice', 30), + | (2, 'Bob', 40), + | (3, 'Chloe', 25);""".stripMargin + + val res = client.run(insert).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.inserted shouldBe 3 + + // VΓ©rification via SELECT + val select = client.run("SELECT * FROM dml_users ORDER BY id ASC").futureValue + assertSelectResult(select) + } + + // --------------------------------------------------------------------------- + // UPDATE + // --------------------------------------------------------------------------- + + it should "update rows using UPDATE ... WHERE and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_accounts ( + | id INT NOT NULL, + | owner KEYWORD, + | balance DOUBLE + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_accounts (id, owner, balance) VALUES + | (1, 'Alice', 100.00), + | (2, 'Bob', 50.00), + | (3, 'Chloe', 75.00);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val update = + """UPDATE dml_accounts + |SET balance = 125 + |WHERE owner = 'Alice';""".stripMargin + + val res = client.run(update).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.updated should be >= 1L + + // VΓ©rification via SELECT + val select = + """SELECT owner, balance + |FROM dml_accounts + |WHERE owner = 'Alice';""".stripMargin + + val q = client.run(select).futureValue + assertSelectResult(q) + } + + // --------------------------------------------------------------------------- + // DELETE + // --------------------------------------------------------------------------- + + it should "delete rows using DELETE ... WHERE and return a DmlResult" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_logs ( + | id INT NOT NULL, + | level KEYWORD, + | message VARCHAR + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_logs (id, level, message) VALUES + | (1, 'INFO', 'started'), + | (2, 'ERROR', 'failed'), + | (3, 'INFO', 'running');""".stripMargin + + assertDml(client.run(insert).futureValue) + + val delete = + """DELETE FROM dml_logs + |WHERE level = 'ERROR';""".stripMargin + + val res = client.run(delete).futureValue + assertDml(res) + + val dml = res.toOption.get.asInstanceOf[DmlResult] + dml.deleted shouldBe 1L + + // VΓ©rification via SELECT + val select = + """SELECT id, level + |FROM dml_logs + |ORDER BY id ASC;""".stripMargin + + val q = client.run(select).futureValue + assertSelectResult(q) + } + + // --------------------------------------------------------------------------- + // INSERT + UPDATE + DELETE chain + // --------------------------------------------------------------------------- + + it should "support a full DML lifecycle: INSERT β†’ UPDATE β†’ DELETE" in { + val create = + """CREATE TABLE IF NOT EXISTS dml_chain ( + | id INT NOT NULL, + | value INT + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dml_chain (id, value) VALUES + | (1, 10), + | (2, 20), + | (3, 30);""".stripMargin + + assertDml(client.run(insert).futureValue) + + val update = + """UPDATE dml_chain + |SET value = 50 + |WHERE id IN (1, 3);""".stripMargin + + assertDml(client.run(update).futureValue) + + val delete = + """DELETE FROM dml_chain + |WHERE value > 40;""".stripMargin + + assertDml(client.run(delete).futureValue) + + // VΓ©rification finale + val select = client.run("SELECT * FROM dml_chain ORDER BY id ASC").futureValue + assertSelectResult(select) + } + + // --------------------------------------------------------------------------- + // COPY INTO integration test + // --------------------------------------------------------------------------- + + it should "support COPY INTO for JSONL and JSON_ARRAY formats" in { + // 1. Create table with primary key + val create = + """CREATE TABLE IF NOT EXISTS copy_into_test ( + | uuid KEYWORD NOT NULL, + | name VARCHAR, + | birthDate DATE, + | childrenCount INT, + | PRIMARY KEY (uuid) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + // 2. Prepare sample documents + val persons = List( + """{"uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0}""", + """{"uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0}""", + """{"uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 2}""" + ) + + // 3. Create a temporary JSONL file + val jsonlFile = java.io.File.createTempFile("copy_into_jsonl", ".jsonl") + jsonlFile.deleteOnExit() + val writer1 = new java.io.PrintWriter(jsonlFile) + persons.foreach(writer1.println) + writer1.close() + + // 4. COPY INTO using JSONL (format inferred) + val copyJsonl = + s"""COPY INTO copy_into_test FROM "${jsonlFile.getAbsolutePath}";""" + + val jsonlResult = client.run(copyJsonl).futureValue + assertDml(jsonlResult, Some(DmlResult(inserted = persons.size))) + + // 5. Create a temporary JSON_ARRAY file + val jsonArrayFile = java.io.File.createTempFile("copy_into_array", ".json") + jsonArrayFile.deleteOnExit() + val writer2 = new java.io.PrintWriter(jsonArrayFile) + writer2.println("[") + writer2.println(persons.mkString(",\n")) + writer2.println("]") + writer2.close() + + // 6. COPY INTO using JSON_ARRAY + val copyArray = + s"""COPY INTO copy_into_test FROM "${jsonArrayFile.getAbsolutePath}" FILE_FORMAT = JSON_ARRAY ON CONFLICT DO UPDATE;""" + + val arrayResult = client.run(copyArray).futureValue + assertDml(arrayResult, Some(DmlResult(inserted = persons.size))) + + // 7. Final verification: SELECT all documents + val select = client.run("SELECT * FROM copy_into_test ORDER BY uuid ASC").futureValue + assertSelectResult( + select, + Seq( + Map( + "uuid" -> "A12", + "name" -> "Homer Simpson", + "birthDate" -> LocalDate.parse("1967-11-21"), + "childrenCount" -> 0 + ), + Map( + "uuid" -> "A14", + "name" -> "Moe Szyslak", + "birthDate" -> LocalDate.parse("1967-11-21"), + "childrenCount" -> 0 + ), + Map( + "uuid" -> "A16", + "name" -> "Barney Gumble", + "birthDate" -> LocalDate.parse("1969-05-09"), + "childrenCount" -> 2 + ) + ) + ) + } + + // =========================================================================== + // 4. DQL β€” SELECT, JOIN, UNNEST, GROUP BY, HAVING, ORDER BY, LIMIT/OFFSET, + // functions, window functions, nested fields, UNION ALL + // =========================================================================== + + behavior of "DQL statements" + + // --------------------------------------------------------------------------- + // Setup for DQL tests: base table with nested fields + // --------------------------------------------------------------------------- + + it should "prepare DQL test data" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_users ( + | id INT NOT NULL, + | name VARCHAR FIELDS( + | raw KEYWORD + | ) OPTIONS (fielddata = true), + | age INT, + | birthdate DATE, + | profile STRUCT FIELDS( + | city VARCHAR OPTIONS (fielddata = true), + | followers INT + | ) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_users (id, name, age, birthdate, profile) VALUES + | (1, 'Alice', 30, '1994-01-01', {city = "Paris", followers = 100}), + | (2, 'Bob', 40, '1984-05-10', {city = "Lyon", followers = 50}), + | (3, 'Chloe', 25, '1999-07-20', {city = "Paris", followers = 200}), + | (4, 'David', 50, '1974-03-15', {city = "Marseille", followers = 10}); + |""".stripMargin + + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 4))) + } + + // --------------------------------------------------------------------------- + // SELECT simple + alias + nested fields + // --------------------------------------------------------------------------- + + it should "execute a simple SELECT with aliases and nested fields" in { + val sql = + """SELECT id, + | name AS full_name, + | profile.city AS city, + | profile.followers AS followers + |FROM dql_users + |ORDER BY id ASC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "full_name" -> "Alice", + "city" -> "Paris", + "followers" -> 100, + "profile" -> Map("city" -> "Paris", "followers" -> 100) + ), + Map( + "id" -> 2, + "full_name" -> "Bob", + "city" -> "Lyon", + "followers" -> 50, + "profile" -> Map("city" -> "Lyon", "followers" -> 50) + ), + Map( + "id" -> 3, + "full_name" -> "Chloe", + "city" -> "Paris", + "followers" -> 200, + "profile" -> Map("city" -> "Paris", "followers" -> 200) + ), + Map( + "id" -> 4, + "full_name" -> "David", + "city" -> "Marseille", + "followers" -> 10, + "profile" -> Map("city" -> "Marseille", "followers" -> 10) + ) + ) + ) + } + + // --------------------------------------------------------------------------- + // UNION ALL + // --------------------------------------------------------------------------- + + it should "execute a UNION ALL query" in { + val sql = + """SELECT id, name FROM dql_users WHERE age > 30 + |UNION ALL + |SELECT id, name FROM dql_users WHERE age <= 30;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map("id" -> 2, "name" -> "Bob"), + Map("id" -> 4, "name" -> "David"), + Map("id" -> 1, "name" -> "Alice"), + Map("id" -> 3, "name" -> "Chloe") + ) + ) + } + + // --------------------------------------------------------------------------- + // JOIN + UNNEST + // --------------------------------------------------------------------------- + + it should "execute a JOIN with UNNEST on array of structs" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_orders ( + | id INT NOT NULL, + | customer_id INT, + | items ARRAY FIELDS( + | product VARCHAR OPTIONS (fielddata = true), + | quantity INT, + | price DOUBLE + | ) OPTIONS (include_in_parent = false) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE dql_orders").futureValue) + table.ddl should include("items ARRAY FIELDS") + + val insert = + """INSERT INTO dql_orders (id, customer_id, items) VALUES + | (1, 1, [ { product = "A", quantity = 2, price = 10.0 }, + | { product = "B", quantity = 1, price = 20.0 } ]), + | (2, 2, [ { product = "C", quantity = 3, price = 5.0 } ]);""".stripMargin + + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 2))) + + val sql = + """SELECT + | o.id, + | items.product, + | items.quantity, + | SUM(items.price * items.quantity) OVER (PARTITION BY o.id) AS total_price + |FROM dql_orders o + |JOIN UNNEST(o.items) AS items + |WHERE items.quantity >= 1 + |ORDER BY o.id ASC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "items" -> Seq( + Map("product" -> "A", "quantity" -> 2, "price" -> 10.0), + Map("product" -> "B", "quantity" -> 1, "price" -> 20.0) + ), + "total_price" -> 40.0 + ), + Map( + "id" -> 2, + "items" -> Seq(Map("product" -> "C", "quantity" -> 3, "price" -> 5.0)), + "total_price" -> 15.0 + ) + ) + ) + } + + // --------------------------------------------------------------------------- + // WHERE β€” complex conditions + // --------------------------------------------------------------------------- + + it should "support WHERE with complex logical and comparison operators" in { + val sql = + """SELECT id, name, age + |FROM dql_users + |WHERE (age > 20 AND profile.followers >= 100) + | OR (profile.city = 'Lyon' AND age < 50) + |ORDER BY age DESC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map("id" -> 1, "name" -> "Alice", "age" -> 30), + Map("id" -> 3, "name" -> "Chloe", "age" -> 25) + ) + ) + } + + // --------------------------------------------------------------------------- + // ORDER BY + LIMIT + OFFSET + // --------------------------------------------------------------------------- + + it should "support ORDER BY with multiple fields, LIMIT and OFFSET" in { + val sql = + """SELECT id, name, age + |FROM dql_users + |ORDER BY age DESC, name ASC + |LIMIT 2 OFFSET 1;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // GROUP BY + HAVING + // --------------------------------------------------------------------------- + + it should "support GROUP BY with aggregations and HAVING" in { + val sql = + """SELECT profile.city AS city, + | COUNT(*) AS cnt, + | AVG(age) AS avg_age + |FROM dql_users + |GROUP BY profile.city + |HAVING COUNT(*) >= 1 + |ORDER BY COUNT(*) DESC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Arithmetic, IN, BETWEEN, IS NULL, LIKE, RLIKE + // --------------------------------------------------------------------------- + + it should "support arithmetic, IN, BETWEEN, IS NULL, LIKE, RLIKE" in { + val sql = + """SELECT id, + | age + 10 AS age_plus_10, + | name + |FROM dql_users + |WHERE age BETWEEN 20 AND 50 + | AND name IN ('Alice', 'Bob', 'Chloe') + | AND name IS NOT NULL + | AND (name LIKE 'A%' OR name RLIKE '.*o.*');""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // CAST, TRY_CAST, SAFE_CAST, operator :: + // --------------------------------------------------------------------------- + + it should "support CAST, TRY_CAST, SAFE_CAST and the :: operator" in { + val sql = + """SELECT id, + | age::BIGINT AS age_bigint, + | CAST(age AS DOUBLE) AS age_double, + | TRY_CAST('123' AS INT) AS try_cast_ok, + | SAFE_CAST('abc' AS INT) AS safe_cast_null + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Numeric + Trigonometric functions + // --------------------------------------------------------------------------- + + it should "support numeric and trigonometric functions" in { + val sql = + """SELECT id, + | ABS(age) AS abs_age, + | CEIL(age) AS ceil_div, + | FLOOR(age) AS floor_div, + | ROUND(age, 2) AS round_div, + | SQRT(age) AS sqrt_age, + | POW(age, 2) AS pow_age, + | LOG(age) AS log_age, + | LOG10(age) AS log10_age, + | EXP(age) AS exp_age, + | SIN(age) AS sin_age, + | COS(age) AS cos_age, + | TAN(age) AS tan_age + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // String functions + // --------------------------------------------------------------------------- + + it should "support string functions" in { + val sql = + """SELECT id, + | CONCAT(name.raw, '_suffix') AS name_concat, + | SUBSTRING(name.raw, 1, 2) AS name_sub, + | LOWER(name.raw) AS name_lower, + | UPPER(name.raw) AS name_upper, + | TRIM(name.raw) AS name_trim, + | LENGTH(name.raw) AS name_len, + | REPLACE(name.raw, 'A', 'X') AS name_repl, + | LEFT(name.raw, 1) AS name_left, + | RIGHT(name.raw, 1) AS name_right, + | REVERSE(name.raw) AS name_rev + |FROM dql_users + |ORDER BY id ASC;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map( + "id" -> 1, + "name_concat" -> Seq("Alice_suffix"), + "name_sub" -> Seq("Al"), + "name_lower" -> Seq("alice"), + "name_upper" -> Seq("ALICE"), + "name_trim" -> Seq("Alice"), + "name_len" -> Seq(5), + "name_repl" -> Seq("Xlice"), + "name_left" -> Seq("A"), + "name_right" -> Seq("e"), + "name_rev" -> Seq("ecilA") + ), + Map( + "id" -> 2, + "name_concat" -> Seq("Bob_suffix"), + "name_sub" -> Seq("Bo"), + "name_lower" -> Seq("bob"), + "name_upper" -> Seq("BOB"), + "name_trim" -> Seq("Bob"), + "name_len" -> Seq(3), + "name_repl" -> Seq("Bob"), + "name_left" -> Seq("B"), + "name_right" -> Seq("b"), + "name_rev" -> Seq("boB") + ), + Map( + "id" -> 3, + "name_concat" -> Seq("Chloe_suffix"), + "name_sub" -> Seq("Ch"), + "name_lower" -> Seq("chloe"), + "name_upper" -> Seq("CHLOE"), + "name_trim" -> Seq("Chloe"), + "name_len" -> Seq(5), + "name_repl" -> Seq("Chloe"), + "name_left" -> Seq("C"), + "name_right" -> Seq("e"), + "name_rev" -> Seq("eolhC") + ), + Map( + "id" -> 4, + "name_concat" -> Seq("David_suffix"), + "name_sub" -> Seq("Da"), + "name_lower" -> Seq("david"), + "name_upper" -> Seq("DAVID"), + "name_trim" -> Seq("David"), + "name_len" -> Seq(5), + "name_repl" -> Seq("David"), + "name_left" -> Seq("D"), + "name_right" -> Seq("d"), + "name_rev" -> Seq("divaD") + ) + ) + ) + } + + // --------------------------------------------------------------------------- + // Date / Time functions + // --------------------------------------------------------------------------- + + it should "support date and time functions" in { + val sql = + """SELECT id, + | YEAR(CURRENT_DATE) AS current_year, + | MONTH(CURRENT_DATE) AS current_month, + | DAY(CURRENT_DATE) AS current_day, + | WEEKDAY(CURRENT_DATE) AS current_weekday, + | YEARDAY(CURRENT_DATE) AS current_yearday, + | HOUR(CURRENT_TIMESTAMP) AS current_hour, + | MINUTE(CURRENT_TIMESTAMP) AS current_minute, + | SECOND(CURRENT_TIMESTAMP) AS current_second, + | NANOSECOND(CURRENT_TIMESTAMP) AS current_nano, + | MICROSECOND(CURRENT_TIMESTAMP) AS current_micro, + | MILLISECOND(CURRENT_TIMESTAMP) AS current_milli, + | YEAR(birthdate) AS year_b, + | MONTH(birthdate) AS month_b, + | DAY(birthdate) AS day_b, + | WEEKDAY(birthdate) AS weekday_b, + | YEARDAY(birthdate) AS yearday_b, + | HOUR(birthdate) AS hour_b, + | MINUTE(birthdate) AS minute_b, + | SECOND(birthdate) AS second_b, + | NANOSECOND(birthdate) AS nano_b, + | MICROSECOND(birthdate) AS micro_b, + | MILLISECOND(birthdate) AS milli_b, + | OFFSET_SECONDS(birthdate) AS epoch_day, + | DATE_TRUNC(birthdate, MONTH) AS trunc_month, + | DATE_ADD(birthdate, INTERVAL 1 DAY) AS plus_one_day, + | DATE_SUB(birthdate, INTERVAL 1 DAY) AS minus_one_day, + | DATE_DIFF(CURRENT_DATE, birthdate, YEAR) AS diff_years + |FROM dql_users;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Geospatial functions + // --------------------------------------------------------------------------- + + it should "support geospatial functions POINT and ST_DISTANCE" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_geo ( + | id INT NOT NULL, + | location GEO_POINT, + | PRIMARY KEY (id) + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val table = assertShowTable(client.run("SHOW TABLE dql_geo").futureValue) + table.ddl should include("location GEO_POINT") + table.find("location").exists(_.dataType == SQLTypes.GeoPoint) shouldBe true + + val insert = + """INSERT INTO dql_geo (id, location) VALUES + | (1, {lon = 2.3522, lat = 48.8566}), + | (2, {lon = 4.8357, lat = 45.7640});""".stripMargin + + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 2))) + + val sql = + """SELECT id, + | ST_DISTANCE(location, POINT(2.3522, 48.8566)) AS dist_paris + |FROM dql_geo;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult(res) + } + + // --------------------------------------------------------------------------- + // Window functions + // --------------------------------------------------------------------------- + + it should "support window functions with OVER clause" in { + val create = + """CREATE TABLE IF NOT EXISTS dql_sales ( + | id INT NOT NULL, + | product KEYWORD, + | customer VARCHAR, + | amount DOUBLE, + | ts TIMESTAMP + |);""".stripMargin + + assertDdl(client.run(create).futureValue) + + val insert = + """INSERT INTO dql_sales (id, product, customer, amount, ts) VALUES + | (1, 'A', 'C1', 10.0, '2024-01-01T10:00:00Z'), + | (2, 'A', 'C2', 20.0, '2024-01-01T11:00:00Z'), + | (3, 'B', 'C1', 30.0, '2024-01-01T12:00:00Z'), + | (4, 'A', 'C3', 40.0, '2024-01-01T13:00:00Z');""".stripMargin + + assertDml(client.run(insert).futureValue, Some(DmlResult(inserted = 4))) + + val sql = + """SELECT + | product, + | customer, + | amount, + | SUM(amount) OVER (PARTITION BY product) AS sum_per_product, + | COUNT(_id) OVER (PARTITION BY product) AS cnt_per_product, + | FIRST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS first_amount, + | LAST_VALUE(amount) OVER (PARTITION BY product ORDER BY ts ASC) AS last_amount, + | ARRAY_AGG(amount) OVER (PARTITION BY product ORDER BY ts ASC LIMIT 10) AS amounts_array + |FROM dql_sales + |ORDER BY product, ts;""".stripMargin + + val res = client.run(sql).futureValue + assertSelectResult( + res, + Seq( + Map( + "product" -> "A", + "customer" -> "C1", + "amount" -> 10.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "A", + "customer" -> "C2", + "amount" -> 20.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "A", + "customer" -> "C3", + "amount" -> 40.0, + "sum_per_product" -> 70.0, + "cnt_per_product" -> 3, + "first_amount" -> 10.0, + "last_amount" -> 40.0, + "amounts_array" -> Seq(10.0, 20.0, 40.0) + ), + Map( + "product" -> "B", + "customer" -> "C1", + "amount" -> 30.0, + "sum_per_product" -> 30.0, + "cnt_per_product" -> 1, + "first_amount" -> 30.0, + "last_amount" -> 30.0, + "amounts_array" -> Seq(30.0) + ) + ) + ) + } + + // =========================================================================== + // 5. PIPELINES β€” CREATE / ALTER / DROP / SHOW + // =========================================================================== + + behavior of "PIPELINE statements" + + // --------------------------------------------------------------------------- + // CREATE OR REPLACE PIPELINE with full processor set + // --------------------------------------------------------------------------- + + it should "create or replace a pipeline with SET, SCRIPT, DATE_INDEX_NAME processors" in { + val sql = + """CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ), + | SCRIPT ( + | description = "age INT SCRIPT AS (DATE_DIFF(birthdate, CURRENT_DATE, YEAR))", + | lang = "painless", + | source = "def param1 = ctx.birthdate; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.age = (param1 == null) ? null : Long.valueOf(ChronoUnit.YEARS.between(param1, param2))", + | ignore_failure = true + | ), + | SET ( + | field = "ingested_at", + | if = "ctx.ingested_at == null", + | description = "DEFAULT _ingest.timestamp", + | ignore_failure = true, + | value = "_ingest.timestamp" + | ), + | SCRIPT ( + | description = "profile.seniority INT SCRIPT AS (DATE_DIFF(profile.join_date, CURRENT_DATE, DAY))", + | lang = "painless", + | source = "def param1 = ctx.profile?.join_date; def param2 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')).toLocalDate(); ctx.profile.seniority = (param1 == null) ? null : Long.valueOf(ChronoUnit.DAYS.between(param1, param2))", + | ignore_failure = true + | ), + | DATE_INDEX_NAME ( + | field = "birthdate", + | index_name_prefix = "users-", + | date_formats = ["yyyy-MM"], + | ignore_failure = true, + | date_rounding = "M", + | description = "PARTITION BY birthdate (MONTH)", + | separator = "-" + | ), + | SET ( + | field = "_id", + | description = "PRIMARY KEY (id)", + | ignore_failure = false, + | ignore_empty_value = false, + | value = "{{id}}" + | ) + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + + val pipeline = assertShowPipeline(client.run("SHOW PIPELINE user_pipeline").futureValue) + pipeline.name shouldBe "user_pipeline" + pipeline.processors.size shouldBe 6 + + val showCreate = client.run("SHOW CREATE PIPELINE user_pipeline").futureValue + val ddl = assertShowCreate(showCreate) + ddl should include("CREATE OR REPLACE PIPELINE user_pipeline") + } + + // --------------------------------------------------------------------------- + // ALTER PIPELINE β€” ADD PROCESSOR, DROP PROCESSOR + // --------------------------------------------------------------------------- + + it should "alter an existing pipeline by adding and dropping processors" in { + val sql = + """ALTER PIPELINE IF EXISTS user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | if = "ctx.status == null", + | description = "status DEFAULT 'active'", + | ignore_failure = true, + | value = "active" + | ), + | DROP PROCESSOR SET (_id) + |);""".stripMargin + + assertDdl(client.run(sql).futureValue) + } + + // --------------------------------------------------------------------------- + // DROP PIPELINE + // --------------------------------------------------------------------------- + + it should "drop a pipeline" in { + val sql = "DROP PIPELINE IF EXISTS user_pipeline;" + assertDdl(client.run(sql).futureValue) + } + + // =========================================================================== + // 6. ERRORS β€” parsing errors, unsupported SQL + // =========================================================================== + + behavior of "SQL error handling" + + // --------------------------------------------------------------------------- + // Parsing error + // --------------------------------------------------------------------------- + + it should "return a parsing error for invalid SQL" in { + val invalidSql = "CREAT TABL missing_keyword" + val res = client.run(invalidSql).futureValue + + res.isFailure shouldBe true + res.toEither.left.get.message should include("Error parsing schema DDL statement") + } + + // --------------------------------------------------------------------------- + // Unsupported SQL + // --------------------------------------------------------------------------- + + it should "return an error for unsupported SQL statements" in { + val unsupportedSql = "GRANT SELECT ON users TO user1" + val res = client.run(unsupportedSql).futureValue + + res.isFailure shouldBe true + res.toEither.left.get.message should include("Error parsing schema DDL statement") + } + +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala new file mode 100644 index 00000000..e52a00f1 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/InsertByQuerySpec.scala @@ -0,0 +1,263 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk.BulkOptions +import app.softnetwork.elastic.client.result.{DmlResult, ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.ExecutionContextExecutor +import scala.language.implicitConversions + +trait InsertByQuerySpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: ElasticClientApi + + // --------------------------------------------------------------------------- + // FIXTURES : MAPPINGS + // --------------------------------------------------------------------------- + + private val customersMapping: String = + """{ + |"properties":{ + | "customer_id":{"type":"keyword"}, + | "name":{"type":"text"}, + | "email":{"type":"keyword"}, + | "country":{"type":"keyword"} + |}, + |"_meta":{"primary_key":["customer_id"]} + |}""".stripMargin.replaceAll("\\s+", "") + + private val productsMapping: String = + """{ + |"properties":{ + | "sku":{"type":"keyword"}, + | "name":{"type":"text"}, + | "price":{"type":"double"}, + | "category":{"type":"keyword"} + |}, + |"_meta":{"primary_key":["sku"]} + |}""".stripMargin.replaceAll("\\s+", "") + + private val ordersMapping: String = + """{ + |"properties":{ + | "order_id":{"type":"keyword"}, + | "customer_id":{"type":"keyword"}, + | "order_date":{"type":"date"}, + | "total":{"type":"double"}, + | "items":{ + | "type":"nested", + | "properties":{ + | "sku":{"type":"keyword"}, + | "qty":{"type":"integer"} + | } + | } + |}, + |"_meta":{ + | "primary_key":["order_id","customer_id"], + | "partition_by":{"column":"order_date","granularity":"d"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + private val stagingOrdersMapping: String = + """{ + |"properties":{ + | "id":{"type":"keyword"}, + | "cust":{"type":"keyword"}, + | "date":{"type":"date"}, + | "amount":{"type":"double"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + private val stagingCustomersMapping: String = + """{ + |"properties":{ + | "id":{"type":"keyword"}, + | "fullname":{"type":"text"}, + | "email":{"type":"keyword"} + |} + |}""".stripMargin.replaceAll("\\s+", "") + + // --------------------------------------------------------------------------- + // FIXTURES : DATASETS JSON PURS + // --------------------------------------------------------------------------- + + private val customers: List[String] = List( + """{"customer_id":"C001","name":"Alice","email":"alice@example.com","country":"FR"}""", + """{"customer_id":"C002","name":"Bob","email":"bob@example.com","country":"US"}""", + """{"customer_id":"C003","name":"Charlie","email":"charlie@example.com","country":"DE"}""" + ) + + private val products: List[String] = List( + """{"sku":"SKU-001","name":"Laptop","price":1299.99,"category":"electronics"}""", + """{"sku":"SKU-002","name":"Mouse","price":29.99,"category":"electronics"}""", + """{"sku":"SKU-003","name":"Desk Chair","price":199.99,"category":"furniture"}""" + ) + + private val orders: List[String] = List( + """{"order_id":"O1001","customer_id":"C001","order_date":"2024-01-10","total":1299.99,"items":[{"sku":"SKU-001","qty":1}]}""", + """{"order_id":"O1002","customer_id":"C002","order_date":"2024-01-11","total":29.99,"items":[{"sku":"SKU-002","qty":1}]}""" + ) + + private val stagingOrders: List[String] = List( + """{"id":"O2001","cust":"C001","date":"2024-02-01","amount":1299.99}""", + """{"id":"O2002","cust":"C003","date":"2024-02-02","amount":199.99}""", + """{"id":"O2003","cust":"C002","date":"2024-02-03","amount":29.99}""" + ) + + private val stagingOrdersUpdates: List[String] = List( + """{"id":"O1001","cust":"C001","date":"2024-01-10","amount":1499.99}""", + """{"id":"O1002","cust":"C002","date":"2024-01-11","amount":39.99}""" + ) + + private val stagingCustomers: List[String] = List( + """{"id":"C010","fullname":"Bob Martin","email":"bob.martin@example.com"}""", + """{"id":"C011","fullname":"Jane Doe","email":"jane.doe@example.com"}""" + ) + + // --------------------------------------------------------------------------- + // FIXTURE : INITIALISATION DES INDEXES + // --------------------------------------------------------------------------- + + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + + private def initIndex(name: String, mapping: String, docs: List[String]): Unit = { + client.createIndex(name, mappings = Some(mapping)) + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = name) + client.bulk[String](docs, identity, Some(name)) + client.refresh(name) + } + + // --------------------------------------------------------------------------- + // TESTS D’INTΓ‰GRATION COMPLETS + // --------------------------------------------------------------------------- + + behavior of "insertByQuery" + + it should "init all indices" in { + initIndex("customers", customersMapping, customers) + initIndex("products", productsMapping, products) + initIndex("orders", ordersMapping, orders) + initIndex("staging_orders", stagingOrdersMapping, stagingOrders) + initIndex("staging_orders_updates", stagingOrdersMapping, stagingOrdersUpdates) + initIndex("staging_customers", stagingCustomersMapping, stagingCustomers) + } + + it should "insert a customer with INSERT ... VALUES" in { + val sql = + """INSERT INTO customers (customer_id, name, email, country) + |VALUES ('C010', 'Bob', 'bob@example.com', 'FR')""".stripMargin + + val result = client.insertByQuery("customers", sql).futureValue + result shouldBe ElasticSuccess(DmlResult(inserted = 1L)) + } + + it should "upsert a product with ON CONFLICT DO UPDATE" in { + val sql = + """INSERT INTO products (sku, name, price, category) + |VALUES ('SKU-001', 'Laptop Pro', 1499.99, 'electronics') + |ON CONFLICT DO UPDATE""".stripMargin.replaceAll("\\s+", " ") + + val result = client.insertByQuery("products", sql).futureValue + result shouldBe ElasticSuccess(DmlResult(inserted = 1L)) + } + + it should "insert orders from a SELECT with alias mapping" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |AS SELECT + | id AS order_id, + | cust AS customer_id, + | date AS order_date, + | amount AS total + |FROM staging_orders""".stripMargin.replaceAll("\\s+", " ") + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe ElasticSuccess(DmlResult(inserted = 3L)) + } + + it should "upsert orders with composite PK using ON CONFLICT DO UPDATE" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |AS SELECT + | id AS order_id, + | cust AS customer_id, + | date AS order_date, + | amount AS total + |FROM staging_orders_updates + |ON CONFLICT (order_id, customer_id) DO UPDATE""".stripMargin + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe ElasticSuccess(DmlResult(inserted = 2L)) + } + + it should "fail when conflictTarget does not match PK" in { + val sql = + """INSERT INTO orders (order_id, customer_id, order_date, total) + |VALUES ('O1001', 'C001', '2024-01-10', 1299.99) + |ON CONFLICT (order_id) DO UPDATE""".stripMargin + + val result = client.insertByQuery("orders", sql).futureValue + result shouldBe a[ElasticFailure] + } + + it should "fail when SELECT does not provide all INSERT columns" in { + val sql = + """INSERT INTO customers (customer_id, name, email) + |AS SELECT id AS customer_id, fullname AS name + |FROM staging_customers""".stripMargin + + val result = client.insertByQuery("customers", sql).futureValue + result shouldBe a[ElasticFailure] + } + + it should "fail when ON CONFLICT DO UPDATE is used without PK or conflictTarget" in { + val sql = + """INSERT INTO staging_orders (id, cust, date, amount) + |VALUES ('X', 'Y', '2024-01-01', 10.0) + |ON CONFLICT DO UPDATE""".stripMargin + + val result = client.insertByQuery("staging_orders", sql).futureValue + result shouldBe a[ElasticFailure] + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala index a86957e2..023bb907 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -22,7 +22,8 @@ import akka.stream.scaladsl.{Flow, Source} import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.ElasticResult import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.query.{SQLAggregation, SQLSearchRequest} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} +import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.serialization._ import org.json4s.Formats import org.slf4j.{Logger, LoggerFactory} @@ -32,7 +33,7 @@ import scala.language.implicitConversions /** Created by smanciot on 12/04/2020. */ -trait MockElasticClientApi extends ElasticClientApi { +trait MockElasticClientApi extends NopeClientApi { def elasticVersion: String @@ -94,7 +95,9 @@ trait MockElasticClientApi extends ElasticClientApi { override private[client] def executeCreateIndex( index: String, - settings: String + settings: String, + mappings: Option[String], + aliases: Seq[TableAlias] ): ElasticResult[Boolean] = ElasticResult.success(true) @@ -110,7 +113,8 @@ trait MockElasticClientApi extends ElasticClientApi { override private[client] def executeReindex( sourceIndex: String, targetIndex: String, - refresh: Boolean + refresh: Boolean, + pipeline: Option[String] ): ElasticResult[(Boolean, Option[Long])] = ElasticResult.success((true, Some(elasticDocuments.getAll.keys.size))) @@ -120,8 +124,7 @@ trait MockElasticClientApi extends ElasticClientApi { // ==================== AliasApi ==================== override private[client] def executeAddAlias( - index: String, - alias: String + alias: TableAlias ): ElasticResult[Boolean] = ElasticResult.success(true) @@ -288,8 +291,8 @@ trait MockElasticClientApi extends ElasticClientApi { // ==================== SearchApi ==================== override private[client] implicit def sqlSearchRequestToJsonQuery( - sqlSearch: SQLSearchRequest - ): String = + sqlSearch: SingleSearch + )(implicit timestamp: Long): String = """{ | "query": { | "match_all": {} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala new file mode 100644 index 00000000..5753e451 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/PipelineApiSpec.scala @@ -0,0 +1,654 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.slf4j.{Logger, LoggerFactory} + +import java.util.concurrent.TimeUnit +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContextExecutor} + +trait PipelineApiSpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: PipelineApi + + override def beforeAll(): Unit = { + self.beforeAll() + log.info("πŸš€ Starting PipelineApiSpec test suite") + } + + override def afterAll(): Unit = { + log.info("🏁 Finishing PipelineApiSpec test suite") + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + self.afterAll() + } + + override def beforeEach(): Unit = { + super.beforeEach() + // Cleanup: delete test pipelines before each test + cleanupTestPipelines() + } + + override def afterEach(): Unit = { + // Cleanup: delete test pipelines after each test + cleanupTestPipelines() + super.afterEach() + } + + private def cleanupTestPipelines(): Unit = { + val testPipelines = List( + "test_pipeline", + "user_pipeline", + "simple_pipeline", + "complex_pipeline", + "pipeline_to_update", + "pipeline_to_delete", + "pipeline_with_script", + "invalid_pipeline" + ) + + testPipelines.foreach { pipelineName => + client.deletePipeline(pipelineName, ifExists = true) match { + case _ => // Ignore result, just cleanup + } + } + } + + // ======================================================================== + // TEST: CREATE PIPELINE + // ======================================================================== + + "PipelineApi" should "create a simple pipeline with SET processor" in { + val pipelineDefinition = + """ + |{ + | "description": "Simple test pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "active" + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("simple_pipeline", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify pipeline was created + val getResult = client.getPipeline("simple_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + } + + it should "create a complex pipeline with multiple processors" in { + val pipelineDefinition = + """ + |{ + | "description": "Complex test pipeline", + | "processors": [ + | { + | "set": { + | "field": "name", + | "value": "anonymous", + | "if": "ctx.name == null" + | } + | }, + | { + | "lowercase": { + | "field": "status" + | } + | }, + | { + | "remove": { + | "field": "temp_field", + | "ignore_failure": true + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("complex_pipeline", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "create a pipeline with script processor" in { + val pipelineDefinition = + """ + |{ + | "description": "Pipeline with script", + | "processors": [ + | { + | "script": { + | "lang": "painless", + | "source": "ctx.age = ChronoUnit.YEARS.between(ZonedDateTime.parse(ctx.birthdate), ZonedDateTime.ofInstant(Instant.ofEpochMilli(ctx['_ingest']['timestamp']), ZoneId.of('Z')))" + | } + | } + | ] + |} + |""".stripMargin + + val result = client.createPipeline("pipeline_with_script", pipelineDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "fail to create pipeline with invalid JSON" in { + val invalidPipelineDefinition = """{ "invalid": json }""" + + val result = client.createPipeline("invalid_pipeline", invalidPipelineDefinition) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "fail to create pipeline with empty name" in { + val pipelineDefinition = + """ + |{ + | "description": "Test", + | "processors": [] + |} + |""".stripMargin + + val result = client.createPipeline("", pipelineDefinition) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "fail to create pipeline with invalid name characters" in { + val pipelineDefinition = + """ + |{ + | "description": "Test", + | "processors": [] + |} + |""".stripMargin + + val result = client.createPipeline("invalid/pipeline*name", pipelineDefinition) + + result shouldBe a[ElasticFailure] + } + + // ======================================================================== + // TEST: UPDATE PIPELINE + // ======================================================================== + + it should "update an existing pipeline" in { + // First create a pipeline + val initialDefinition = + """ + |{ + | "description": "Initial pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "pending" + | } + | } + | ] + |} + |""".stripMargin + + client.createPipeline("pipeline_to_update", initialDefinition) + + // Now update it + val updatedDefinition = + """ + |{ + | "description": "Updated pipeline", + | "processors": [ + | { + | "set": { + | "field": "status", + | "value": "active" + | } + | }, + | { + | "set": { + | "field": "updated", + | "value": true + | } + | } + | ] + |} + |""".stripMargin + + val result = client.updatePipeline("pipeline_to_update", updatedDefinition) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify the update + val getResult = client.getPipeline("pipeline_to_update") + getResult shouldBe a[ElasticSuccess[_]] + val pipelineJson = getResult.toOption.get.get + pipelineJson should include("Updated pipeline") + } + + // ======================================================================== + // TEST: GET PIPELINE + // ======================================================================== + + it should "retrieve an existing pipeline" in { + val pipelineDefinition = + """ + |{ + | "description": "Test retrieval", + | "processors": [ + | { + | "set": { + | "field": "test", + | "value": "value" + | } + | } + | ] + |} + |""".stripMargin + + client.createPipeline("test_pipeline", pipelineDefinition) + + val result = client.getPipeline("test_pipeline") + + result shouldBe a[ElasticSuccess[_]] + val maybePipeline = result.toOption.get + maybePipeline shouldBe defined + maybePipeline.get should include("Test retrieval") + } + + it should "return None when getting a non-existent pipeline" in { + val result = client.getPipeline("non_existent_pipeline") + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe None + } + + // ======================================================================== + // TEST: DELETE PIPELINE + // ======================================================================== + + it should "delete an existing pipeline" in { + val pipelineDefinition = + """ + |{ + | "description": "Pipeline to delete", + | "processors": [] + |} + |""".stripMargin + + client.createPipeline("pipeline_to_delete", pipelineDefinition) + + val deleteResult = client.deletePipeline("pipeline_to_delete", ifExists = false) + + deleteResult shouldBe a[ElasticSuccess[_]] + deleteResult.toOption.get shouldBe true + + // Verify deletion + val getResult = client.getPipeline("pipeline_to_delete") + getResult.toOption.get shouldBe None + } + + it should "handle deletion of non-existent pipeline gracefully" in { + val result = client.deletePipeline("non_existent_pipeline", ifExists = true) + + // Should succeed but return false or handle gracefully + result shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - CREATE PIPELINE + // ======================================================================== + + it should "execute CREATE PIPELINE DDL statement" in { + val sql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET ( + | field = "name", + | if = "ctx.name == null", + | description = "DEFAULT 'anonymous'", + | ignore_failure = true, + | value = "anonymous" + | ) + |) + |""".stripMargin + + val result = client.pipeline(sql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify pipeline was created + val getResult = client.getPipeline("user_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + } + + it should "execute CREATE OR REPLACE PIPELINE DDL statement" in { + // First create + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "status", value = "pending") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then replace + val replaceSql = + """ + |CREATE OR REPLACE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "status", value = "active") + |) + |""".stripMargin + + val result = client.pipeline(replaceSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "execute CREATE PIPELINE IF NOT EXISTS DDL statement" in { + val sql = + """ + |CREATE PIPELINE IF NOT EXISTS user_pipeline WITH PROCESSORS ( + | SET (field = "test", value = "value") + |) + |""".stripMargin + + // First execution should create + val result1 = client.pipeline(sql) + result1 shouldBe a[ElasticSuccess[_]] + + // Second execution should not fail + val result2 = client.pipeline(sql) + result2 shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - DROP PIPELINE + // ======================================================================== + + it should "execute DROP PIPELINE DDL statement" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE test_pipeline WITH PROCESSORS ( + | SET (field = "test", value = "value") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then drop it + val dropSql = "DROP PIPELINE test_pipeline" + + val result = client.pipeline(dropSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify deletion + val getResult = client.getPipeline("test_pipeline") + getResult.toOption.get shouldBe None + } + + it should "execute DROP PIPELINE IF EXISTS DDL statement" in { + val sql = "DROP PIPELINE IF EXISTS non_existent_pipeline" + + val result = client.pipeline(sql) + + // Should not fail even if pipeline doesn't exist + result shouldBe a[ElasticSuccess[_]] + } + + // ======================================================================== + // TEST: DDL - ALTER PIPELINE + // ======================================================================== + + it should "execute ALTER PIPELINE with ADD PROCESSOR" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then alter it + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | ADD PROCESSOR SET ( + | field = "status", + | value = "active" + | ) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + + // Verify the pipeline was updated + val getResult = client.getPipeline("user_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + val pipelineJson = getResult.toOption.get.get + pipelineJson should include("status") + } + + it should "execute ALTER PIPELINE with DROP PROCESSOR" in { + // First create a pipeline with multiple processors + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous"), + | SET (field = "status", value = "active") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then drop one processor + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | DROP PROCESSOR SET (status) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "execute ALTER PIPELINE with multiple actions" in { + // First create a pipeline + val createSql = + """ + |CREATE PIPELINE user_pipeline WITH PROCESSORS ( + | SET (field = "name", value = "anonymous") + |) + |""".stripMargin + + client.pipeline(createSql) + + // Then alter with multiple actions + val alterSql = + """ + |ALTER PIPELINE user_pipeline ( + | ADD PROCESSOR SET (field = "status", value = "active"), + | ADD PROCESSOR REMOVE (field = "temp_field", ignore_failure = true) + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticSuccess[_]] + result.toOption.get shouldBe true + } + + it should "fail to ALTER non-existent pipeline" in { + val alterSql = + """ + |ALTER PIPELINE non_existent_pipeline ( + | ADD PROCESSOR SET (field = "test", value = "value") + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(404) + error.message should include("not found") + case _ => fail("Expected an error for empty pipeline name") + } + } + + it should "execute ALTER PIPELINE IF EXISTS without error" in { + val alterSql = + """ + |ALTER PIPELINE IF EXISTS non_existent_pipeline ( + | ADD PROCESSOR SET (field = "test", value = "value") + |) + |""".stripMargin + + val result = client.pipeline(alterSql) + + // Should handle gracefully with IF EXISTS + result match { + case ElasticSuccess(_) => succeed + case ElasticFailure(error) => + error.statusCode shouldBe Some(404) + } + } + + // ======================================================================== + // TEST: ERROR HANDLING + // ======================================================================== + + it should "fail with invalid SQL syntax" in { + val invalidSql = "CREATE PIPELINE INVALID SYNTAX" + + val result = client.pipeline(invalidSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for invalid pipeline ddl statement") + } + } + + it should "fail with unsupported DDL statement" in { + val unsupportedSql = "SELECT * FROM pipelines" + + val result = client.pipeline(unsupportedSql) + + result shouldBe a[ElasticFailure] + result.error match { + case Some(error) => + error.statusCode shouldBe Some(400) + case _ => fail("Expected an error for unsupported pipeline ddl statement") + } + } + + // ======================================================================== + // TEST: INTEGRATION WITH INDEX + // ======================================================================== + + it should "create pipeline and use it with index" in { + // Create an index with pipeline + createIndex("test_index_with_pipeline") + + // Create a pipeline + val pipelineDefinition = + """ + |{ + | "description": "Pipeline for test index", + | "processors": [ + | { + | "set": { + | "field": "indexed_at", + | "value": "{{_ingest.timestamp}}" + | } + | } + | ] + |} + |""".stripMargin + + val createResult = client.createPipeline("test_index_pipeline", pipelineDefinition) + createResult shouldBe a[ElasticSuccess[_]] + + // Verify pipeline exists + val getResult = client.getPipeline("test_index_pipeline") + getResult shouldBe a[ElasticSuccess[_]] + getResult.toOption.get shouldBe defined + + // Cleanup + deleteIndex("test_index_with_pipeline") + client.deletePipeline("test_index_pipeline", ifExists = false) + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala new file mode 100644 index 00000000..40d18793 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/TemplateApiSpec.scala @@ -0,0 +1,712 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.scalatest.ElasticTestKit +import app.softnetwork.elastic.sql.serialization.JacksonConfig +import app.softnetwork.persistence.generateUUID +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Seconds, Span} +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.ExecutionContextExecutor + +trait TemplateApiSpec + extends AnyFlatSpecLike + with Matchers + with ScalaFutures + with BeforeAndAfterAll + with BeforeAndAfterEach { + self: ElasticTestKit => + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val patience: PatienceConfig = PatienceConfig(timeout = Span(10, Seconds)) + + def client: TemplateApi with VersionApi + + // ==================== HELPER METHODS ==================== + + def supportComposableTemplates: Boolean = { + client.version match { + case ElasticSuccess(v) => ElasticsearchVersion.supportsComposableTemplates(v) + case ElasticFailure(error) => + log.error(s"❌ Failed to retrieve Elasticsearch version: ${error.message}") + false + } + } + + def esVersion: String = { + client.version match { + case ElasticSuccess(v) => v + case ElasticFailure(_) => "unknown" + } + } + + /** Sanitize pattern to create valid alias name + * + * Removes invalid characters: [' ','"','*',',','/','<','>','?','\','|'] + * + * @param pattern + * the index pattern (e.g., "test-*") + * @return + * valid alias name (e.g., "test-alias") + */ + private def sanitizeAliasName(pattern: String): String = { + pattern + .replaceAll("[\\s\"*,/<>?\\\\|]", "") // Remove invalid characters + .replaceAll("-+", "-") // Replace multiple dashes with single dash + .replaceAll("^-|-$", "") // Remove leading/trailing dashes + .toLowerCase + } + + /** Generate composable template JSON + * + * @param pattern + * the index pattern (e.g., "test-*") + * @param priority + * the template priority (default: 100) + * @param withAliases + * whether to include aliases (default: true) + * @return + * JSON string for composable template + */ + def composableTemplate( + pattern: String, + priority: Int = 100, + withAliases: Boolean = true + ): String = { + val aliasName = sanitizeAliasName(pattern) + val aliasesJson = if (withAliases) { + s""" + | "aliases": { + | "${aliasName}-alias": {} + | } + |""".stripMargin + } else { + "" + } + + s""" + |{ + | "index_patterns": ["$pattern"], + | "priority": $priority, + | "template": { + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 0 + | }, + | "mappings": { + | "properties": { + | "timestamp": { "type": "date" }, + | "message": { "type": "text" } + | } + | }${if (withAliases) s",\n\t$aliasesJson" else ""} + | }, + | "version": 1, + | "_meta": { + | "description": "Test template" + | } + |} + |""".stripMargin + } + + /** Generate legacy template JSON + * + * @param pattern + * the index pattern (e.g., "test-*") + * @param order + * the template order (default: 1) + * @param withAliases + * whether to include aliases (default: true) + * @return + * JSON string for legacy template + */ + def legacyTemplate( + pattern: String, + order: Int = 1, + withAliases: Boolean = true, + version: Int = 1 + ): String = { + val aliasName = sanitizeAliasName(pattern) + val aliasesJson = if (withAliases) { + s""" + | "aliases": { + | "${aliasName}-alias": {} + | }, + |""".stripMargin + } else { + "" + } + + s""" + |{ + | "index_patterns": ["$pattern"], + | "order": $order, + | "settings": { + | "number_of_shards": 1, + | "number_of_replicas": 0 + | }, + | "mappings": { + | "_doc": { + | "properties": { + | "timestamp": { "type": "date" }, + | "message": { "type": "text" } + | } + | } + | }${if (withAliases) s",\n\t$aliasesJson" else ""} + | "version": $version + |} + |""".stripMargin + } + + def cleanupTemplate(name: String): Unit = { + client.deleteTemplate(name, ifExists = true) + } + + override def afterEach(): Unit = { + super.afterEach() + // Cleanup all test templates + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.filter(_.startsWith("test-")).foreach(cleanupTemplate) + case _ => // Ignore + } + } + + override def beforeAll(): Unit = { + self.beforeAll() + info(s"Running tests against Elasticsearch $esVersion") + info(s"Composable templates supported: $supportComposableTemplates") + } + + // ==================== TEST SUITE ==================== + + behavior of "TemplateApi" + + // ========== CREATE TEMPLATE ========== + + it should "create a composable template (ES 7.8+)" in { + assume(supportComposableTemplates, "Composable templates not supported") + + val template = composableTemplate("test-composable-*") + val result = client.createTemplate("test-composable", template) + + result shouldBe a[ElasticSuccess[_]] + result.asInstanceOf[ElasticSuccess[Boolean]].value shouldBe true + } + + it should "create a legacy template (ES < 7.8)" in { + assume(!supportComposableTemplates, "Legacy templates only for ES < 7.8") + + val template = legacyTemplate("test-legacy-*") + val result = client.createTemplate("test-legacy", template) + + result shouldBe a[ElasticSuccess[_]] + result.asInstanceOf[ElasticSuccess[Boolean]].value shouldBe true + } + + it should "auto-convert legacy format to composable (ES 7.8+)" in { + assume(supportComposableTemplates, "Composable templates not supported") + + val legacyFormat = legacyTemplate("test-auto-convert-*") + val result = client.createTemplate("test-auto-convert", legacyFormat) + + result shouldBe a[ElasticSuccess[_]] + + // Verify it was stored as composable + client.getTemplate("test-auto-convert") match { + case ElasticSuccess(Some(json)) => + json should include("priority") + json should include("template") + json should not include "order" + case _ => fail("Failed to get template") + } + } + + it should "auto-convert composable format to legacy (ES < 7.8)" in { + assume(!supportComposableTemplates, "Legacy templates only for ES < 7.8") + + val composableFormat = composableTemplate("test-auto-convert-legacy-*") + val result = client.createTemplate("test-auto-convert-legacy", composableFormat) + + result shouldBe a[ElasticSuccess[_]] + + // Verify it was stored as legacy + client.getTemplate("test-auto-convert-legacy") match { + case ElasticSuccess(Some(json)) => + json should include("order") + json should not include "priority" + json should not include "\"template\":" + case _ => fail("Failed to get template") + } + } + + it should "preserve order field in legacy templates" in { + assume(!supportComposableTemplates, "Test for legacy templates only") + + // Test with different orders + val orders = List(0, 1, 5, 10, 100) + + orders.foreach { order => + val templateName = s"test-order-$order" + val template = legacyTemplate(s"test-order-$order-*", order = order) + + // Check the JSON before creation + template should include(s""""order": $order""") + + // Create the template + val createResult = client.createTemplate(templateName, template) + createResult shouldBe a[ElasticSuccess[_]] + + // Retrieve and verify + client.getTemplate(templateName) match { + case ElasticSuccess(Some(json)) => + log.info(s"Template JSON for order=$order: $json") + + // Parse the JSON to check the order + val mapper = JacksonConfig.objectMapper + val root = mapper.readTree(json) + + if (root.has("order")) { + val actualOrder = root.get("order").asInt() + actualOrder shouldBe order + } else { + fail(s"Template does not contain 'order' field. JSON: $json") + } + + case other => fail(s"Failed to get template: $other") + } + + // Cleanup + client.deleteTemplate(templateName, ifExists = true) + } + } + + it should "update an existing template" in { + val template1 = if (supportComposableTemplates) { + composableTemplate("test-update-*", priority = 100) + } else { + legacyTemplate("test-update-*", order = 1) + } + + val template2 = if (supportComposableTemplates) { + composableTemplate("test-update-*", priority = 200) + } else { + legacyTemplate("test-update-*", order = 2, version = 2) + } + + // Create initial + val createResult = client.createTemplate("test-update", template1) + createResult shouldBe a[ElasticSuccess[_]] + + // Verify initial state + client.getTemplate("test-update") match { + case ElasticSuccess(Some(json)) => + info(s"Initial template: $json") + if (supportComposableTemplates) { + json should include("\"priority\":100") + } else { + json should include("\"order\":1") + json should include("\"version\":1") + } + case other => fail(s"Failed to get initial template: $other") + } + + // Update + val updateResult = client.createTemplate("test-update", template2) + updateResult shouldBe a[ElasticSuccess[_]] + + // Verify update + client.getTemplate("test-update") match { + case ElasticSuccess(Some(json)) => + info(s"Updated template: $json") + + if (supportComposableTemplates) { + json should include("\"priority\":200") + } else { + // Parse JSON to verify exact values + val mapper = JacksonConfig.objectMapper + val root = mapper.readTree(json) + + root.get("order").asInt() shouldBe 2 + root.get("version").asInt() shouldBe 2 + } + + case other => fail(s"Failed to get updated template: $other") + } + + // Cleanup + client.deleteTemplate("test-update", ifExists = true) + } + + it should "reject invalid template name" in { + val invalidNames = Seq("", " ", "UPPERCASE", "with space", "_underscore", "-dash", "with/slash") + + invalidNames.foreach { name => + val result = client.createTemplate(name, composableTemplate("test-*")) + result shouldBe a[ElasticFailure] + } + } + + it should "reject invalid JSON definition" in { + val invalidJson = "{ invalid json }" + val result = client.createTemplate("test-invalid-json", invalidJson) + result shouldBe a[ElasticFailure] + } + + it should "reject template without index_patterns" in { + val noPatterns = """{"priority": 1, "template": {}}""" + val result = client.createTemplate("test-no-patterns", noPatterns) + result shouldBe a[ElasticFailure] + } + + // ========== GET TEMPLATE ========== + + it should "get an existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-get-*") + } else { + legacyTemplate("test-get-*") + } + + client.createTemplate("test-get", template) + + val result = client.getTemplate("test-get") + result match { + case ElasticSuccess(Some(json)) => + json should include("test-get-*") + json should include("number_of_shards") + case _ => fail("Failed to get template") + } + } + + it should "return None for non-existing template" in { + val result = client.getTemplate("non-existing-template") + result shouldBe ElasticSuccess(None) + } + + it should "reject invalid template name in get" in { + val result = client.getTemplate("") + result shouldBe a[ElasticFailure] + } + + // ========== LIST TEMPLATES ========== + + it should "list all templates" in { + val template1 = if (supportComposableTemplates) { + composableTemplate("test-list-1-*") + } else { + legacyTemplate("test-list-1-*") + } + + val template2 = if (supportComposableTemplates) { + composableTemplate("test-list-2-*") + } else { + legacyTemplate("test-list-2-*") + } + + client.createTemplate("test-list-1", template1) + client.createTemplate("test-list-2", template2) + + val result = client.listTemplates() + result match { + case ElasticSuccess(templates) => + templates.keys should contain allOf ("test-list-1", "test-list-2") + templates.values.foreach { json => + json should not be empty + } + case _ => fail("Failed to list templates") + } + } + + it should "return empty map when no templates exist" in { + // Cleanup all test templates + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.filter(_.startsWith("test-")).foreach(cleanupTemplate) + case _ => // Ignore + } + + val result = client.listTemplates() + result match { + case ElasticSuccess(templates) => + templates.keys.filter(_.startsWith("test-")) shouldBe empty + case _ => fail("Failed to list templates") + } + } + + // ========== TEMPLATE EXISTS ========== + + it should "return true for existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-exists-*") + } else { + legacyTemplate("test-exists-*") + } + + client.createTemplate("test-exists", template) + + val result = client.templateExists("test-exists") + result shouldBe ElasticSuccess(true) + } + + it should "return false for non-existing template" in { + val result = client.templateExists("non-existing-template") + result shouldBe ElasticSuccess(false) + } + + it should "reject invalid template name in exists" in { + val result = client.templateExists("") + result shouldBe a[ElasticFailure] + } + + // ========== DELETE TEMPLATE ========== + + it should "delete an existing template" in { + val template = if (supportComposableTemplates) { + composableTemplate("test-delete-*") + } else { + legacyTemplate("test-delete-*") + } + + client.createTemplate("test-delete", template) + client.templateExists("test-delete") shouldBe ElasticSuccess(true) + + val deleteResult = client.deleteTemplate("test-delete") + deleteResult shouldBe ElasticSuccess(true) + + client.templateExists("test-delete") shouldBe ElasticSuccess(false) + } + + it should "succeed when deleting non-existing template with ifExists=true" in { + val result = client.deleteTemplate("non-existing-template", ifExists = true) + result shouldBe ElasticSuccess(false) + } + + it should "fail when deleting non-existing template with ifExists=false" in { + val result = client.deleteTemplate("non-existing-template", ifExists = false) + // Comportement dΓ©pend de l'implΓ©mentation, mais devrait Γ©chouer ou retourner false + result match { + case ElasticSuccess(false) => succeed + case ElasticFailure(_) => succeed + case _ => fail("Should fail or return false") + } + } + + it should "reject invalid template name in delete" in { + val result = client.deleteTemplate("") + result shouldBe a[ElasticFailure] + } + + // ========== EDGE CASES ========== + + it should "handle template with complex mappings" in { + val complexTemplate = + s""" + |{ + | "index_patterns": ["test-complex-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 2, + | "number_of_replicas": 1, + | "refresh_interval": "30s" + | }, + | "mappings": { + | "properties": { + | "timestamp": { "type": "date", "format": "strict_date_optional_time||epoch_millis" }, + | "message": { "type": "text", "analyzer": "standard" }, + | "level": { "type": "keyword" }, + | "tags": { "type": "keyword" }, + | "metadata": { + | "type": "object", + | "properties": { + | "host": { "type": "keyword" }, + | "service": { "type": "keyword" } + | } + | } + | } + | }, + | "aliases": { + | "logs-all": {}, + | "logs-recent": { + | "filter": { + | "range": { + | "timestamp": { + | "gte": "now-7d" + | } + | } + | } + | } + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-complex", complexTemplate) + result shouldBe a[ElasticSuccess[_]] + + // Verify + client.getTemplate("test-complex") match { + case ElasticSuccess(Some(json)) => + json should include("test-complex-*") + json should include("metadata") + json should include("logs-all") + case _ => fail("Failed to get complex template") + } + } + + it should "handle template with multiple index patterns" in { + val multiPatternTemplate = + s""" + |{ + | "index_patterns": ["test-multi-1-*", "test-multi-2-*", "test-multi-3-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 1 + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-multi-pattern", multiPatternTemplate) + result shouldBe a[ElasticSuccess[_]] + + client.getTemplate("test-multi-pattern") match { + case ElasticSuccess(Some(json)) => + json should include("test-multi-1-*") + json should include("test-multi-2-*") + json should include("test-multi-3-*") + case _ => fail("Failed to get multi-pattern template") + } + } + + it should "handle template with version and metadata" in { + val versionedTemplate = + s""" + |{ + | "index_patterns": ["test-versioned-*"], + | ${if (supportComposableTemplates) "\"priority\": 100," else "\"order\": 1,"} + | "version": 42, + | ${if (supportComposableTemplates) "\"_meta\": {" else ""} + | ${if (supportComposableTemplates) "\"description\": \"Test versioned template\"," + else ""} + | ${if (supportComposableTemplates) "\"author\": \"test-suite\"" else ""} + | ${if (supportComposableTemplates) "}," else ""} + | ${if (supportComposableTemplates) "\"template\": {" else ""} + | "settings": { + | "number_of_shards": 1 + | } + | ${if (supportComposableTemplates) "}" else ""} + |} + |""".stripMargin + + val result = client.createTemplate("test-versioned", versionedTemplate) + result shouldBe a[ElasticSuccess[_]] + + client.getTemplate("test-versioned") match { + case ElasticSuccess(Some(json)) => + json should include("\"version\":42") + if (supportComposableTemplates) { + json should include("_meta") + } + case _ => fail("Failed to get versioned template") + } + } + + // ========== CONCURRENT OPERATIONS ========== + + it should "handle concurrent template operations" in { + import scala.concurrent.Future + + val futures = (1 to 5).map { i => + Future { + val template = if (supportComposableTemplates) { + composableTemplate(s"test-concurrent-$i-*") + } else { + legacyTemplate(s"test-concurrent-$i-*") + } + client.createTemplate(s"test-concurrent-$i", template) + } + } + + val results = Future.sequence(futures).futureValue + results.foreach { result => + result shouldBe a[ElasticSuccess[_]] + } + + // Verify all were created + val listResult = client.listTemplates() + listResult match { + case ElasticSuccess(templates) => + (1 to 5).foreach { i => + templates.keys should contain(s"test-concurrent-$i") + } + case _ => fail("Failed to list templates") + } + + // Cleanup + (1 to 5).foreach { i => + client.deleteTemplate(s"test-concurrent-$i", ifExists = true) + } + } + + // ========== PERFORMANCE ========== + + it should "handle bulk template operations efficiently" in { + val startTime = System.currentTimeMillis() + + // Create 20 templates + (1 to 20).foreach { i => + val template = if (supportComposableTemplates) { + composableTemplate(s"test-bulk-$i-*") + } else { + legacyTemplate(s"test-bulk-$i-*") + } + client.createTemplate(s"test-bulk-$i", template) + } + + // List all + client.listTemplates() match { + case ElasticSuccess(templates) => + templates.keys.count(_.startsWith("test-bulk-")) should be >= 20 + case _ => fail("Failed to list templates") + } + + // Delete all + (1 to 20).foreach { i => + client.deleteTemplate(s"test-bulk-$i", ifExists = true) + } + + val duration = System.currentTimeMillis() - startTime + log.info(s"Bulk operations completed in ${duration}ms") + + duration should be < 30000L // Should complete in less than 30 seconds + } + +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala index 83952c62..16f1f1ad 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/WindowFunctionSpec.scala @@ -79,7 +79,7 @@ trait WindowFunctionSpec | } |}""".stripMargin - client.createIndex("emp").get shouldBe true + client.createIndex("emp", mappings = None, aliases = Nil).get shouldBe true client.setMapping("emp", mapping).get shouldBe true