diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 05121e16e3896..3833f497536c3 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -46,6 +46,7 @@ BWC_VERSION: - "2.19.1" - "2.19.2" - "2.19.3" + - "2.19.4" - "3.0.0" - "3.1.0" - "3.2.0" diff --git a/.gitattributes b/.gitattributes index 47b4a52e5726e..c19ea2202f725 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,5 +10,6 @@ *.bcfks binary *.crt binary *.p12 binary +*.ttf binary *.txt text=auto CHANGELOG.md merge=union diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index 22ce83c69a5d8..7923bac599888 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/') + if: github.repository == 'opensearch-project/OpenSearch' && (startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/')) steps: - name: Delete merged branch uses: actions/github-script@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d610041ae02a..8cc2e517d844c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Expand fetch phase profiling to support inner hits and top hits aggregation phases ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936)) - Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920)) +- Implement Query Rewriting Infrastructure ([#19060](https://github.com/opensearch-project/OpenSearch/pull/19060)) - The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065)) - Add a toBuilder method in EngineConfig to support easy modification of configs([#19054](https://github.com/opensearch-project/OpenSearch/pull/19054)) - Add StoreFactory plugin interface for custom Store implementations([#19091](https://github.com/opensearch-project/OpenSearch/pull/19091)) +- Use S3CrtClient for higher throughput while uploading files to S3 ([#18800](https://github.com/opensearch-project/OpenSearch/pull/18800)) - Add a dynamic setting to change skip_cache_factor and min_frequency for querycache ([#18351](https://github.com/opensearch-project/OpenSearch/issues/18351)) +- Add overload constructor for Translog to accept Channel Factory as a parameter ([#18918](https://github.com/opensearch-project/OpenSearch/pull/18918)) ### Changed - Add CompletionStage variants to methods in the Client Interface and default to ActionListener impl ([#18998](https://github.com/opensearch-project/OpenSearch/pull/18998)) - IllegalArgumentException when scroll ID references a node not found in Cluster ([#19031](https://github.com/opensearch-project/OpenSearch/pull/19031)) - Adding ScriptedAvg class to painless spi to allowlist usage from plugins ([#19006](https://github.com/opensearch-project/OpenSearch/pull/19006)) +- Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154)) +- Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161)) +- Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163)) ### Fixed - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261)) @@ -25,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix flaky tests in CloseIndexIT by addressing cluster state synchronization issues ([#18878](https://github.com/opensearch-project/OpenSearch/issues/18878)) - [Tiered Caching] Handle query execution exception ([#19000](https://github.com/opensearch-project/OpenSearch/issues/19000)) - Grant access to testclusters dir for tests ([#19085](https://github.com/opensearch-project/OpenSearch/issues/19085)) +- Fix assertion error when collapsing search results with concurrent segment search enabled ([#19053](https://github.com/opensearch-project/OpenSearch/pull/19053)) - Fix skip_unavailable setting changing to default during node drop issue ([#18766](https://github.com/opensearch-project/OpenSearch/pull/18766)) ### Dependencies @@ -33,12 +40,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `commons-cli:commons-cli` from 1.9.0 to 1.10.0 ([#19021](https://github.com/opensearch-project/OpenSearch/pull/19021)) - Bump `org.jline:jline` from 3.30.4 to 3.30.5 ([#19013](https://github.com/opensearch-project/OpenSearch/pull/19013)) - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.3 to 4.9.4 ([#19015](https://github.com/opensearch-project/OpenSearch/pull/19015)) -- Bump `com.azure:azure-storage-common` from 12.29.1 to 12.30.1 ([#19016](https://github.com/opensearch-project/OpenSearch/pull/19016)) +- Bump `com.azure:azure-storage-common` from 12.29.1 to 12.30.2 ([#19016](https://github.com/opensearch-project/OpenSearch/pull/19016), [#19145](https://github.com/opensearch-project/OpenSearch/pull/19145)) - Update OpenTelemetry to 1.53.0 and OpenTelemetry SemConv to 1.34.0 ([#19068](https://github.com/opensearch-project/OpenSearch/pull/19068)) - Bump `1password/load-secrets-action` from 2 to 3 ([#19100](https://github.com/opensearch-project/OpenSearch/pull/19100)) - Bump `com.nimbusds:nimbus-jose-jwt` from 10.3 to 10.4.2 ([#19099](https://github.com/opensearch-project/OpenSearch/pull/19099), [#19101](https://github.com/opensearch-project/OpenSearch/pull/19101)) - Bump netty from 4.1.121.Final to 4.1.124.Final ([#19103](https://github.com/opensearch-project/OpenSearch/pull/19103)) -- Bump google cloud storage from 1.113.1 to 2.55.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547)) +- Bump Google Cloud Storage SDK from 1.113.1 to 2.55.0 ([#18922](https://github.com/opensearch-project/OpenSearch/pull/18922)) +- Bump `com.google.auth:google-auth-library-oauth2-http` from 1.37.1 to 1.38.0 ([#19144](https://github.com/opensearch-project/OpenSearch/pull/19144)) +- Bump `com.squareup.okio:okio` from 3.15.0 to 3.16.0 ([#19146](https://github.com/opensearch-project/OpenSearch/pull/19146)) +- Bump Slf4j from 1.7.36 to 2.0.17 ([#19136](https://github.com/opensearch-project/OpenSearch/pull/19136)) +- Bump `org.apache.tika` from 2.9.2 to 3.2.2 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125)) +- Bump `org.apache.commons:commons-compress` from 1.26.1 to 1.28.0 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125)) ### Deprecated @@ -47,4 +59,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security -[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.1...main +[Unreleased 3.x]: https://github.com/opensearch-project/OpenSearch/compare/3.2...main diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 1e3adf762fa48..8f97f90cf21bb 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -114,7 +114,7 @@ dependencies { api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5' api 'org.jdom:jdom2:2.0.6.1' api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}" - api 'de.thetaphi:forbiddenapis:3.8' + api 'de.thetaphi:forbiddenapis:3.9' api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12' api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}" api 'org.apache.maven:maven-model:3.9.6' diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java b/buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java index 5fd155400cec7..cde18b1138947 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java @@ -36,7 +36,7 @@ * This class models the different Docker base images that are used to build Docker distributions of OpenSearch. */ public enum DockerBase { - CENTOS("centos:8"); + ALMALINUX("almalinux:8"); private final String image; diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java index 6b89aa8b60197..c42b7ea975de5 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java @@ -40,7 +40,6 @@ import org.opensearch.gradle.ExportOpenSearchBuildResourcesTask; import org.opensearch.gradle.info.BuildParams; import org.opensearch.gradle.util.GradleUtils; -import org.gradle.api.JavaVersion; import org.gradle.api.Project; import org.gradle.api.Task; import org.gradle.api.plugins.ExtraPropertiesExtension; @@ -53,6 +52,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin { @Override @@ -89,10 +89,6 @@ public TaskProvider createTask(Project project) { t.setClasspath(project.files(sourceSet.getRuntimeClasspath()).plus(sourceSet.getCompileClasspath())); t.setTargetCompatibility(BuildParams.getRuntimeJavaVersion().getMajorVersion()); - if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) > 0) { - // TODO: forbidden apis does not yet support java 15, rethink using runtime version - t.setTargetCompatibility(JavaVersion.VERSION_14.getMajorVersion()); - } t.setBundledSignatures(new HashSet<>(Arrays.asList("jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out"))); t.setSignaturesFiles( project.files( @@ -140,6 +136,18 @@ public Void call(Object... names) { return null; } }); + // Use of the deprecated security manager APIs are pervasive so set them to warn + // globally for all projects. Replacements for (most of) these APIs are available + // so usages can move to the non-deprecated variants to avoid the warnings. + t.setSignaturesWithSeverityWarn( + Set.of( + "java.security.AccessController", + "java.security.AccessControlContext", + "java.lang.System#getSecurityManager()", + "java.lang.SecurityManager", + "java.security.Policy" + ) + ); }); TaskProvider forbiddenApis = project.getTasks().named("forbiddenApis"); forbiddenApis.configure(t -> t.setGroup("")); diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 22fb38ded3bde..ed5eedb65e140 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -105,9 +105,6 @@ testingConventions { thirdPartyAudit { ignoreMissingClasses( 'org.conscrypt.Conscrypt', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', //commons-logging optional dependencies 'org.apache.avalon.framework.logger.Logger', 'org.apache.log.Hierarchy', diff --git a/client/rest/licenses/slf4j-api-1.7.36.jar.sha1 b/client/rest/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/client/rest/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/client/rest/licenses/slf4j-api-2.0.17.jar.sha1 b/client/rest/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/client/rest/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index cc371a3275570..ecc2d2c5c5766 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -132,8 +132,8 @@ project.ext { } void addCopyDockerContextTask(Architecture architecture, DockerBase base) { - if (base != DockerBase.CENTOS) { - throw new GradleException("The only allowed docker base image for builds is CENTOS") + if (base != DockerBase.ALMALINUX) { + throw new GradleException("The only allowed docker base image for builds is ALMALINUX") } tasks.register(taskName("copy", architecture, base, "DockerContext"), Sync) { @@ -181,8 +181,8 @@ opensearch_distributions { tasks.named("preProcessFixture").configure { dependsOn opensearch_distributions.docker // always run the task, otherwise the folders won't be created - outputs.upToDateWhen { - false + outputs.upToDateWhen { + false } doLast { // tests expect to have an empty repo @@ -208,8 +208,8 @@ tasks.named("check").configure { } void addBuildDockerImage(Architecture architecture, DockerBase base) { - if (base != DockerBase.CENTOS) { - throw new GradleException("The only allowed docker base image for builds is CENTOS") + if (base != DockerBase.ALMALINUX) { + throw new GradleException("The only allowed docker base image for builds is ALMALINUX") } final TaskProvider buildDockerImageTask = @@ -232,9 +232,9 @@ void addBuildDockerImage(Architecture architecture, DockerBase base) { } for (final Architecture architecture : Architecture.values()) { - // We only create Docker images for the distribution on CentOS. + // We only create Docker images for the distribution on AlmaLinux. for (final DockerBase base : DockerBase.values()) { - if (base == DockerBase.CENTOS) { + if (base == DockerBase.ALMALINUX) { addCopyDockerContextTask(architecture, base) addBuildDockerImage(architecture, base) } @@ -257,7 +257,7 @@ subprojects { Project subProject -> apply plugin: 'distribution' final Architecture architecture = subProject.name.contains('arm64-') ? Architecture.ARM64 : Architecture.X64 - final DockerBase base = DockerBase.CENTOS + final DockerBase base = DockerBase.ALMALINUX final String arch = architecture == Architecture.ARM64 ? '-arm64' : '' final String extension = 'docker.tar' diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index a5bea2935b3ea..3426df47780dc 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -19,7 +19,7 @@ tasks.register("buildDockerBuildContext", Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "opensearch" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, DockerBase.CENTOS, false) + with dockerBuildContext(null, DockerBase.ALMALINUX, false) } tasks.named("assemble").configure { dependsOn "buildDockerBuildContext" } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index c980217b0b8dc..fc2b66aaf7d53 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -63,16 +63,12 @@ FROM ${base_image} ENV OPENSEARCH_CONTAINER true -RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \\ - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* && \\ - for iter in {1..10}; do \\ - ${package_manager} update --setopt=tsflags=nodocs -y && \\ - ${package_manager} install --setopt=tsflags=nodocs -y \\ - nc shadow-utils zip unzip && \\ - ${package_manager} clean all && exit_code=0 && break || exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && \\ - sleep 10; \\ - done; \\ - (exit \$exit_code) +RUN set -e \\ + && dnf -y update \\ + && dnf -y install --setopt=tsflags=nodocs \\ + nmap-ncat shadow-utils zip unzip \\ + && dnf clean all \\ + && rm -rf /var/cache/dnf RUN groupadd -g 1000 opensearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/opensearch opensearch && \\ diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle index 8beb17bb8bf9a..41f80eb39a81f 100644 --- a/distribution/tools/plugin-cli/build.gradle +++ b/distribution/tools/plugin-cli/build.gradle @@ -81,5 +81,10 @@ thirdPartyAudit.ignoreMissingClasses( 'org.tukaani.xz.XZOutputStream', 'org.apache.commons.codec.digest.PureJavaCrc32C', 'org.apache.commons.codec.digest.XXHash32', - 'org.apache.commons.lang3.reflect.FieldUtils' + 'org.apache.commons.lang3.reflect.FieldUtils', + 'org.apache.commons.lang3.ArrayFill', + 'org.apache.commons.lang3.ArrayUtils', + 'org.apache.commons.lang3.StringUtils', + 'org.apache.commons.lang3.SystemProperties', + 'org.apache.commons.lang3.function.Suppliers' ) diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1 deleted file mode 100644 index 912bda85de18a..0000000000000 --- a/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -44331c1130c370e726a2e1a3e6fba6d2558ef04a \ No newline at end of file diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1 new file mode 100644 index 0000000000000..5edae62aeeb5d --- /dev/null +++ b/distribution/tools/plugin-cli/licenses/commons-compress-1.28.0.jar.sha1 @@ -0,0 +1 @@ +e482f2c7a88dac3c497e96aa420b6a769f59c8d7 \ No newline at end of file diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java index ea76e051d253e..c71728056b4c4 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java @@ -399,7 +399,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()") boolean urlExists(Terminal terminal, String urlString) throws IOException { terminal.println(VERBOSE, "Checking if url exists: " + urlString); - URL url = new URL(urlString); + URL url = URI.create(urlString).toURL(); assert "https".equals(url.getProtocol()) : "Use of https protocol is required"; HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer"); @@ -427,7 +427,7 @@ private List checkMisspelledPlugin(String pluginId) { @SuppressForbidden(reason = "We use getInputStream to download plugins") Path downloadZip(Terminal terminal, String urlString, Path tmpDir, boolean isBatch) throws IOException { terminal.println(VERBOSE, "Retrieving zip from " + urlString); - URL url = new URL(urlString); + URL url = URI.create(urlString).toURL(); Path zip = Files.createTempFile(tmpDir, null, ".zip"); URLConnection urlConnection = url.openConnection(); urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer"); @@ -684,7 +684,7 @@ InputStream getPublicKey() { */ // pkg private for tests URL openUrl(String urlString) throws IOException { - URL checksumUrl = new URL(urlString); + URL checksumUrl = URI.create(urlString).toURL(); HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection(); if (connection.getResponseCode() == 404) { return null; diff --git a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java index 70cccc94a26f9..57cf65a4a2c51 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java @@ -526,7 +526,7 @@ public void testSpaceInUrl() throws Exception { Path pluginDir = createPluginDir(temp); String pluginZip = createPluginUrl("fake", pluginDir); Path pluginZipWithSpaces = createTempFile("foo bar", ".zip"); - try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip))) { + try (InputStream in = FileSystemUtils.openFileURLStream(URI.create(pluginZip).toURL())) { Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING); } installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1()); @@ -536,8 +536,8 @@ public void testSpaceInUrl() throws Exception { public void testMalformedUrlNotMaven() throws Exception { Tuple env = createEnv(fs, temp); // has two colons, so it appears similar to maven coordinates - MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1())); - assertTrue(e.getMessage(), e.getMessage().contains("no protocol")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> installPlugin("://host:1234", env.v1())); + assertThat(e.getMessage(), startsWith("Expected scheme name")); } public void testFileNotMaven() throws Exception { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 3cd058d83ba4b..7d0e2d31f0baf 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,7 +14,7 @@ snakeyaml = "2.1" icu4j = "77.1" supercsv = "2.4.0" log4j = "2.21.0" -slf4j = "1.7.36" +slf4j = "2.0.17" asm = "9.7" jettison = "1.5.4" woodstox = "6.4.0" @@ -50,7 +50,7 @@ httpasyncclient = "4.1.5" commonslogging = "1.2" commonscodec = "1.18.0" commonslang = "3.18.0" -commonscompress = "1.26.1" +commonscompress = "1.28.0" commonsio = "2.16.0" # plugin dependencies aws = "2.30.31" diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index a5b682d653295..5bb93e53ec1ee 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -117,6 +117,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_19_1 = new Version(2190199, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_2_19_2 = new Version(2190299, org.apache.lucene.util.Version.LUCENE_9_12_1); public static final Version V_2_19_3 = new Version(2190399, org.apache.lucene.util.Version.LUCENE_9_12_2); + public static final Version V_2_19_4 = new Version(2190499, org.apache.lucene.util.Version.LUCENE_9_12_2); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_10_1_0); public static final Version V_3_1_0 = new Version(3010099, org.apache.lucene.util.Version.LUCENE_10_2_1); public static final Version V_3_2_0 = new Version(3020099, org.apache.lucene.util.Version.LUCENE_10_2_2); diff --git a/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java b/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java index 8b29378dfde12..08f5f120f879d 100644 --- a/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java +++ b/libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.ByteBuffer; @@ -132,21 +133,21 @@ public void testIsHidden() { } public void testOpenFileURLStream() throws IOException { - URL urlWithWrongProtocol = new URL("http://www.google.com"); + URL urlWithWrongProtocol = URI.create("http://www.google.com").toURL(); try (InputStream is = FileSystemUtils.openFileURLStream(urlWithWrongProtocol)) { fail("Should throw IllegalArgumentException due to invalid protocol"); } catch (IllegalArgumentException e) { assertEquals("Invalid protocol [http], must be [file] or [jar]", e.getMessage()); } - URL urlWithHost = new URL("file", "localhost", txtFile.toString()); + URL urlWithHost = URI.create("file://localhost/" + txtFile.toString()).toURL(); try (InputStream is = FileSystemUtils.openFileURLStream(urlWithHost)) { fail("Should throw IllegalArgumentException due to host"); } catch (IllegalArgumentException e) { assertEquals("URL cannot have host. Found: [localhost]", e.getMessage()); } - URL urlWithPort = new URL("file", "", 80, txtFile.toString()); + URL urlWithPort = URI.create("file://:80/" + txtFile.toString()).toURL(); try (InputStream is = FileSystemUtils.openFileURLStream(urlWithPort)) { fail("Should throw IllegalArgumentException due to port"); } catch (IllegalArgumentException e) { diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java index 82f4e94e31ae6..9a723fe491394 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java @@ -77,12 +77,12 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) { private void assertHasTrustedIssuer(X509ExtendedTrustManager trustManager, String name) { final String lowerName = name.toLowerCase(Locale.ROOT); final Optional ca = Stream.of(trustManager.getAcceptedIssuers()) - .filter(cert -> cert.getSubjectDN().getName().toLowerCase(Locale.ROOT).contains(lowerName)) + .filter(cert -> cert.getSubjectX500Principal().getName().toLowerCase(Locale.ROOT).contains(lowerName)) .findAny(); if (ca.isPresent() == false) { logger.info("Failed to find issuer [{}] in trust manager, but did find ...", lowerName); for (X509Certificate cert : trustManager.getAcceptedIssuers()) { - logger.info(" - {}", cert.getSubjectDN().getName().replaceFirst("^\\w+=([^,]+),.*", "$1")); + logger.info(" - {}", cert.getSubjectX500Principal().getName().replaceFirst("^\\w+=([^,]+),.*", "$1")); } Assert.fail("Cannot find trusted issuer with name [" + name + "]."); } diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java index 70cb76ceaec51..51e69a758ad44 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java @@ -154,8 +154,8 @@ private void assertCertificateAndKey(PemKeyConfig keyConfig, String expectedDN) assertThat(chain, notNullValue()); assertThat(chain, arrayWithSize(1)); final X509Certificate certificate = chain[0]; - assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1")); - assertThat(certificate.getSubjectDN().getName(), is(expectedDN)); + assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1")); + assertThat(certificate.getSubjectX500Principal().getName(), is(expectedDN)); assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2)); assertThat( certificate.getSubjectAlternativeNames(), diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java index 773b4071313d9..05bf4dd194b00 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java @@ -146,7 +146,7 @@ private void assertCertificateChain(PemTrustConfig trustConfig, String... caName final X509ExtendedTrustManager trustManager = trustConfig.createTrustManager(); final X509Certificate[] issuers = trustManager.getAcceptedIssuers(); final Set issuerNames = Stream.of(issuers) - .map(X509Certificate::getSubjectDN) + .map(X509Certificate::getSubjectX500Principal) .map(Principal::getName) .collect(Collectors.toSet()); diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java index 1745c547d04ee..fdf98dc38bca5 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java @@ -183,8 +183,8 @@ private void assertKeysLoaded(StoreKeyConfig keyConfig, String... names) throws assertThat(chain, notNullValue()); assertThat(chain, arrayWithSize(1)); final X509Certificate certificate = chain[0]; - assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1")); - assertThat(certificate.getSubjectDN().getName(), is("CN=" + name)); + assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1")); + assertThat(certificate.getSubjectX500Principal().getName(), is("CN=" + name)); assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2)); assertThat( certificate.getSubjectAlternativeNames(), diff --git a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreTrustConfigTests.java b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreTrustConfigTests.java index 8058ffe95dc93..656d8c468be60 100644 --- a/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreTrustConfigTests.java +++ b/libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreTrustConfigTests.java @@ -140,7 +140,7 @@ private void assertCertificateChain(StoreTrustConfig trustConfig, String... caNa final X509ExtendedTrustManager trustManager = trustConfig.createTrustManager(); final X509Certificate[] issuers = trustManager.getAcceptedIssuers(); final Set issuerNames = Stream.of(issuers) - .map(X509Certificate::getSubjectDN) + .map(X509Certificate::getSubjectX500Principal) .map(Principal::getName) .collect(Collectors.toSet()); diff --git a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/DisableGraphQueryTests.java index 738c81c13cb6c..261c1bfafd2ba 100644 --- a/modules/analysis-common/src/test/java/org/opensearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/opensearch/analysis/common/DisableGraphQueryTests.java @@ -93,10 +93,9 @@ public void setup() { .put("index.analysis.analyzer.text_shingle_unigram.tokenizer", "whitespace") .put("index.analysis.analyzer.text_shingle_unigram.filter", "lowercase, shingle_unigram") .build(); - indexService = createIndex( + indexService = createIndexWithSimpleMappings( "test", settings, - "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DateProcessorTests.java index 8a4f3b4a898b4..02ac2b866ce71 100644 --- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DateProcessorTests.java @@ -46,12 +46,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.IllformedLocaleException; import java.util.List; import java.util.Locale; import java.util.Map; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; public class DateProcessorTests extends OpenSearchTestCase { @@ -315,7 +317,7 @@ public void testInvalidLocale() { () -> processor.execute(RandomDocumentPicks.randomIngestDocument(random(), document)) ); assertThat(e.getMessage(), equalTo("unable to parse date [2010]")); - assertThat(e.getCause().getMessage(), equalTo("Unknown language: invalid")); + assertThat(e.getCause(), instanceOf(IllformedLocaleException.class)); } public void testOutputFormat() { diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java index c19d4f361b2b6..c55cb4707d464 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/Compiler.java @@ -50,7 +50,7 @@ import java.lang.reflect.Method; import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.security.CodeSource; import java.security.SecureClassLoader; import java.security.cert.Certificate; @@ -77,7 +77,7 @@ final class Compiler { static { try { // Setup the code privileges. - CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null); + CODESOURCE = new CodeSource(URI.create("file:" + BootstrapInfo.UNTRUSTED_CODEBASE).toURL(), (Certificate[]) null); } catch (MalformedURLException impossible) { throw new RuntimeException(impossible); } diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java index e155a890c03d1..e2291754a26e4 100644 --- a/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/opensearch/painless/lookup/PainlessLookupBuilder.java @@ -57,7 +57,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.security.AccessController; import java.security.CodeSource; import java.security.PrivilegedAction; @@ -120,7 +120,7 @@ Class defineBridge(String name, byte[] bytes) { static { try { - CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null); + CODESOURCE = new CodeSource(URI.create("file:" + BootstrapInfo.UNTRUSTED_CODEBASE).toURL(), (Certificate[]) null); } catch (MalformedURLException mue) { throw new RuntimeException(mue); } diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/NeedsScoreTests.java index 9f87fbedb2a8f..f036968d96658 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/NeedsScoreTests.java @@ -52,7 +52,7 @@ public class NeedsScoreTests extends OpenSearchSingleNodeTestCase { public void testNeedsScores() { - IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double"); + IndexService index = createIndexWithSimpleMappings("test", Settings.EMPTY, "d", "type=double"); Map, List> contexts = new HashMap<>(); contexts.put(NumberSortScript.CONTEXT, Allowlist.BASE_ALLOWLISTS); diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java index d1ab998c314b0..ccc7fa1c99332 100644 --- a/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java +++ b/modules/lang-painless/src/test/java/org/opensearch/painless/action/PainlessExecuteApiTests.java @@ -89,7 +89,7 @@ public void testDefaults() throws IOException { public void testFilterExecutionContext() throws IOException { ScriptService scriptService = getInstanceFromNode(ScriptService.class); - IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "field", "type=long"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "field", "type=long"); Request.ContextSetup contextSetup = new Request.ContextSetup("index", new BytesArray("{\"field\": 3}"), null); contextSetup.setXContentType(MediaTypeRegistry.JSON); @@ -120,7 +120,7 @@ public void testFilterExecutionContext() throws IOException { public void testScoreExecutionContext() throws IOException { ScriptService scriptService = getInstanceFromNode(ScriptService.class); - IndexService indexService = createIndex("index", Settings.EMPTY, "doc", "rank", "type=long", "text", "type=text"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "rank", "type=long", "text", "type=text"); Request.ContextSetup contextSetup = new Request.ContextSetup( "index", diff --git a/modules/mapper-extras/src/javaRestTest/java/org/opensearch/index/mapper/ScaledFloatDerivedSourceIT.java b/modules/mapper-extras/src/javaRestTest/java/org/opensearch/index/mapper/ScaledFloatDerivedSourceIT.java new file mode 100644 index 0000000000000..234825541d26d --- /dev/null +++ b/modules/mapper-extras/src/javaRestTest/java/org/opensearch/index/mapper/ScaledFloatDerivedSourceIT.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.io.IOException; + +import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; + +public class ScaledFloatDerivedSourceIT extends OpenSearchIntegTestCase { + + private static final String INDEX_NAME = "test"; + + public void testScaledFloatDerivedSource() throws Exception { + Settings.Builder settings = Settings.builder(); + settings.put(indexSettings()); + settings.put("index.derived_source.enabled", "true"); + + prepareCreate(INDEX_NAME).setSettings(settings) + .setMapping( + jsonBuilder().startObject() + .startObject("properties") + .startObject("foo") + .field("type", "scaled_float") + .field("scaling_factor", "100") + .endObject() + .endObject() + .endObject() + ) + .get(); + + ensureGreen(INDEX_NAME); + + String docId = "one_doc"; + assertEquals(DocWriteResponse.Result.CREATED, prepareIndex(docId, 1.2123422f).get().getResult()); + + RefreshResponse refreshResponse = refresh(INDEX_NAME); + assertEquals(RestStatus.OK, refreshResponse.getStatus()); + assertEquals(0, refreshResponse.getFailedShards()); + assertEquals(INDEX_NUMBER_OF_SHARDS_SETTING.get(settings.build()).intValue(), refreshResponse.getSuccessfulShards()); + + GetResponse getResponse = client().prepareGet() + .setFetchSource(true) + .setId(docId) + .setIndex(INDEX_NAME) + .get(TimeValue.timeValueMinutes(1)); + assertTrue(getResponse.isExists()); + assertEquals(1.21d, getResponse.getSourceAsMap().get("foo")); + } + + private IndexRequestBuilder prepareIndex(String id, float number) throws IOException { + return client().prepareIndex(INDEX_NAME) + .setId(id) + .setSource(jsonBuilder().startObject().field("foo", number).endObject().toString(), XContentType.JSON); + } +} diff --git a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java index 97e80c66e3f4e..5f4925a4ae577 100644 --- a/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/opensearch/percolator/PercolatorQuerySearchTests.java @@ -281,7 +281,7 @@ public void testPercolateQueryWithNestedDocuments_doLeakFieldDataCacheEntries() public void testMapUnmappedFieldAsText() throws IOException { Settings.Builder settings = Settings.builder().put("index.percolator.map_unmapped_fields_as_text", true); - createIndex("test", settings.build(), "query", "query", "type=percolator"); + createIndexWithSimpleMappings("test", settings.build(), "query", "type=percolator"); client().prepareIndex("test") .setId("1") .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()) @@ -302,10 +302,9 @@ public void testMapUnmappedFieldAsText() throws IOException { } public void testRangeQueriesWithNow() throws Exception { - IndexService indexService = createIndex( + IndexService indexService = createIndexWithSimpleMappings( "test", Settings.builder().put("index.number_of_shards", 1).build(), - "_doc", "field1", "type=keyword", "field2", diff --git a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java index 02e858cb8d1f2..395f741c67133 100644 --- a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java +++ b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobContainer.java @@ -43,6 +43,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.NoSuchFileException; import java.security.AccessController; @@ -136,9 +137,11 @@ public DeleteResult delete() { @Override public InputStream readBlob(String name) throws IOException { try { - return new BufferedInputStream(getInputStream(new URL(path, name)), blobStore.bufferSizeInBytes()); + return new BufferedInputStream(getInputStream(this.path.toURI().resolve(name).toURL()), blobStore.bufferSizeInBytes()); } catch (FileNotFoundException fnfe) { throw new NoSuchFileException("[" + name + "] blob not found"); + } catch (URISyntaxException e) { + throw new IOException(e); } } diff --git a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java index 0fad0cbe21033..dda206ae540f5 100644 --- a/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java +++ b/modules/repository-url/src/main/java/org/opensearch/common/blobstore/url/URLBlobStore.java @@ -41,6 +41,7 @@ import org.opensearch.core.common.unit.ByteSizeValue; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; /** @@ -97,7 +98,7 @@ public int bufferSizeInBytes() { public BlobContainer blobContainer(BlobPath path) { try { return new URLBlobContainer(this, path, buildPath(path)); - } catch (MalformedURLException ex) { + } catch (MalformedURLException | URISyntaxException ex) { throw new BlobStoreException("malformed URL " + path, ex); } } @@ -113,17 +114,15 @@ public void close() { * @param path relative path * @return Base URL + path */ - private URL buildPath(BlobPath path) throws MalformedURLException { + private URL buildPath(BlobPath path) throws MalformedURLException, URISyntaxException { String[] paths = path.toArray(); if (paths.length == 0) { return path(); } - URL blobPath = new URL(this.path, paths[0] + "/"); - if (paths.length > 1) { - for (int i = 1; i < paths.length; i++) { - blobPath = new URL(blobPath, paths[i] + "/"); - } + var uri = this.path.toURI(); + for (String pathElement : paths) { + uri = uri.resolve(pathElement + "/"); } - return blobPath; + return uri.toURL(); } } diff --git a/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java b/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java index 4c8d8aab4532b..0780002f175ab 100644 --- a/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java +++ b/modules/repository-url/src/main/java/org/opensearch/repositories/url/URLRepository.java @@ -50,6 +50,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.net.MalformedURLException; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.Arrays; @@ -85,10 +86,10 @@ public class URLRepository extends BlobStoreRepository { Property.NodeScope ); - public static final Setting URL_SETTING = new Setting<>("url", "http:", URLRepository::parseURL, Property.NodeScope); + public static final Setting URL_SETTING = new Setting<>("url", "http://?", URLRepository::parseURL, Property.NodeScope); public static final Setting REPOSITORIES_URL_SETTING = new Setting<>( "repositories.url.url", - (s) -> s.get("repositories.uri.url", "http:"), + (s) -> s.get("repositories.uri.url", "http://?"), URLRepository::parseURL, Property.NodeScope ); @@ -194,7 +195,7 @@ public boolean isReadOnly() { private static URL parseURL(String s) { try { - return new URL(s); + return URI.create(s).toURL(); } catch (MalformedURLException e) { throw new IllegalArgumentException("Unable to parse URL repository setting", e); } diff --git a/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index 27cef3f7d7251..c18e84f46e471 100644 --- a/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/yamlRestTest/java/org/opensearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -55,7 +55,6 @@ import java.io.IOException; import java.net.InetAddress; import java.net.URI; -import java.net.URL; import java.util.List; import java.util.Map; @@ -120,7 +119,7 @@ public void registerRepositories() throws IOException { List allowedUrls = (List) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings); for (String allowedUrl : allowedUrls) { try { - InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost()); + InetAddress inetAddress = InetAddress.getByName(URI.create(allowedUrl).getHost()); if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) { Request createUrlRepositoryRequest = new Request("PUT", "/_snapshot/repository-url"); createUrlRepositoryRequest.setEntity(buildRepositorySettings("url", Settings.builder().put("url", allowedUrl).build())); diff --git a/plugins/arrow-flight-rpc/build.gradle b/plugins/arrow-flight-rpc/build.gradle index eb14e4ecea577..034a0043a4a61 100644 --- a/plugins/arrow-flight-rpc/build.gradle +++ b/plugins/arrow-flight-rpc/build.gradle @@ -130,10 +130,6 @@ tasks.named('thirdPartyAudit').configure { 'org.apache.commons.logging.Log', 'org.apache.commons.logging.LogFactory', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', - // from Log4j (deliberate, Netty will fallback to Log4j 2) 'org.apache.log4j.Level', 'org.apache.log4j.Logger', diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/arrow-flight-rpc/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/cache-ehcache/build.gradle b/plugins/cache-ehcache/build.gradle index 6390b045db8ea..64cf3a963db74 100644 --- a/plugins/cache-ehcache/build.gradle +++ b/plugins/cache-ehcache/build.gradle @@ -79,9 +79,6 @@ thirdPartyAudit { 'org.osgi.framework.BundleActivator', 'org.osgi.framework.BundleContext', 'org.osgi.framework.ServiceReference', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder' ) } diff --git a/plugins/cache-ehcache/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/cache-ehcache/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/cache-ehcache/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/cache-ehcache/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/cache-ehcache/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/cache-ehcache/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/crypto-kms/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/crypto-kms/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/crypto-kms/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 7a7eb8da24fb6..8aeae37742c19 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -162,9 +162,6 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.avalon.framework.logger.Logger', 'org.apache.log.Hierarchy', 'org.apache.log.Logger', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', 'software.amazon.eventstream.HeaderValue', 'software.amazon.eventstream.Message', 'software.amazon.eventstream.MessageDecoder', diff --git a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/discovery-ec2/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/discovery-ec2/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/discovery-ec2/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/examples/rest-handler/src/javaRestTest/java/org/opensearch/example/resthandler/ExampleFixtureIT.java b/plugins/examples/rest-handler/src/javaRestTest/java/org/opensearch/example/resthandler/ExampleFixtureIT.java index 0d50f9efbecd4..0ff9f78e34bef 100644 --- a/plugins/examples/rest-handler/src/javaRestTest/java/org/opensearch/example/resthandler/ExampleFixtureIT.java +++ b/plugins/examples/rest-handler/src/javaRestTest/java/org/opensearch/example/resthandler/ExampleFixtureIT.java @@ -40,6 +40,7 @@ import java.io.OutputStreamWriter; import java.net.InetAddress; import java.net.Socket; +import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -53,7 +54,7 @@ public void testExample() throws Exception { final String externalAddress = System.getProperty("external.address"); assertNotNull("External address must not be null", externalAddress); - final URL url = new URL("http://" + externalAddress); + final URL url = URI.create("http://" + externalAddress).toURL(); final InetAddress address = InetAddress.getByName(url.getHost()); try ( Socket socket = new Socket(address, url.getPort()); diff --git a/plugins/identity-shiro/build.gradle b/plugins/identity-shiro/build.gradle index f72155e1d28b2..223a69f9eb353 100644 --- a/plugins/identity-shiro/build.gradle +++ b/plugins/identity-shiro/build.gradle @@ -66,9 +66,6 @@ thirdPartyAudit.ignoreMissingClasses( 'org.apache.log4j.Logger', 'org.apache.log4j.Priority', 'org.cryptacular.bean.HashBean', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', 'org.springframework.context.MessageSource', 'org.springframework.context.support.MessageSourceAccessor' ) diff --git a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/identity-shiro/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/identity-shiro/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/identity-shiro/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/identity-shiro/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle index f6a5f104cac79..0a6306be7daac 100644 --- a/plugins/ingest-attachment/build.gradle +++ b/plugins/ingest-attachment/build.gradle @@ -38,8 +38,8 @@ opensearchplugin { } versions << [ - 'tika' : '2.9.2', - 'pdfbox': '2.0.31', + 'tika' : '3.2.2', + 'pdfbox': '3.0.5', 'poi' : '5.4.1', 'mime4j': '0.8.11' ] @@ -75,10 +75,11 @@ dependencies { // external parser libraries // HTML - api 'org.ccil.cowan.tagsoup:tagsoup:1.2.1' + api 'org.jsoup:jsoup:1.20.1' // Adobe PDF api "org.apache.pdfbox:pdfbox:${versions.pdfbox}" api "org.apache.pdfbox:fontbox:${versions.pdfbox}" + api "org.apache.pdfbox:pdfbox-io:${versions.pdfbox}" api "org.apache.pdfbox:jempbox:1.8.17" api "commons-logging:commons-logging:${versions.commonslogging}" // OpenOffice @@ -121,6 +122,7 @@ forbiddenPatterns { exclude '**/*.pdf' exclude '**/*.epub' exclude '**/*.vsdx' + exclude '**/*.ttf' } thirdPartyAudit { diff --git a/plugins/ingest-attachment/licenses/Roboto-OFL.txt b/plugins/ingest-attachment/licenses/Roboto-OFL.txt new file mode 100644 index 0000000000000..65a3057b1f24b --- /dev/null +++ b/plugins/ingest-attachment/licenses/Roboto-OFL.txt @@ -0,0 +1,93 @@ +Copyright 2011 The Roboto Project Authors (https://github.com/googlefonts/roboto-classic) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://openfontlicense.org + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1 deleted file mode 100644 index 912bda85de18a..0000000000000 --- a/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -44331c1130c370e726a2e1a3e6fba6d2558ef04a \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.28.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.28.0.jar.sha1 new file mode 100644 index 0000000000000..5edae62aeeb5d --- /dev/null +++ b/plugins/ingest-attachment/licenses/commons-compress-1.28.0.jar.sha1 @@ -0,0 +1 @@ +e482f2c7a88dac3c497e96aa420b6a769f59c8d7 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1 deleted file mode 100644 index d45d45a66e072..0000000000000 --- a/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -96999ecdb7324bf718b88724818fa62f81286c36 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/fontbox-3.0.5.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-3.0.5.jar.sha1 new file mode 100644 index 0000000000000..241eda72e6dae --- /dev/null +++ b/plugins/ingest-attachment/licenses/fontbox-3.0.5.jar.sha1 @@ -0,0 +1 @@ +b4a068e1dba2b9832a108cdf6e9a3249680e3ce8 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/jsoup-1.20.1.jar.sha1 b/plugins/ingest-attachment/licenses/jsoup-1.20.1.jar.sha1 new file mode 100644 index 0000000000000..9a2329562aae0 --- /dev/null +++ b/plugins/ingest-attachment/licenses/jsoup-1.20.1.jar.sha1 @@ -0,0 +1 @@ +769377896610be1736f8d6d51fc52a6042d1ce82 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/jsoup-LICENSE.txt b/plugins/ingest-attachment/licenses/jsoup-LICENSE.txt new file mode 100644 index 0000000000000..e4bf2be9fb7f2 --- /dev/null +++ b/plugins/ingest-attachment/licenses/jsoup-LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2009-2025 Jonathan Hedley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/plugins/ingest-attachment/licenses/tagsoup-NOTICE.txt b/plugins/ingest-attachment/licenses/jsoup-NOTICE.txt similarity index 100% rename from plugins/ingest-attachment/licenses/tagsoup-NOTICE.txt rename to plugins/ingest-attachment/licenses/jsoup-NOTICE.txt diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1 deleted file mode 100644 index fa256ed9a65d2..0000000000000 --- a/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -29b25053099bc30784a766ccb821417e06f4b8a1 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/pdfbox-3.0.5.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-3.0.5.jar.sha1 new file mode 100644 index 0000000000000..6a6fad5245aa2 --- /dev/null +++ b/plugins/ingest-attachment/licenses/pdfbox-3.0.5.jar.sha1 @@ -0,0 +1 @@ +c34109061c3a0d85d871d9edc469ac0682f81856 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/pdfbox-io-3.0.5.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-io-3.0.5.jar.sha1 new file mode 100644 index 0000000000000..e70c851dbd9c2 --- /dev/null +++ b/plugins/ingest-attachment/licenses/pdfbox-io-3.0.5.jar.sha1 @@ -0,0 +1 @@ +402151a8d1aa427ea879cc7160e9227e9f5088ba \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/pdfbox-io-LICENSE.txt b/plugins/ingest-attachment/licenses/pdfbox-io-LICENSE.txt new file mode 100644 index 0000000000000..97553f24a432a --- /dev/null +++ b/plugins/ingest-attachment/licenses/pdfbox-io-LICENSE.txt @@ -0,0 +1,344 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +EXTERNAL COMPONENTS + +Apache PDFBox includes a number of components with separate copyright notices +and license terms. Your use of these components is subject to the terms and +conditions of the following licenses. + +Contributions made to the original PDFBox and FontBox projects: + + Copyright (c) 2002-2007, www.pdfbox.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of pdfbox; nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +Adobe Font Metrics (AFM) for PDF Core 14 Fonts + + This file and the 14 PostScript(R) AFM files it accompanies may be used, + copied, and distributed for any purpose and without charge, with or without + modification, provided that all copyright notices are retained; that the + AFM files are not distributed without this file; that all modifications + to this file or any of the AFM files are prominently noted in the modified + file(s); and that this paragraph is not modified. Adobe Systems has no + responsibility or obligation to support the use of the AFM files. + +CMaps for PDF Fonts (http://opensource.adobe.com/wiki/display/cmap/Downloads) + + Copyright 1990-2009 Adobe Systems Incorporated. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +PaDaF PDF/A preflight (http://sourceforge.net/projects/padaf) + + Copyright 2010 Atos Worldline SAS + + Licensed by Atos Worldline SAS under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + Atos Worldline SAS licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OSXAdapter + + Version: 2.0 + + Disclaimer: IMPORTANT: This Apple software is supplied to you by + Apple Inc. ("Apple") in consideration of your agreement to the + following terms, and your use, installation, modification or + redistribution of this Apple software constitutes acceptance of these + terms. If you do not agree with these terms, please do not use, + install, modify or redistribute this Apple software. + + In consideration of your agreement to abide by the following terms, and + subject to these terms, Apple grants you a personal, non-exclusive + license, under Apple's copyrights in this original Apple software (the + "Apple Software"), to use, reproduce, modify and redistribute the Apple + Software, with or without modifications, in source and/or binary forms; + provided that if you redistribute the Apple Software in its entirety and + without modifications, you must retain this notice and the following + text and disclaimers in all such redistributions of the Apple Software. + Neither the name, trademarks, service marks or logos of Apple Inc. + may be used to endorse or promote products derived from the Apple + Software without specific prior written permission from Apple. Except + as expressly stated in this notice, no other rights or licenses, express + or implied, are granted by Apple herein, including but not limited to + any patent rights that may be infringed by your derivative works or by + other works in which the Apple Software may be incorporated. + + The Apple Software is provided by Apple on an "AS IS" basis. APPLE + MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION + THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND + OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + + IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL + OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, + MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED + AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), + STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/ingest-attachment/licenses/pdfbox-io-NOTICE.txt b/plugins/ingest-attachment/licenses/pdfbox-io-NOTICE.txt new file mode 100644 index 0000000000000..3c85708256104 --- /dev/null +++ b/plugins/ingest-attachment/licenses/pdfbox-io-NOTICE.txt @@ -0,0 +1,22 @@ +Apache PDFBox +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Based on source code originally developed in the PDFBox and +FontBox projects. + +Copyright (c) 2002-2007, www.pdfbox.org + +Based on source code originally developed in the PaDaF project. +Copyright (c) 2010 Atos Worldline SAS + +Includes the Adobe Glyph List +Copyright 1997, 1998, 2002, 2007, 2010 Adobe Systems Incorporated. + +Includes the Zapf Dingbats Glyph List +Copyright 2002, 2010 Adobe Systems Incorporated. + +Includes OSXAdapter +Copyright (C) 2003-2007 Apple, Inc., All Rights Reserved diff --git a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/ingest-attachment/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/ingest-attachment/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/ingest-attachment/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tagsoup-1.2.1.jar.sha1 b/plugins/ingest-attachment/licenses/tagsoup-1.2.1.jar.sha1 deleted file mode 100644 index 5d227b11a0fa6..0000000000000 --- a/plugins/ingest-attachment/licenses/tagsoup-1.2.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5584627487e984c03456266d3f8802eb85a9ce97 diff --git a/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1 deleted file mode 100644 index 80635a63d29fe..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -796a21391780339e3d4862626339b49df170024e \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-core-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..01df6be02361e --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-core-3.2.2.jar.sha1 @@ -0,0 +1 @@ +f1f16ecac7a81e145051f906927ea6b58ce7e914 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1 deleted file mode 100644 index a4bb6d48c6a08..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a48a287e464b456a85c79f318d7bad7db201518 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..b692ab8befa3b --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-3.2.2.jar.sha1 @@ -0,0 +1 @@ +3ee2907773fe2aaa1013829e00cd62778d6a2ff9 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1 deleted file mode 100644 index dbaee880d1251..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -758dac27c246c51b019562bab7e266d2da6a6e01 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-apple-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-apple-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..7ef86ac18757b --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-apple-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +fde21727740a39beead899c9ca6e642f92d86e3a \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1 deleted file mode 100644 index b4806746301ef..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -47f6a4c46b92616d14e82cd7ad4d05cb43077b83 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-html-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-html-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..351a9d6963000 --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-html-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +e6acd314da558703977a681661c215f3ef92dbbd \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1 deleted file mode 100644 index da1ae42bac652..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -235a20823c02c699ce3d57f3d6b9550db05d91a9 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..bcc475b3f4c1d --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +41ff68abccde91ab17d7b181eb7a5fccf16e8b5c \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1 deleted file mode 100644 index 7ceed9e1643b8..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7688a4220d07c32b505230479f957cd495c0bef2 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..a7ac03630fe9c --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +d4078f950ca55c5235cdfcad744235242f9edc05 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1 deleted file mode 100644 index e780c1b92d525..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4d0f0e3f6eff184040402094f4fabbb3c5c7d09f \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-pdf-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-pdf-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..c9baba749d403 --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-pdf-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +a972d70ef0762b460c048c5e0e8a46c46bb170aa \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1 deleted file mode 100644 index 6e56fcffc5f88..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3a93e538ba6cb4066aba96d629febf181ec9f92 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-text-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-text-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..c84219d17252b --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-text-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +a19be47ecca1a061349dc2d019ab6f2741ff1dee \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1 deleted file mode 100644 index 27062077b92bf..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ff707716c0c4748ffeb21996aefa8d269b3eab5b \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-xml-module-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xml-module-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..e63b0f71f2d19 --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-xml-module-3.2.2.jar.sha1 @@ -0,0 +1 @@ +9dd2f1c52ab2663600e82dae3a8003ce6ede372f \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1 deleted file mode 100644 index 396e2655b14db..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -69104107ff85194df5acf682178128771863e442 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..98b09c1785d78 --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-3.2.2.jar.sha1 @@ -0,0 +1 @@ +f1dfa02a2c672153013d44501e0c21d5682aa822 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1 deleted file mode 100644 index bda62033e4e8c..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fcea85a56f93a5c0cb81f3d6dd8673f3d81c598 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parser-zip-commons-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-zip-commons-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..ac860449a84dd --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parser-zip-commons-3.2.2.jar.sha1 @@ -0,0 +1 @@ +d46b71ea5697f575c3febfd7343e5d8b2c338bd5 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1 deleted file mode 100644 index bb76974b6344e..0000000000000 --- a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c8408deb51fa617ef4e912b4d161712e695d3a29 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-3.2.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-3.2.2.jar.sha1 new file mode 100644 index 0000000000000..f6e9d188908cd --- /dev/null +++ b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-3.2.2.jar.sha1 @@ -0,0 +1 @@ +c91fb85f5ee46e2c1f1e3399b04efb9d1ff85485 \ No newline at end of file diff --git a/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java b/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java index d999d20537485..068f1ae5d6d78 100644 --- a/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java +++ b/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java @@ -32,6 +32,16 @@ package org.opensearch.ingest.attachment; +import org.apache.fontbox.FontBoxFont; +import org.apache.fontbox.ttf.TTFParser; +import org.apache.fontbox.ttf.TrueTypeFont; +import org.apache.pdfbox.io.RandomAccessReadBuffer; +import org.apache.pdfbox.pdmodel.font.CIDFontMapping; +import org.apache.pdfbox.pdmodel.font.FontMapper; +import org.apache.pdfbox.pdmodel.font.FontMappers; +import org.apache.pdfbox.pdmodel.font.FontMapping; +import org.apache.pdfbox.pdmodel.font.PDCIDSystemInfo; +import org.apache.pdfbox.pdmodel.font.PDFontDescriptor; import org.apache.tika.Tika; import org.apache.tika.exception.TikaException; import org.apache.tika.metadata.Metadata; @@ -47,6 +57,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.lang.reflect.ReflectPermission; import java.net.URISyntaxException; @@ -75,6 +86,44 @@ */ final class TikaImpl { + static { + /* + * Stop PDFBox from consulting the OS for fonts at all, use classpath instead with dummy fonts because font + * does not matter for ingestion + */ + FontMappers.set(new FontMapper() { + @Override + public FontMapping getTrueTypeFont(String baseFont, PDFontDescriptor fd) { + try (InputStream in = TikaImpl.class.getResourceAsStream("/fonts/Roboto-Regular.ttf")) { + if (in == null) return new FontMapping<>(null, true); + byte[] bytes = in.readAllBytes(); + TrueTypeFont ttf = new TTFParser().parse(new RandomAccessReadBuffer(bytes)); + return new FontMapping<>(ttf, true); + } catch (IOException e) { + return new FontMapping<>(null, true); + } + } + + @Override + public FontMapping getFontBoxFont(String baseFont, PDFontDescriptor fd) { + try (InputStream in = TikaImpl.class.getResourceAsStream("/fonts/Roboto-Regular.ttf")) { + if (in == null) return new FontMapping<>(null, true); + byte[] bytes = in.readAllBytes(); + TrueTypeFont ttf = new TTFParser().parse(new RandomAccessReadBuffer(bytes)); + return new FontMapping<>(ttf, true); + } catch (IOException e) { + return new FontMapping<>(null, true); + } + } + + @Override + public CIDFontMapping getCIDFont(String baseFont, PDFontDescriptor fd, PDCIDSystemInfo cid) { + // No CID substitutions from the OS either; signal "fallback only". + return new CIDFontMapping(null, null, true); + } + }); + } + /** Exclude some formats */ private static final Set EXCLUDES = new HashSet<>( Arrays.asList( @@ -91,7 +140,7 @@ final class TikaImpl { /** subset of parsers for types we support */ private static final Parser PARSERS[] = new Parser[] { // documents - new org.apache.tika.parser.html.HtmlParser(), + new org.apache.tika.parser.html.JSoupParser(), new org.apache.tika.parser.pdf.PDFParser(), new org.apache.tika.parser.txt.TXTParser(), new org.apache.tika.parser.microsoft.rtf.RTFParser(), diff --git a/plugins/ingest-attachment/src/main/resources/fonts/Roboto-Regular.ttf b/plugins/ingest-attachment/src/main/resources/fonts/Roboto-Regular.ttf new file mode 100644 index 0000000000000..7e3bb2f8ce7ae Binary files /dev/null and b/plugins/ingest-attachment/src/main/resources/fonts/Roboto-Regular.ttf differ diff --git a/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums b/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums index 227d7d833a231..cbbf7dc49bd8e 100644 --- a/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums +++ b/plugins/ingest-attachment/src/test/resources/org/opensearch/ingest/attachment/test/.checksums @@ -3,7 +3,7 @@ "testWORD_1img.docx": "367e2ade13ca3c19bcd8a323e21d51d407e017ac", "testMasterFooter.odp": "bcc59df70699c739423a50e362c722b81ae76498", "testTXTNonASCIIUTF8.txt": "1ef514431ca8d838f11e99f8e4a0637730b77aa0", - "EmbeddedOutlook.docx": "c544a6765c19ba11b0bf3edb55c79e1bd8565c6e", + "EmbeddedOutlook.docx": "770c14c1f8d1cb3ff431a6ea7d0cbd9f5091f1f5", "testWORD_override_list_numbering.docx": "4e892319b921322916225def763f451e4bbb4e16", "testTextBoxes.key": "b01581d5bd2483ce649a1a1406136359f4b93167", "testPPT_masterText.pptx": "9fee8337b76dc3e196f4554dcde22b9dd1c3b3e8", @@ -64,9 +64,9 @@ "testRTFTableCellSeparation2.rtf": "62782ca40ff0ed6c3ba90f8055ee724b44af203f", "testPagesHeadersFootersRomanLower.pages": "2410fc803907001eb39c201ad4184b243e271c6d", "headerPic.docx": "c704bb648feac7975dff1024a5f762325be7cbc2", - "testHTMLNoisyMetaEncoding_4.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a", + "testHTMLNoisyMetaEncoding_4.html": "83d08bacf04d72f04b9ac67df81e9e63a891d744", "testRTFBoldItalic.rtf": "0475d224078682cf3f9f3f4cbc14a63456c5a0d8", - "test-outlook.msg": "1f202fc11a873e305d5b4d4607409f3f734065ec", + "test-outlook.msg": "ef14d2bbbe167b5d3500dcab3950cfa22cd94665", "testRTFVarious.rtf": "bf6ea9cf57886e680c5e6743a66a12b950a09083", "testXHTML.html": "c6da900f81c1c550518e65d579d3dd62dd7c5c0c", "EmbeddedPDF.docx": "454476bdf4a968189a6f53e75c146382bf58a434", @@ -101,13 +101,13 @@ "testWORD_override_list_numbering.doc": "60e47a3e71ba08af20af96131d61740a1f0bafa3", "testPDF_twoAuthors.pdf": "c5f0296cc21f9ae99ceb649b561c55f99d7d9452", "testPDF_Version.10.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2", - "testHTMLNoisyMetaEncoding_2.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a", + "testHTMLNoisyMetaEncoding_2.html": "83d08bacf04d72f04b9ac67df81e9e63a891d744", "testFooter.odt": "cd5d0fcbcf48d6f005d087c47d00e84f39bcc321", "testPPT.pptm": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6", "testPPT_various.ppt": "399e27a9893284f106dc44f15b5e636454db681e", "testRTFListMicrosoftWord.rtf": "0303eb3e2f30530621a7a407847b759a3b21467e", "testWORD_bold_character_runs2.doc": "f10e562d8825ec2e17e0d9f58646f8084a658cfa", - "boilerplate-whitespace.html": "a9372bc75d7d84cbcbb0bce68fcaed73ad8ef52c", + "boilerplate-whitespace.html": "bf1fd3ffcf798afd688254bbc899e388eda9e546", "testEXCEL_95.xls": "20d9b9b0f3aecd28607516b4b837c8bab3524b6c", "testPPT_embedded_two_slides.pptx": "", "testPDF_bookmarks.pdf": "5fc486c443511452db4f1aa6530714c6aa49c831", @@ -121,14 +121,14 @@ "testPDF_Version.4.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2", "testBinControlWord.rtf": "ef858fbb7584ea7f92ffed8d0a08c1cc35ffee07", "testWORD_null_style.docx": "0be9dcfb83423c78a06af514ec21e4e7770ec48e", - "test-outlook2003.msg": "bb3c35eb7e95d657d7977c1d3d52862734f9f329", + "test-outlook2003.msg": "b9c21661a59254c8d6a9b665e28070757a354cbe", "testPDFVarious.pdf": "c66bbbacb10dd27430f7d0bed9518e75793cedae", - "testHTMLNoisyMetaEncoding_3.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a", + "testHTMLNoisyMetaEncoding_3.html": "83d08bacf04d72f04b9ac67df81e9e63a891d744", "testRTFCorruptListOverride.rtf": "116a782d02a7f25010a15cbbb189bf98e6b89855", "testEXCEL_custom_props.xls": "b5584d9b13ab1566ce539238dc75e7eb3449ba7f", "testPDF_Version.7.x.pdf": "03b60dfc8c103dbabeedfd682e979f96dd8983a2", "testPDFEmbeddingAndEmbedded.docx": "e7b648adb15cd16cdd84437c2b9524a8eeb213e4", - "testHTMLNoisyMetaEncoding_1.html": "630e14e3495a78580c4e26fa3bbe3123ccf4fd8a", + "testHTMLNoisyMetaEncoding_1.html": "83d08bacf04d72f04b9ac67df81e9e63a891d744", "testWORD_3imgs.doc": "818aa8c6c44dd78c49100c3c38e95abdf3812981", "testRTFEmbeddedLink.rtf": "2720ffb5ff3a6bbb2c5c1cb43fb4922362ed788a", "testKeynote.key": "11387b59fc6339bb73653fcbb26d387521b98ec9", @@ -156,7 +156,7 @@ "testWORD_custom_props.doc": "e7a737a5237a6aa9c6b3fc677eb8fa65c30d6dfe", "testPDF_Version.11.x.PDFA-1b.pdf": "71853c6197a6a7f222db0f1978c7cb232b87c5ee", "testAnnotations.pdf": "5f599e7916198540e1b52c3e472a525f50fd45f6", - "tika434.html": "7d74122631f52f003a48018cc376026ccd8d984e", + "tika434.html": "51cafe6636423e37c05e676cb1454e72961b8f04", "testPagesHeadersFootersAlphaLower.pages": "fc1d766908134ff4689fa63fa3e91c3e9b08d975", "testRTFRegularImages.rtf": "756b1db45cb05357ceaf9c8efcf0b76e3913e190", "testRTFUmlautSpaces2.rtf": "1fcd029357062241d74d789e93477c101ff24e3f", @@ -166,7 +166,7 @@ "testMasterSlideTable.key": "1d61e2fa3c3f3615500c7f72f62971391b9e9a2f", "testWORD_various.doc": "8cbdf1a4e0d78471eb90403612c4e92866acf0cb", "testEXCEL_textbox.xlsx": "1e81121e91e58a74d838e414ae0fc0055a4b4100", - "big-preamble.html": "a9d759b46b6c6c1857d0d89c3a75ee2f3ace70c9", + "big-preamble.html": "edecdb8304a31bca1a71faab2153fa133989e6d8", "testWORD.docx": "f72140bef19475e950e56084d1ab1cb926697b19", "testComment.rtf": "f6351d0f1f20c4ee0fff70adca6abbc6e638610e", "testRTFUnicodeUCNControlWordCharacterDoubling.rtf": "3e6f2f38682e38ffc96a476ca51bec2291a27fa7", @@ -190,7 +190,7 @@ "testRTFIgnoredControlWord.rtf": "1eb6a2f2fd32b1bb4227c0c02a35cb6027d9ec8c", "testComment.xls": "4de962f16452159ce302fc4a412b06a06cf9a0f6", "testPPT.ppsm": "71333ef84f7825d8ad6aba2ba993d04b4bab41c6", - "boilerplate.html": "b3558f02c3179e4aeeb6057594d87bda79964e7b", + "boilerplate.html": "f1e3c82a4f16f67590a5afe4b64d90d98330d216", "testEXCEL_embeded.xls": "", "testEXCEL.xlsx": "", "testPPT_2imgs.ppt": "9a68072ffcf171389e78cf8bc018c4b568a6202d", diff --git a/plugins/ingestion-kafka/build.gradle b/plugins/ingestion-kafka/build.gradle index 6a9809674b39a..abd1b1a5c038c 100644 --- a/plugins/ingestion-kafka/build.gradle +++ b/plugins/ingestion-kafka/build.gradle @@ -41,6 +41,7 @@ dependencies { testImplementation "org.testcontainers:kafka:${versions.testcontainers}" testImplementation "org.rnorth.duct-tape:duct-tape:${versions.ducttape}" testImplementation "org.apache.commons:commons-compress:${versions.commonscompress}" + testImplementation "org.apache.commons:commons-lang3:${versions.commonslang}" testImplementation "commons-io:commons-io:${versions.commonsio}" testImplementation 'org.awaitility:awaitility:4.2.0' } @@ -67,9 +68,6 @@ thirdPartyAudit { 'net.jpountz.util.SafeUtils', 'net.jpountz.xxhash.XXHash32', 'net.jpountz.xxhash.XXHashFactory', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', 'com.google.common.util.concurrent.ListenableFuture', 'io.grpc.BindableService', 'io.grpc.CallOptions', diff --git a/plugins/ingestion-kafka/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingestion-kafka/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/ingestion-kafka/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingestion-kafka/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/ingestion-kafka/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/ingestion-kafka/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/ingestion-kinesis/build.gradle b/plugins/ingestion-kinesis/build.gradle index a8100018c7f4a..7acc7b8fbff46 100644 --- a/plugins/ingestion-kinesis/build.gradle +++ b/plugins/ingestion-kinesis/build.gradle @@ -126,10 +126,6 @@ thirdPartyAudit { 'org.apache.log4j.Logger', 'org.apache.log4j.Priority', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', - 'org.graalvm.nativeimage.hosted.Feature', 'org.graalvm.nativeimage.hosted.Feature$AfterImageWriteAccess', diff --git a/plugins/ingestion-kinesis/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/ingestion-kinesis/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/ingestion-kinesis/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/ingestion-kinesis/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/ingestion-kinesis/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/ingestion-kinesis/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java index e7e8d92cee65a..49aab68be416b 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingTests.java @@ -60,7 +60,7 @@ protected Collection> getPlugins() { } public void testSizeEnabled() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true"); + IndexService service = createIndexWithSimpleMappings("test", Settings.EMPTY, "_size", "enabled=true"); DocumentMapper docMapper = service.mapperService().documentMapper(); BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()); @@ -77,7 +77,7 @@ public void testSizeEnabled() throws Exception { } public void testSizeDisabled() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=false"); + IndexService service = createIndexWithSimpleMappings("test", Settings.EMPTY, "_size", "enabled=false"); DocumentMapper docMapper = service.mapperService().documentMapper(); BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()); @@ -87,7 +87,7 @@ public void testSizeDisabled() throws Exception { } public void testSizeNotSet() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME); + IndexService service = createIndexWithSimpleMappings("test", Settings.EMPTY); DocumentMapper docMapper = service.mapperService().documentMapper(); BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()); @@ -97,7 +97,7 @@ public void testSizeNotSet() throws Exception { } public void testThatDisablingWorksWhenMerging() throws Exception { - IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true"); + IndexService service = createIndexWithSimpleMappings("test", Settings.EMPTY, "_size", "enabled=true"); DocumentMapper docMapper = service.mapperService().documentMapper(); assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(true)); diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle index 78257161a5c82..3ba2c591644da 100644 --- a/plugins/repository-azure/build.gradle +++ b/plugins/repository-azure/build.gradle @@ -47,7 +47,7 @@ dependencies { api 'com.azure:azure-core:1.55.5' api 'com.azure:azure-json:1.5.0' api 'com.azure:azure-xml:1.2.0' - api 'com.azure:azure-storage-common:12.30.1' + api 'com.azure:azure-storage-common:12.30.2' api 'com.azure:azure-core-http-netty:1.15.12' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" @@ -56,7 +56,7 @@ dependencies { api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') - api 'com.azure:azure-storage-blob:12.30.1' + api 'com.azure:azure-storage-blob:12.31.2' api 'com.azure:azure-identity:1.14.2' // Start of transitive dependencies for azure-identity api 'com.microsoft.azure:msal4j-persistence-extension:1.3.0' @@ -170,9 +170,6 @@ thirdPartyAudit { 'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters', 'org.osgi.framework.BundleActivator', 'org.osgi.framework.BundleContext', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', 'io.micrometer.common.KeyValue', 'io.micrometer.common.KeyValues', 'io.micrometer.common.docs.KeyName', diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 deleted file mode 100644 index 34189c82a88ba..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-blob-12.30.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -deaa55c7c985bec01cbbc4fef41d2da3d511dcbc \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-blob-12.31.2.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-blob-12.31.2.jar.sha1 new file mode 100644 index 0000000000000..1a22d5360fe1a --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-blob-12.31.2.jar.sha1 @@ -0,0 +1 @@ +092c5c3fb7796f42bece7f3f6d3fc51072b71475 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.30.1.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.30.1.jar.sha1 deleted file mode 100644 index 16690b638df84..0000000000000 --- a/plugins/repository-azure/licenses/azure-storage-common-12.30.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e94e0c1e780e479bc328ccaf35f10fd2c76c9778 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/azure-storage-common-12.30.2.jar.sha1 b/plugins/repository-azure/licenses/azure-storage-common-12.30.2.jar.sha1 new file mode 100644 index 0000000000000..b78e3fc5f5ad2 --- /dev/null +++ b/plugins/repository-azure/licenses/azure-storage-common-12.30.2.jar.sha1 @@ -0,0 +1 @@ +203214375d7fbf214f5cacefd2c851e87a708a98 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-azure/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/repository-azure/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/repository-azure/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index e8338976fae5d..881f56f91ca61 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -74,8 +74,8 @@ dependencies { runtimeOnly "com.google.code.gson:gson:2.12.1" runtimeOnly "com.google.api.grpc:proto-google-common-protos:2.60.0" runtimeOnly "com.google.api.grpc:proto-google-iam-v1:1.55.0" - implementation "com.google.auth:google-auth-library-credentials:1.37.1" - implementation "com.google.auth:google-auth-library-oauth2-http:1.37.1" + implementation "com.google.auth:google-auth-library-credentials:1.38.0" + implementation "com.google.auth:google-auth-library-oauth2-http:1.38.0" runtimeOnly "com.google.oauth-client:google-oauth-client:1.39.0" // 1.39.0 in bom implementation "com.google.api-client:google-api-client:2.7.2" implementation "com.google.http-client:google-http-client:1.47.1" @@ -277,11 +277,6 @@ thirdPartyAudit { 'org.graalvm.nativeimage.hosted.Feature$DuringAnalysisAccess', 'org.graalvm.nativeimage.hosted.Feature$FeatureAccess', 'org.graalvm.nativeimage.hosted.RuntimeReflection', - //slf4j dependencies - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', - 'org.slf4j.spi.LoggingEventBuilder', ) } diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 deleted file mode 100644 index fd3bcfebb878e..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.37.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -894c1cd371380e254290ac7c7df04372bf547a8f \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-credentials-1.38.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.38.0.jar.sha1 new file mode 100644 index 0000000000000..866b777fb139b --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-credentials-1.38.0.jar.sha1 @@ -0,0 +1 @@ +0fa8a919c22292e2617e6adf2554dc3e9260797d \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 deleted file mode 100644 index a0e34c8071d43..0000000000000 --- a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.37.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -86a3c90a6b80128fccac09dead6158fe7cc5e7bd \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.38.0.jar.sha1 b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.38.0.jar.sha1 new file mode 100644 index 0000000000000..d42722a0ea0f5 --- /dev/null +++ b/plugins/repository-gcs/licenses/google-auth-library-oauth2-http-1.38.0.jar.sha1 @@ -0,0 +1 @@ +7910bf19b88fd9c34b1c8dce353102c2eb0f9399 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-gcs/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/repository-gcs/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/repository-gcs/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1 deleted file mode 100644 index 912bda85de18a..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -44331c1130c370e726a2e1a3e6fba6d2558ef04a \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.28.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.28.0.jar.sha1 new file mode 100644 index 0000000000000..5edae62aeeb5d --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-compress-1.28.0.jar.sha1 @@ -0,0 +1 @@ +e482f2c7a88dac3c497e96aa420b6a769f59c8d7 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-hdfs/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/repository-hdfs/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/repository-hdfs/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index b1a83565f0e87..643b34797ccc8 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -78,7 +78,8 @@ dependencies { api "software.amazon.awssdk:aws-query-protocol:${versions.aws}" api "software.amazon.awssdk:sts:${versions.aws}" api "software.amazon.awssdk:netty-nio-client:${versions.aws}" - + api "software.amazon.awssdk:crt-core:${versions.aws}" + api "software.amazon.awssdk:aws-crt-client:${versions.aws}" api "org.apache.httpcomponents:httpclient:${versions.httpclient}" api "org.apache.httpcomponents:httpcore:${versions.httpcore}" api "commons-logging:commons-logging:${versions.commonslogging}" @@ -545,13 +546,6 @@ thirdPartyAudit { 'software.amazon.awssdk.arns.Arn', 'software.amazon.awssdk.arns.ArnResource', - 'software.amazon.awssdk.crtcore.CrtConfigurationUtils', - 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration', - 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder', - 'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder', - 'software.amazon.awssdk.crtcore.CrtProxyConfiguration', - 'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder', - 'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder', 'software.amazon.eventstream.HeaderValue', 'software.amazon.eventstream.Message', 'software.amazon.eventstream.MessageDecoder' diff --git a/plugins/repository-s3/licenses/aws-crt-client-2.30.31.jar.sha1 b/plugins/repository-s3/licenses/aws-crt-client-2.30.31.jar.sha1 new file mode 100644 index 0000000000000..61ce2ed2a2234 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-crt-client-2.30.31.jar.sha1 @@ -0,0 +1 @@ +05dd1f7501ec4062622f2dd2231caad8d54079e3 \ No newline at end of file diff --git a/plugins/ingest-attachment/licenses/tagsoup-LICENSE.txt b/plugins/repository-s3/licenses/aws-crt-client-LICENSE.txt similarity index 99% rename from plugins/ingest-attachment/licenses/tagsoup-LICENSE.txt rename to plugins/repository-s3/licenses/aws-crt-client-LICENSE.txt index 261eeb9e9f8b2..d645695673349 100644 --- a/plugins/ingest-attachment/licenses/tagsoup-LICENSE.txt +++ b/plugins/repository-s3/licenses/aws-crt-client-LICENSE.txt @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/plugins/repository-s3/licenses/aws-crt-client-NOTICE.txt b/plugins/repository-s3/licenses/aws-crt-client-NOTICE.txt new file mode 100644 index 0000000000000..6c7dc983f8c7a --- /dev/null +++ b/plugins/repository-s3/licenses/aws-crt-client-NOTICE.txt @@ -0,0 +1,12 @@ +OpenSearch (https://opensearch.org/) +Copyright OpenSearch Contributors + +This product includes software developed by +Elasticsearch (http://www.elastic.co). +Copyright 2009-2018 Elasticsearch + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by +Joda.org (http://www.joda.org/). diff --git a/plugins/repository-s3/licenses/crt-core-2.30.31.jar.sha1 b/plugins/repository-s3/licenses/crt-core-2.30.31.jar.sha1 new file mode 100644 index 0000000000000..4b26ce35772ed --- /dev/null +++ b/plugins/repository-s3/licenses/crt-core-2.30.31.jar.sha1 @@ -0,0 +1 @@ +0da8346395a4b95003c1effd9ed4df7708185e5a \ No newline at end of file diff --git a/plugins/repository-s3/licenses/crt-core-LICENSE.txt b/plugins/repository-s3/licenses/crt-core-LICENSE.txt new file mode 100644 index 0000000000000..1eef70a9b9f42 --- /dev/null +++ b/plugins/repository-s3/licenses/crt-core-LICENSE.txt @@ -0,0 +1,206 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Note: Other license terms may apply to certain, identified software files contained within or distributed + with the accompanying software if such terms are included in the directory containing the accompanying software. + Such other license terms will then apply in lieu of the terms of the software license above. diff --git a/plugins/repository-s3/licenses/crt-core-NOTICE.txt b/plugins/repository-s3/licenses/crt-core-NOTICE.txt new file mode 100644 index 0000000000000..4c36a6c147c4a --- /dev/null +++ b/plugins/repository-s3/licenses/crt-core-NOTICE.txt @@ -0,0 +1,25 @@ +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). diff --git a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 deleted file mode 100644 index 77b9917528382..0000000000000 --- a/plugins/repository-s3/licenses/slf4j-api-1.7.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6c62681a2f655b49963a5983b8b0950a6120ae14 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/slf4j-api-2.0.17.jar.sha1 b/plugins/repository-s3/licenses/slf4j-api-2.0.17.jar.sha1 new file mode 100644 index 0000000000000..435f6c13a28b6 --- /dev/null +++ b/plugins/repository-s3/licenses/slf4j-api-2.0.17.jar.sha1 @@ -0,0 +1 @@ +d9e58ac9c7779ba3bf8142aff6c830617a7fe60f \ No newline at end of file diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java index d54abb413c6fd..15dd2b875ebc4 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -31,6 +31,8 @@ package org.opensearch.repositories.s3; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; @@ -87,6 +89,7 @@ import static org.hamcrest.Matchers.equalTo; @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") +@ThreadLeakFilters(filters = EventLoopThreadFilter.class) // Need to set up a new cluster for each test because cluster settings use randomized authentication settings @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class S3BlobStoreRepositoryTests extends OpenSearchMockAPIBasedRepositoryIntegTestCase { diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java index 79b5cc654b921..8f198f144b23c 100644 --- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -31,6 +31,8 @@ package org.opensearch.repositories.s3; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import software.amazon.awssdk.services.s3.model.StorageClass; import org.opensearch.common.SuppressForbidden; @@ -53,6 +55,7 @@ import static org.hamcrest.Matchers.blankOrNullString; import static org.hamcrest.Matchers.not; +@ThreadLeakFilters(filters = EventLoopThreadFilter.class) public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { @Override diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java index afbeaff323d51..862e91c291073 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3AsyncService.java @@ -22,8 +22,9 @@ import software.amazon.awssdk.core.retry.RetryPolicy; import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient; +import software.amazon.awssdk.http.crt.ProxyConfiguration; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; -import software.amazon.awssdk.http.nio.netty.ProxyConfiguration; import software.amazon.awssdk.http.nio.netty.SdkEventLoopGroup; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -63,7 +64,10 @@ class S3AsyncService implements Closeable { private static final String DEFAULT_S3_ENDPOINT = "s3.amazonaws.com"; - private volatile Map clientsCache = emptyMap(); + // We will need to support the cache with both type of clients. Since S3ClientSettings doesn't contain Http Client. + // Also adding the Http Client type in S3ClientSettings is not good option since it is used by Async and Sync clients. + // We can segregate the types of cache here itself + private volatile Map> s3HttpClientTypesClientsCache = emptyMap(); /** * Client settings calculated from static configuration and settings in the keystore. @@ -82,12 +86,24 @@ class S3AsyncService implements Closeable { private final @Nullable ScheduledExecutorService clientExecutorService; S3AsyncService(final Path configPath, @Nullable ScheduledExecutorService clientExecutorService) { + staticClientSettings = MapBuilder.newMapBuilder() - .put("default", S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath)) + .put( + buildClientName("default", S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE), + S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath) + ) + .put( + buildClientName("default", S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE), + S3ClientSettings.getClientSettings(Settings.EMPTY, "default", configPath) + ) .immutableMap(); this.clientExecutorService = clientExecutorService; } + private String buildClientName(final String clientValue, final String asyncClientType) { + return clientValue + "-" + asyncClientType; + } + S3AsyncService(final Path configPath) { this(configPath, null); } @@ -102,9 +118,24 @@ public synchronized void refreshAndClearCache(Map clie // shutdown all unused clients // others will shutdown on their respective release releaseCachedClients(); - this.staticClientSettings = MapBuilder.newMapBuilder(clientsSettings).immutableMap(); + MapBuilder defaultBuilder = MapBuilder.newMapBuilder(); + for (Map.Entry entrySet : clientsSettings.entrySet()) { + defaultBuilder.put( + buildClientName(entrySet.getKey(), S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE), + clientsSettings.get(entrySet.getKey()) + ); + defaultBuilder.put( + buildClientName(entrySet.getKey(), S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE), + clientsSettings.get(entrySet.getKey()) + ); + } + + staticClientSettings = defaultBuilder.immutableMap(); derivedClientSettings = emptyMap(); - assert this.staticClientSettings.containsKey("default") : "always at least have 'default'"; + assert this.staticClientSettings.containsKey(buildClientName("default", S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE)) + : "Static Client Settings should contain default Netty client"; + assert this.staticClientSettings.containsKey(buildClientName("default", S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE)) + : "Static Client Settings should contain default CRT client"; // clients are built lazily by {@link client} } @@ -118,28 +149,57 @@ public AmazonAsyncS3Reference client( AsyncExecutorContainer priorityExecutorBuilder, AsyncExecutorContainer normalExecutorBuilder ) { + String asyncHttpClientType = S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE.get(repositoryMetadata.settings()); + final S3ClientSettings clientSettings = settings(repositoryMetadata); - { - final AmazonAsyncS3Reference clientReference = clientsCache.get(clientSettings); - if (clientReference != null && clientReference.tryIncRef()) { - return clientReference; - } + AmazonAsyncS3Reference clientReference = getCachedClientForHttpTypeAndClientSettings(asyncHttpClientType, clientSettings); + if (clientReference != null) { + return clientReference; } + synchronized (this) { - final AmazonAsyncS3Reference existing = clientsCache.get(clientSettings); - if (existing != null && existing.tryIncRef()) { - return existing; + AmazonAsyncS3Reference existingClient = getCachedClientForHttpTypeAndClientSettings(asyncHttpClientType, clientSettings); + if (existingClient != null) { + return existingClient; } - final AmazonAsyncS3Reference clientReference = new AmazonAsyncS3Reference( - buildClient(clientSettings, urgentExecutorBuilder, priorityExecutorBuilder, normalExecutorBuilder) + // If the client reference is not found in cache. Let's create it. + final AmazonAsyncS3Reference newClientReference = new AmazonAsyncS3Reference( + buildClient(clientSettings, urgentExecutorBuilder, priorityExecutorBuilder, normalExecutorBuilder, asyncHttpClientType) ); - clientReference.incRef(); - clientsCache = MapBuilder.newMapBuilder(clientsCache).put(clientSettings, clientReference).immutableMap(); - return clientReference; + newClientReference.incRef(); + + // Get or create new client cache map for the HTTP client type + Map clientsCacheForType = s3HttpClientTypesClientsCache.getOrDefault( + asyncHttpClientType, + emptyMap() + ); + + // Update both cache levels atomically + s3HttpClientTypesClientsCache = MapBuilder.newMapBuilder(s3HttpClientTypesClientsCache) + .put( + asyncHttpClientType, + MapBuilder.newMapBuilder(clientsCacheForType).put(clientSettings, newClientReference).immutableMap() + ) + .immutableMap(); + return newClientReference; } } + private AmazonAsyncS3Reference getCachedClientForHttpTypeAndClientSettings( + final String asyncHttpClientType, + final S3ClientSettings clientSettings + ) { + final Map clientsCacheMap = s3HttpClientTypesClientsCache.get(asyncHttpClientType); + if (clientsCacheMap != null && !clientsCacheMap.isEmpty()) { + final AmazonAsyncS3Reference clientReference = clientsCacheMap.get(clientSettings); + if (clientReference != null && clientReference.tryIncRef()) { + return clientReference; + } + } + return null; + } + /** * Either fetches {@link S3ClientSettings} for a given {@link RepositoryMetadata} from cached settings or creates them * by overriding static client settings from {@link #staticClientSettings} with settings found in the repository metadata. @@ -154,7 +214,10 @@ S3ClientSettings settings(RepositoryMetadata repositoryMetadata) { return existing; } } - final String clientName = S3Repository.CLIENT_NAME.get(settings); + final String clientName = buildClientName( + S3Repository.CLIENT_NAME.get(settings), + S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE.get(repositoryMetadata.settings()) + ); final S3ClientSettings staticSettings = staticClientSettings.get(clientName); if (staticSettings != null) { synchronized (this) { @@ -180,7 +243,8 @@ synchronized AmazonAsyncS3WithCredentials buildClient( final S3ClientSettings clientSettings, AsyncExecutorContainer urgentExecutorBuilder, AsyncExecutorContainer priorityExecutorBuilder, - AsyncExecutorContainer normalExecutorBuilder + AsyncExecutorContainer normalExecutorBuilder, + String asyncHttpClientType ) { setDefaultAwsProfilePath(); final S3AsyncClientBuilder builder = S3AsyncClient.builder(); @@ -209,7 +273,7 @@ synchronized AmazonAsyncS3WithCredentials buildClient( builder.forcePathStyle(true); } - builder.httpClient(buildHttpClient(clientSettings, urgentExecutorBuilder.getAsyncTransferEventLoopGroup())); + builder.httpClient(buildHttpClient(clientSettings, urgentExecutorBuilder.getAsyncTransferEventLoopGroup(), asyncHttpClientType)); builder.asyncConfiguration( ClientAsyncConfiguration.builder() .advancedOption( @@ -220,7 +284,7 @@ synchronized AmazonAsyncS3WithCredentials buildClient( ); final S3AsyncClient urgentClient = SocketAccess.doPrivileged(builder::build); - builder.httpClient(buildHttpClient(clientSettings, priorityExecutorBuilder.getAsyncTransferEventLoopGroup())); + builder.httpClient(buildHttpClient(clientSettings, priorityExecutorBuilder.getAsyncTransferEventLoopGroup(), asyncHttpClientType)); builder.asyncConfiguration( ClientAsyncConfiguration.builder() .advancedOption( @@ -231,7 +295,7 @@ synchronized AmazonAsyncS3WithCredentials buildClient( ); final S3AsyncClient priorityClient = SocketAccess.doPrivileged(builder::build); - builder.httpClient(buildHttpClient(clientSettings, normalExecutorBuilder.getAsyncTransferEventLoopGroup())); + builder.httpClient(buildHttpClient(clientSettings, normalExecutorBuilder.getAsyncTransferEventLoopGroup(), asyncHttpClientType)); builder.asyncConfiguration( ClientAsyncConfiguration.builder() .advancedOption( @@ -241,38 +305,32 @@ synchronized AmazonAsyncS3WithCredentials buildClient( .build() ); final S3AsyncClient client = SocketAccess.doPrivileged(builder::build); - return AmazonAsyncS3WithCredentials.create(client, priorityClient, urgentClient, credentials); } - static ClientOverrideConfiguration buildOverrideConfiguration( - final S3ClientSettings clientSettings, - ScheduledExecutorService clientExecutorService + static SdkAsyncHttpClient buildHttpClient( + S3ClientSettings clientSettings, + AsyncTransferEventLoopGroup asyncTransferEventLoopGroup, + final String asyncHttpClientType ) { - RetryPolicy retryPolicy = SocketAccess.doPrivileged( - () -> RetryPolicy.builder() - .numRetries(clientSettings.maxRetries) - .throttlingBackoffStrategy( - clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD) : BackoffStrategy.none() - ) - .build() - ); - ClientOverrideConfiguration.Builder builder = ClientOverrideConfiguration.builder(); - if (clientExecutorService != null) { - builder = builder.scheduledExecutorService(clientExecutorService); + logger.debug("S3 Http client type [{}]", asyncHttpClientType); + if (S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE.equals(asyncHttpClientType)) { + return buildAsyncNettyHttpClient(clientSettings, asyncTransferEventLoopGroup); } - - return builder.retryPolicy(retryPolicy).apiCallAttemptTimeout(Duration.ofMillis(clientSettings.requestTimeoutMillis)).build(); + return buildAsyncCrtHttpClient(clientSettings); } - // pkg private for tests - static SdkAsyncHttpClient buildHttpClient(S3ClientSettings clientSettings, AsyncTransferEventLoopGroup asyncTransferEventLoopGroup) { + static SdkAsyncHttpClient buildAsyncNettyHttpClient( + final S3ClientSettings clientSettings, + final AsyncTransferEventLoopGroup asyncTransferEventLoopGroup + ) { // the response metadata cache is only there for diagnostics purposes, // but can force objects from every response to the old generation. NettyNioAsyncHttpClient.Builder clientBuilder = NettyNioAsyncHttpClient.builder(); if (clientSettings.proxySettings.getType() != ProxySettings.ProxyType.DIRECT) { - ProxyConfiguration.Builder proxyConfiguration = ProxyConfiguration.builder(); + software.amazon.awssdk.http.nio.netty.ProxyConfiguration.Builder proxyConfiguration = + software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder(); proxyConfiguration.scheme(clientSettings.proxySettings.getType().toProtocol().toString()); proxyConfiguration.host(clientSettings.proxySettings.getHostName()); proxyConfiguration.port(clientSettings.proxySettings.getPort()); @@ -292,6 +350,46 @@ static SdkAsyncHttpClient buildHttpClient(S3ClientSettings clientSettings, Async return clientBuilder.build(); } + static SdkAsyncHttpClient buildAsyncCrtHttpClient(final S3ClientSettings clientSettings) { + AwsCrtAsyncHttpClient.Builder crtClientBuilder = AwsCrtAsyncHttpClient.builder(); + + if (clientSettings.proxySettings.getType() != ProxySettings.ProxyType.DIRECT) { + ProxyConfiguration.Builder crtProxyConfiguration = ProxyConfiguration.builder(); + + crtProxyConfiguration.scheme(clientSettings.proxySettings.getType().toProtocol().toString()); + crtProxyConfiguration.host(clientSettings.proxySettings.getHostName()); + crtProxyConfiguration.port(clientSettings.proxySettings.getPort()); + crtProxyConfiguration.username(clientSettings.proxySettings.getUsername()); + crtProxyConfiguration.password(clientSettings.proxySettings.getPassword()); + + crtClientBuilder.proxyConfiguration(crtProxyConfiguration.build()); + } + + crtClientBuilder.connectionTimeout(Duration.ofMillis(clientSettings.connectionTimeoutMillis)); + crtClientBuilder.maxConcurrency(clientSettings.maxConnections); + return crtClientBuilder.build(); + } + + static ClientOverrideConfiguration buildOverrideConfiguration( + final S3ClientSettings clientSettings, + ScheduledExecutorService clientExecutorService + ) { + RetryPolicy retryPolicy = SocketAccess.doPrivileged( + () -> RetryPolicy.builder() + .numRetries(clientSettings.maxRetries) + .throttlingBackoffStrategy( + clientSettings.throttleRetries ? BackoffStrategy.defaultThrottlingStrategy(RetryMode.STANDARD) : BackoffStrategy.none() + ) + .build() + ); + ClientOverrideConfiguration.Builder builder = ClientOverrideConfiguration.builder(); + if (clientExecutorService != null) { + builder = builder.scheduledExecutorService(clientExecutorService); + } + + return builder.retryPolicy(retryPolicy).apiCallAttemptTimeout(Duration.ofMillis(clientSettings.requestTimeoutMillis)).build(); + } + // pkg private for tests static AwsCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { final AwsCredentials basicCredentials = clientSettings.credentials; @@ -388,13 +486,16 @@ private static IrsaCredentials buildFromEnvironment(IrsaCredentials defaults) { } public synchronized void releaseCachedClients() { - // the clients will shutdown when they will not be used anymore - for (final AmazonAsyncS3Reference clientReference : clientsCache.values()) { - clientReference.decRef(); + // There will be 2 types of caches CRT and Netty + for (Map clientTypeCaches : s3HttpClientTypesClientsCache.values()) { + // the clients will shutdown when they will not be used anymore + for (final AmazonAsyncS3Reference clientReference : clientTypeCaches.values()) { + clientReference.decRef(); + } } // clear previously cached clients, they will be build lazily - clientsCache = emptyMap(); + s3HttpClientTypesClientsCache = emptyMap(); derivedClientSettings = emptyMap(); } @@ -453,7 +554,6 @@ public AwsCredentials resolveCredentials() { @Override public void close() { releaseCachedClients(); - } @Nullable diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java index 1c894203a805c..12bd9202a1838 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java @@ -124,6 +124,10 @@ class S3Repository extends MeteredBlobStoreRepository { static final Setting BUCKET_SETTING = Setting.simpleString("bucket"); static final String BUCKET_DEFAULT_ENCRYPTION_TYPE = "bucket_default"; + + public static final String NETTY_ASYNC_HTTP_CLIENT_TYPE = "netty"; + public static final String CRT_ASYNC_HTTP_CLIENT_TYPE = "crt"; + /** * The type of S3 Server Side Encryption to use. * Defaults to AES256. @@ -171,6 +175,15 @@ class S3Repository extends MeteredBlobStoreRepository { } }); + /** + * Type of Async client to be used for S3 Uploads. Defaults to crt. + */ + static final Setting S3_ASYNC_HTTP_CLIENT_TYPE = Setting.simpleString( + "s3_async_client_type", + CRT_ASYNC_HTTP_CLIENT_TYPE, + Setting.Property.NodeScope + ); + /** * Maximum size of files that can be uploaded using a single upload request. */ @@ -604,6 +617,15 @@ private void validateRepositoryMetadata(RepositoryMetadata newRepositoryMetadata validateStorageClass(STORAGE_CLASS_SETTING.get(settings)); validateCannedACL(CANNED_ACL_SETTING.get(settings)); + validateHttpClientType(S3_ASYNC_HTTP_CLIENT_TYPE.get(settings)); + } + + // package access for tests + void validateHttpClientType(String httpClientType) { + if (!(httpClientType.equalsIgnoreCase(NETTY_ASYNC_HTTP_CLIENT_TYPE) + || httpClientType.equalsIgnoreCase(CRT_ASYNC_HTTP_CLIENT_TYPE))) { + throw new BlobStoreException("Invalid http client type. `" + httpClientType + "`"); + } } private static void validateStorageClass(String storageClassStringValue) { diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java index 80aea8263e5a0..0f501eae27ad0 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java @@ -216,6 +216,7 @@ public Collection createComponents( int urgentEventLoopThreads = urgentPoolCount(clusterService.getSettings()); int priorityEventLoopThreads = priorityPoolCount(clusterService.getSettings()); int normalEventLoopThreads = normalPoolCount(clusterService.getSettings()); + this.urgentExecutorBuilder = new AsyncExecutorContainer( threadPool.executor(URGENT_FUTURE_COMPLETION), threadPool.executor(URGENT_STREAM_READER), @@ -371,7 +372,8 @@ public List> getSettings() { S3Repository.REDIRECT_LARGE_S3_UPLOAD, S3Repository.UPLOAD_RETRY_ENABLED, S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT, - S3Repository.PERMIT_BACKED_TRANSFER_ENABLED + S3Repository.PERMIT_BACKED_TRANSFER_ENABLED, + S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE ); } @@ -387,8 +389,14 @@ public void reload(Settings settings) { public void close() throws IOException { service.close(); s3AsyncService.close(); - urgentExecutorBuilder.getAsyncTransferEventLoopGroup().close(); - priorityExecutorBuilder.getAsyncTransferEventLoopGroup().close(); - normalExecutorBuilder.getAsyncTransferEventLoopGroup().close(); + if (urgentExecutorBuilder.getAsyncTransferEventLoopGroup() != null) { + urgentExecutorBuilder.getAsyncTransferEventLoopGroup().close(); + } + if (priorityExecutorBuilder.getAsyncTransferEventLoopGroup() != null) { + priorityExecutorBuilder.getAsyncTransferEventLoopGroup().close(); + } + if (normalExecutorBuilder.getAsyncTransferEventLoopGroup() != null) { + normalExecutorBuilder.getAsyncTransferEventLoopGroup().close(); + } } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/EventLoopThreadFilter.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/EventLoopThreadFilter.java new file mode 100644 index 0000000000000..2ed6b123cbb48 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/EventLoopThreadFilter.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import com.carrotsearch.randomizedtesting.ThreadFilter; + +/** + * While using CRT client we are seeing ThreadLeak for the AwsEventLoop threads. These are Native threads and are + * initialized one thread per core. We tried to specifically close the thread but couldn't get it terminated. + * We have opened a git-hub issue "..." for the same. + * Currently, we are using thread filter. + */ +public class EventLoopThreadFilter implements ThreadFilter { + + @Override + public boolean reject(Thread t) { + return t.getName().startsWith("AwsEventLoop"); + } +} diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java index de9ad46bb222d..cdddf19d142ff 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3AsyncServiceTests.java @@ -8,6 +8,10 @@ package org.opensearch.repositories.s3; +import software.amazon.awssdk.http.async.SdkAsyncHttpClient; +import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; + import org.opensearch.cli.SuppressForbidden; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.MockSecureSettings; @@ -20,6 +24,12 @@ import java.util.Map; import java.util.concurrent.Executors; +import io.netty.channel.nio.NioEventLoopGroup; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + public class S3AsyncServiceTests extends OpenSearchTestCase implements ConfigPathSupport { @Override @@ -32,9 +42,23 @@ public void setUp() throws Exception { public void testCachedClientsAreReleased() { final S3AsyncService s3AsyncService = new S3AsyncService(configPath()); - final Settings settings = Settings.builder().put("endpoint", "http://first").put("region", "us-east-2").build(); + final Settings settings = Settings.builder() + .put("endpoint", "http://first") + .put("region", "us-east-2") + .put(S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE.getKey(), S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE) + .build(); + + final Settings crtSettings = Settings.builder() + .put("endpoint", "http://first") + .put("region", "us-east-2") + .put(S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE.getKey(), S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE) + .build(); + final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); final RepositoryMetadata metadata2 = new RepositoryMetadata("second", "s3", settings); + + final RepositoryMetadata metadata3 = new RepositoryMetadata("second", "s3", crtSettings); + final RepositoryMetadata metadata4 = new RepositoryMetadata("second", "s3", crtSettings); final AsyncExecutorContainer asyncExecutorContainer = new AsyncExecutorContainer( Executors.newSingleThreadExecutor(), Executors.newSingleThreadExecutor(), @@ -46,6 +70,23 @@ public void testCachedClientsAreReleased() { final AmazonAsyncS3Reference reference = SocketAccess.doPrivileged( () -> s3AsyncService.client(metadata1, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer) ); + + final AmazonAsyncS3Reference reference2 = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata2, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer) + ); + + final AmazonAsyncS3Reference reference3 = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata3, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer) + ); + + final AmazonAsyncS3Reference reference4 = SocketAccess.doPrivileged( + () -> s3AsyncService.client(metadata4, asyncExecutorContainer, asyncExecutorContainer, asyncExecutorContainer) + ); + + assertSame(reference, reference2); + assertSame(reference3, reference4); + assertNotSame(reference, reference3); + reference.close(); s3AsyncService.close(); final AmazonAsyncS3Reference referenceReloaded = SocketAccess.doPrivileged( @@ -92,4 +133,74 @@ public void testCachedClientsWithCredentialsAreReleased() { final S3ClientSettings clientSettingsReloaded = s3AsyncService.settings(metadata1); assertNotSame(clientSettings, clientSettingsReloaded); } + + public void testBuildHttpClientWithNetty() { + final int port = randomIntBetween(10, 1080); + final String userName = randomAlphaOfLength(10); + final String password = randomAlphaOfLength(10); + final String proxyType = randomFrom("http", "https", "socks"); + final S3AsyncService s3AsyncService = new S3AsyncService(configPath()); + + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", userName); + secureSettings.setString("s3.client.default.proxy.password", password); + + final Settings settings = Settings.builder() + .put("endpoint", "http://first") + .put("region", "us-east-2") + .put("s3.client.default.proxy.type", proxyType) + .put("s3.client.default.proxy.host", randomFrom("127.0.0.10")) + .put("s3.client.default.proxy.port", randomFrom(port)) + .setSecureSettings(secureSettings) + .build(); + final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); + final S3ClientSettings clientSettings = s3AsyncService.settings(metadata1); + + AsyncTransferEventLoopGroup eventLoopGroup = mock(AsyncTransferEventLoopGroup.class); + when(eventLoopGroup.getEventLoopGroup()).thenReturn(mock(NioEventLoopGroup.class)); + + SdkAsyncHttpClient asyncClient = S3AsyncService.buildHttpClient( + clientSettings, + eventLoopGroup, + S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE + ); + assertNotNull(asyncClient); + assertTrue(asyncClient instanceof NettyNioAsyncHttpClient); + verify(eventLoopGroup).getEventLoopGroup(); + } + + public void testBuildHttpClientWithCRT() { + final int port = randomIntBetween(10, 1080); + final String userName = randomAlphaOfLength(10); + final String password = randomAlphaOfLength(10); + final String proxyType = randomFrom("http", "https", "socks"); + final S3AsyncService s3AsyncService = new S3AsyncService(configPath()); + + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", userName); + secureSettings.setString("s3.client.default.proxy.password", password); + + final Settings settings = Settings.builder() + .put("endpoint", "http://first") + .put("region", "us-east-2") + .put("s3.client.default.proxy.type", proxyType) + .put("s3.client.default.proxy.host", randomFrom("127.0.0.10")) + .put("s3.client.default.proxy.port", randomFrom(port)) + .setSecureSettings(secureSettings) + .build(); + + final RepositoryMetadata metadata1 = new RepositoryMetadata("first", "s3", settings); + final S3ClientSettings clientSettings = s3AsyncService.settings(metadata1); + + AsyncTransferEventLoopGroup eventLoopGroup = mock(AsyncTransferEventLoopGroup.class); + when(eventLoopGroup.getEventLoopGroup()).thenReturn(mock(NioEventLoopGroup.class)); + + SdkAsyncHttpClient asyncClient = S3AsyncService.buildHttpClient( + clientSettings, + eventLoopGroup, + S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE + ); + assertNotNull(asyncClient); + assertTrue(asyncClient instanceof AwsCrtAsyncHttpClient); + } } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java index 4193609ac520d..786a56d973551 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -31,6 +31,8 @@ package org.opensearch.repositories.s3; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.io.SdkDigestInputStream; import software.amazon.awssdk.utils.internal.Base16; @@ -118,6 +120,7 @@ * This class tests how a {@link S3BlobContainer} and its underlying AWS S3 client are retrying requests when reading or writing blobs. */ @SuppressForbidden(reason = "use a http server") +@ThreadLeakFilters(filters = EventLoopThreadFilter.class) public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTestCase implements ConfigPathSupport { private S3Service service; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java index c0ee9cb6d980f..799cbe90103e5 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java @@ -74,6 +74,7 @@ public void testGetExecutorBuilders() throws IOException { + "] is deprecated" ); } + assertTrue(plugin.getSettings().contains(S3Repository.S3_ASYNC_HTTP_CLIENT_TYPE)); } finally { if (threadPool != null) { terminate(threadPool); diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java index f8e9903bb3577..49c6a31e32816 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java @@ -35,6 +35,7 @@ import software.amazon.awssdk.services.s3.S3Client; import org.opensearch.cluster.metadata.RepositoryMetadata; +import org.opensearch.common.blobstore.BlobStoreException; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -157,6 +158,23 @@ public void testRestrictedSettingsDefault() { } } + public void testValidateHttpLClientType_Valid_Values() { + final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); + try (S3Repository s3Repo = createS3Repo(metadata)) { + // Don't expect any Exception + s3Repo.validateHttpClientType(S3Repository.CRT_ASYNC_HTTP_CLIENT_TYPE); + s3Repo.validateHttpClientType(S3Repository.NETTY_ASYNC_HTTP_CLIENT_TYPE); + } + } + + public void testValidateHttpLClientType_Invalid_Values() { + final RepositoryMetadata metadata = new RepositoryMetadata("dummy-repo", "mock", Settings.EMPTY); + try (S3Repository s3Repo = createS3Repo(metadata)) { + // Don't expect any Exception + assertThrows(BlobStoreException.class, () -> s3Repo.validateHttpClientType(randomAlphaOfLength(4))); + } + } + private S3Repository createS3Repo(RepositoryMetadata metadata) { return new S3Repository( metadata, diff --git a/qa/evil-tests/src/test/java/org/opensearch/bootstrap/SystemCallFilterTests.java b/qa/evil-tests/src/test/java/org/opensearch/bootstrap/SystemCallFilterTests.java index 99c9ee7e96d01..8e77cbc979dfb 100644 --- a/qa/evil-tests/src/test/java/org/opensearch/bootstrap/SystemCallFilterTests.java +++ b/qa/evil-tests/src/test/java/org/opensearch/bootstrap/SystemCallFilterTests.java @@ -39,7 +39,7 @@ public class SystemCallFilterTests extends OpenSearchTestCase { /** command to try to run in tests */ - static final String EXECUTABLE = Constants.WINDOWS ? "calc" : "ls"; + static final String[] EXECUTABLE = new String[] { Constants.WINDOWS ? "calc" : "ls" }; @SuppressWarnings("removal") @Override diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 455b348e7433b..f49927cbae12d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -509,3 +509,104 @@ setup: - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" } - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._seq_no: 0 } - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._primary_term: 1 } + +--- +"Test field collapsing with sort": + - skip: + version: " - 3.2.99" + reason: Fixed in 3.3.0 + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + sort_field: { type: integer } + collapse_field: { type: integer } + marker: {type: keyword} + + - do: + index: + index: test_1 + refresh: true + id: 1 + body: { sort_field: 1, collapse_field: 1, marker: "doc1" } + - do: + index: + index: test_1 + refresh: true + id: 2 + body: { sort_field: 1, collapse_field: 2, marker: "doc2" } + - do: + index: + index: test_1 + refresh: true + id: 3 + body: { sort_field: 1, collapse_field: 2, marker: "doc3" } + + - do: + search: + index: test_1 + size: 2 + body: + collapse: { field: collapse_field } + sort: [{ sort_field: desc }] + - match: { hits.total.value: 3 } + - length: { hits.hits: 2 } + - match: { hits.hits.0._id: '1' } + - match: { hits.hits.0._source.marker: 'doc1' } + - match: { hits.hits.1._id: '2' } + - match: { hits.hits.1._source.marker: 'doc2' } + +--- +"Test field collapsing with sort when concurrent segment search enabled": + - skip: + version: " - 3.2.99" + reason: Fixed in 3.3.0 + - do: + indices.create: + index: test_1 + body: + mappings: + properties: + sort_field: { type: integer } + collapse_field: { type: integer } + marker: {type: keyword} + + - do: + index: + index: test_1 + refresh: true + id: 1 + body: { sort_field: 1, collapse_field: 1, marker: "doc1" } + - do: + index: + index: test_1 + refresh: true + id: 2 + body: { sort_field: 1, collapse_field: 2, marker: "doc2" } + - do: + index: + index: test_1 + refresh: true + id: 3 + body: { sort_field: 1, collapse_field: 2, marker: "doc3" } + - do: + indices.put_settings: + index: test_1 + body: + index.search.concurrent_segment_search.mode: 'all' + + - do: + search: + index: test_1 + size: 2 + body: + collapse: { field: collapse_field } + sort: [{ sort_field: desc }] + - match: { hits.total.value: 3 } + - length: { hits.hits: 2 } + - match: { hits.hits.0._id: '1' } + - match: { hits.hits.0._source.marker: 'doc1' } + - match: { hits.hits.1._id: '2' } + - match: { hits.hits.1._source.marker: 'doc2' } diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java index c81d491719e4b..e3ba967a28154 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/ReloadSecureSettingsIT.java @@ -50,7 +50,6 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; -import java.security.AccessControlException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -449,20 +448,9 @@ public void onFailure(Exception e) { } } - @SuppressWarnings("removal") private SecureSettings writeEmptyKeystore(Environment environment, char[] password) throws Exception { final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create(); - try { - keyStoreWrapper.save(environment.configDir(), password); - } catch (final AccessControlException e) { - if (e.getPermission() instanceof RuntimePermission && e.getPermission().getName().equals("accessUserInformation")) { - // this is expected: the save method is extra diligent and wants to make sure - // the keystore is readable, not relying on umask and whatnot. It's ok, we don't - // care about this in tests. - } else { - throw e; - } - } + keyStoreWrapper.save(environment.configDir(), password); return keyStoreWrapper; } diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 2ce50c8b5a768..8cd6fb7ed5aa6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -77,7 +77,6 @@ import org.opensearch.index.engine.MergedSegmentWarmerFactory; import org.opensearch.index.engine.NoOpEngine; import org.opensearch.index.flush.FlushStats; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.seqno.SequenceNumbers; @@ -477,7 +476,7 @@ public void testMaybeRollTranslogGeneration() throws Exception { .put("index.number_of_shards", 1) .put("index.translog.generation_threshold_size", generationThreshold + "b") .build(); - createIndex("test", settings, MapperService.SINGLE_MAPPING_NAME); + createIndexWithSimpleMappings("test", settings); ensureGreen("test"); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexService test = indicesService.indexService(resolveIndex("test")); @@ -813,7 +812,7 @@ public void testShardChangesWithDefaultDocType() throws Exception { .put("index.translog.flush_threshold_size", "512mb") // do not flush .put("index.soft_deletes.enabled", true) .build(); - IndexService indexService = createIndex("index", settings, "user_doc", "title", "type=keyword"); + IndexService indexService = createIndexWithSimpleMappings("index", settings, "title", "type=keyword"); int numOps = between(1, 10); for (int i = 0; i < numOps; i++) { if (randomBoolean()) { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java index b8d1d3cad77b4..c6ca4d36a86d7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/FiltersAggsRewriteIT.java @@ -51,7 +51,7 @@ public class FiltersAggsRewriteIT extends OpenSearchSingleNodeTestCase { public void testWrapperQueryIsRewritten() throws IOException { - createIndex("test", Settings.EMPTY, "test", "title", "type=text"); + createIndexWithSimpleMappings("test", Settings.EMPTY, "title", "type=text"); client().prepareIndex("test").setId("1").setSource("title", "foo bar baz").get(); client().prepareIndex("test").setId("2").setSource("title", "foo foo foo").get(); client().prepareIndex("test").setId("3").setSource("title", "bar baz bax").get(); diff --git a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java index 4ab1eee4e089f..e453d8690d9c6 100644 --- a/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java +++ b/server/src/main/java/org/apache/lucene/search/grouping/CollapseTopFieldDocs.java @@ -43,6 +43,7 @@ import org.opensearch.core.common.util.CollectionUtils; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +56,14 @@ public final class CollapseTopFieldDocs extends TopFieldDocs { public final String field; /** The collapse value for each top doc */ public final Object[] collapseValues; + /** Internal comparator with shardIndex */ + private static final Comparator SHARD_INDEX_TIE_BREAKER = Comparator.comparingInt(d -> d.shardIndex); + + /** Internal comparator with docID */ + private static final Comparator DOC_ID_TIE_BREAKER = Comparator.comparingInt(d -> d.doc); + + /** Default comparator */ + private static final Comparator DEFAULT_TIE_BREAKER = SHARD_INDEX_TIE_BREAKER.thenComparing(DOC_ID_TIE_BREAKER); public CollapseTopFieldDocs(String field, TotalHits totalHits, ScoreDoc[] scoreDocs, SortField[] sortFields, Object[] values) { super(totalHits, scoreDocs, sortFields); @@ -67,55 +76,35 @@ private static final class ShardRef { // Which shard (index into shardHits[]): final int shardIndex; - // True if we should use the incoming ScoreDoc.shardIndex for sort order - final boolean useScoreDocIndex; - // Which hit within the shard: int hitIndex; - ShardRef(int shardIndex, boolean useScoreDocIndex) { + ShardRef(int shardIndex) { this.shardIndex = shardIndex; - this.useScoreDocIndex = useScoreDocIndex; } @Override public String toString() { return "ShardRef(shardIndex=" + shardIndex + " hitIndex=" + hitIndex + ")"; } - - int getShardIndex(ScoreDoc scoreDoc) { - if (useScoreDocIndex) { - if (scoreDoc.shardIndex == -1) { - throw new IllegalArgumentException( - "setShardIndex is false but TopDocs[" + shardIndex + "].scoreDocs[" + hitIndex + "] is not set" - ); - } - return scoreDoc.shardIndex; - } else { - // NOTE: we don't assert that shardIndex is -1 here, because caller could in fact have set it but asked us to ignore it now - return shardIndex; - } - } } /** - * if we need to tie-break since score / sort value are the same we first compare shard index (lower shard wins) - * and then iff shard index is the same we use the hit index. + * Use the default tie breaker. If tie breaker returns 0 signifying equal values, we use hit + * indices to tie break intra shard ties */ static boolean tieBreakLessThan(ShardRef first, ScoreDoc firstDoc, ShardRef second, ScoreDoc secondDoc) { - final int firstShardIndex = first.getShardIndex(firstDoc); - final int secondShardIndex = second.getShardIndex(secondDoc); - // Tie break: earlier shard wins - if (firstShardIndex < secondShardIndex) { - return true; - } else if (firstShardIndex > secondShardIndex) { - return false; - } else { + int value = DEFAULT_TIE_BREAKER.compare(firstDoc, secondDoc); + + if (value == 0) { + // Equal Values // Tie break in same shard: resolve however the // shard had resolved it: assert first.hitIndex != second.hitIndex; return first.hitIndex < second.hitIndex; } + + return value < 0; } private static class MergeSortQueue extends PriorityQueue { @@ -173,8 +162,10 @@ public boolean lessThan(ShardRef first, ShardRef second) { /** * Returns a new CollapseTopDocs, containing topN collapsed results across * the provided CollapseTopDocs, sorting by score. Each {@link CollapseTopFieldDocs} instance must be sorted. + * docIDs are expected to be in consistent pattern i.e. either all ScoreDocs have their shardIndex set, + * or all have them as -1 (signifying that all hits belong to same shard) **/ - public static CollapseTopFieldDocs merge(Sort sort, int start, int size, CollapseTopFieldDocs[] shardHits, boolean setShardIndex) { + public static CollapseTopFieldDocs merge(Sort sort, int start, int size, CollapseTopFieldDocs[] shardHits) { String collapseField = shardHits[0].field; for (int i = 1; i < shardHits.length; i++) { if (collapseField.equals(shardHits[i].field) == false) { @@ -200,12 +191,13 @@ public static CollapseTopFieldDocs merge(Sort sort, int start, int size, Collaps } if (CollectionUtils.isEmpty(shard.scoreDocs) == false) { availHitCount += shard.scoreDocs.length; - queue.add(new ShardRef(shardIDX, setShardIndex == false)); + queue.add(new ShardRef(shardIDX)); } } final ScoreDoc[] hits; final Object[] values; + boolean unsetShardIndex = false; if (availHitCount <= start) { hits = new ScoreDoc[0]; values = new Object[0]; @@ -223,6 +215,15 @@ public static CollapseTopFieldDocs merge(Sort sort, int start, int size, Collaps ShardRef ref = queue.top(); final ScoreDoc hit = shardHits[ref.shardIndex].scoreDocs[ref.hitIndex]; final Object collapseValue = shardHits[ref.shardIndex].collapseValues[ref.hitIndex++]; + // Irrespective of whether we use shard indices for tie breaking or not, we check for + // consistent order in shard indices to defend against potential bugs + if (hitUpto > 0) { + if (unsetShardIndex != (hit.shardIndex == -1)) { + throw new IllegalArgumentException("Inconsistent order of shard indices"); + } + } + unsetShardIndex |= hit.shardIndex == -1; + if (seen.contains(collapseValue)) { if (ref.hitIndex < shardHits[ref.shardIndex].scoreDocs.length) { queue.updateTop(); @@ -232,9 +233,6 @@ public static CollapseTopFieldDocs merge(Sort sort, int start, int size, Collaps continue; } seen.add(collapseValue); - if (setShardIndex) { - hit.shardIndex = ref.shardIndex; - } if (hitUpto >= start) { hitList.add(hit); collapseList.add(collapseValue); diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java index 503252a814401..40a2805563369 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseController.java @@ -233,7 +233,7 @@ static TopDocs mergeTopDocs(Collection results, int topN, int from) { } else if (topDocs instanceof CollapseTopFieldDocs) { final CollapseTopFieldDocs[] shardTopDocs = results.toArray(new CollapseTopFieldDocs[numShards]); final Sort sort = createSort(shardTopDocs); - mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs, false); + mergedTopDocs = CollapseTopFieldDocs.merge(sort, from, topN, shardTopDocs); } else if (topDocs instanceof TopFieldDocs) { final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[numShards]); final Sort sort = createSort(shardTopDocs); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index ad7b8e78744ef..b0a5f4c83e2b4 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -371,10 +371,10 @@ public Iterator> settings() { ); public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled"; + public static final String SETTING_REMOTE_STORE_SSE_ENABLED = "index.remote_store.sse.enabled"; public static final String SETTING_INDEX_APPEND_ONLY_ENABLED = "index.append_only.enabled"; public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository"; - public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository"; /** @@ -414,6 +414,38 @@ public Iterator> settings() { Property.Dynamic ); + /** + * Used to specify if the index data should be persisted in the remote store. + */ + public static final Setting INDEX_REMOTE_STORE_SSE_ENABLED_SETTING = Setting.boolSetting( + SETTING_REMOTE_STORE_SSE_ENABLED, + false, + new Setting.Validator<>() { + + @Override + public void validate(final Boolean value) {} + + @Override + public void validate(final Boolean value, final Map, Object> settings) { + final Boolean isRemoteStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING); + if (!isRemoteStoreEnabled && value) { + throw new IllegalArgumentException( + "Server Side Encryption can be enabled when " + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + " is enabled. " + ); + } + } + + @Override + public Iterator> settings() { + final List> settings = List.of(INDEX_REMOTE_STORE_ENABLED_SETTING); + return settings.iterator(); + } + }, + Property.IndexScope, + Property.PrivateIndex, + Property.Dynamic + ); + /** * Used to specify if the index data should be persisted in the remote store. */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index a889091140d12..2ec7bb98c9777 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1056,7 +1056,7 @@ static Settings aggregateIndexSettings( indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings, combinedTemplateSettings, clusterSettings); - updateRemoteStoreSettings(indexSettingsBuilder, currentState, clusterSettings, settings, request.index()); + updateRemoteStoreSettings(indexSettingsBuilder, currentState, clusterSettings, settings, request.index(), false); if (sourceMetadata != null) { assert request.resizeType() != null; @@ -1149,6 +1149,25 @@ public static void updateReplicationStrategy( settingsBuilder.put(SETTING_REPLICATION_TYPE, indexReplicationType); } + public static void updateRemoteStoreSettings( + Settings.Builder settingsBuilder, + ClusterState clusterState, + ClusterSettings clusterSettings, + Settings nodeSettings, + String indexName, + IndexMetadata indexMetadata + ) { + if ((isRemoteDataAttributePresent(nodeSettings) + && clusterSettings.get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(RemoteStoreNodeService.CompatibilityMode.STRICT)) + || isMigratingToRemoteStore(clusterSettings)) { + boolean sseEnabledIndex = IndexMetadata.INDEX_REMOTE_STORE_SSE_ENABLED_SETTING.get(indexMetadata.getSettings()); + if (sseEnabledIndex) { + settingsBuilder.put(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, true); + } + updateRemoteStoreSettings(settingsBuilder, clusterState, clusterSettings, nodeSettings, indexName, true); + } + } + /** * Updates index settings to enable remote store by default based on node attributes * @param settingsBuilder index settings builder to be updated with relevant settings @@ -1162,11 +1181,13 @@ public static void updateRemoteStoreSettings( ClusterState clusterState, ClusterSettings clusterSettings, Settings nodeSettings, - String indexName + String indexName, + boolean isRestoreFromSnapshot ) { if ((isRemoteDataAttributePresent(nodeSettings) && clusterSettings.get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(RemoteStoreNodeService.CompatibilityMode.STRICT)) || isMigratingToRemoteStore(clusterSettings)) { + String segmentRepo, translogRepo; Optional remoteNode = clusterState.nodes() @@ -1176,22 +1197,25 @@ public static void updateRemoteStoreSettings( .filter(DiscoveryNode::isRemoteStoreNode) .findFirst(); + if (!isRestoreFromSnapshot && RemoteStoreNodeAttribute.isRemoteStoreServerSideEncryptionEnabled()) { + settingsBuilder.put(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, true); + } + if (remoteNode.isPresent()) { - translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes()); - segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes()); - if (segmentRepo != null) { - settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true).put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo); - if (translogRepo != null) { - settingsBuilder.put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepo); - } else if (isMigratingToRemoteStore(clusterSettings)) { - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors( - Collections.singletonList( - "Cluster is migrating to remote store but remote translog is not configured, failing index creation" - ) - ); - throw new IndexCreationException(indexName, validationException); - } + Map indexSettings = settingsBuilder.keys() + .stream() + .collect(Collectors.toMap(key -> key, settingsBuilder::get)); + + Settings.Builder currentSettingsBuilder = Settings.builder(); + Settings currentIndexSettings = currentSettingsBuilder.loadFromMap(indexSettings).build(); + + translogRepo = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(currentIndexSettings); + segmentRepo = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(currentIndexSettings); + + if (segmentRepo != null && translogRepo != null) { + settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) + .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo) + .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepo); } else { ValidationException validationException = new ValidationException(); validationException.addValidationErrors( diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 8aec7386fcf81..3f954cd9f9c37 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -804,6 +804,8 @@ public void apply(Settings value, Settings current, Settings previous) { BlobStoreRepository.SNAPSHOT_REPOSITORY_DATA_CACHE_THRESHOLD, SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING, + SearchService.QUERY_REWRITING_ENABLED_SETTING, + SearchService.QUERY_REWRITING_TERMS_THRESHOLD_SETTING, // Composite index settings CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index a8b76a3f2ac01..c3887de524aae 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -241,6 +241,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // Settings for remote store enablement IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, + IndexMetadata.INDEX_REMOTE_STORE_SSE_ENABLED_SETTING, IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING, diff --git a/server/src/main/java/org/opensearch/common/util/LocaleUtils.java b/server/src/main/java/org/opensearch/common/util/LocaleUtils.java index c684b1b2d781f..05c5cd89705b1 100644 --- a/server/src/main/java/org/opensearch/common/util/LocaleUtils.java +++ b/server/src/main/java/org/opensearch/common/util/LocaleUtils.java @@ -37,7 +37,7 @@ import java.util.MissingResourceException; /** - * Utilities for for dealing with {@link Locale} objects + * Utilities for dealing with {@link Locale} objects * * @opensearch.internal */ @@ -90,23 +90,18 @@ public static Locale parse(String localeStr) { } private static Locale parseParts(String[] parts) { - switch (parts.length) { - case 3: - // lang, country, variant - return new Locale(parts[0], parts[1], parts[2]); - case 2: - // lang, country - return new Locale(parts[0], parts[1]); - case 1: + return switch (parts.length) { + case 3 -> new Locale.Builder().setLanguage(parts[0]).setRegion(parts[1]).setVariant(parts[2]).build(); + case 2 -> new Locale.Builder().setLanguage(parts[0]).setRegion(parts[1]).build(); + case 1 -> { if ("ROOT".equalsIgnoreCase(parts[0])) { - return Locale.ROOT; + yield Locale.ROOT; } - // lang - return new Locale(parts[0]); - default: - throw new IllegalArgumentException( - "Locales can have at most 3 parts but got " + parts.length + ": " + Arrays.asList(parts) - ); - } + yield new Locale.Builder().setLanguage(parts[0]).build(); + } + default -> throw new IllegalArgumentException( + "Locales can have at most 3 parts but got " + parts.length + ": " + Arrays.asList(parts) + ); + }; } } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 22441df923bf8..02f4f35705ba6 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -697,7 +697,7 @@ public synchronized IndexShard createShard( } remoteDirectory = ((RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory).newDirectory( - RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getSettings()), this.indexSettings.getUUID(), shardId, this.indexSettings.getRemoteStorePathStrategy(), diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index a10f9d8152a79..c68c0c73ce7ca 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -950,6 +950,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { */ private final boolean isCompositeIndex; + private boolean isRemoteStoreSSEnabled; + /** * Denotes whether search via star tree index is enabled for this index */ @@ -1035,12 +1037,15 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); + isRemoteStoreSSEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, false); isWarmIndex = settings.getAsBoolean(IndexModule.IS_WARM_INDEX_SETTING.getKey(), false); - remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); + remoteStoreRepository = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(indexMetadata.getSettings()); + remoteStoreTranslogRepository = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexMetadata.getSettings()); + remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings); - remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY); + this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings); String rawPrefix = IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(settings); // Only set the prefix if it's explicitly set and not empty @@ -1239,6 +1244,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti ); scopedSettings.addSettingsUpdateConsumer(ALLOW_DERIVED_FIELDS, this::setAllowDerivedField); scopedSettings.addSettingsUpdateConsumer(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING, this::setRemoteStoreEnabled); + + scopedSettings.addSettingsUpdateConsumer(IndexMetadata.INDEX_REMOTE_STORE_SSE_ENABLED_SETTING, this::setRemoteStoreSseEnabled); + scopedSettings.addSettingsUpdateConsumer( IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, this::setRemoteStoreRepository @@ -1427,6 +1435,13 @@ public boolean isRemoteStoreEnabled() { return isRemoteStoreEnabled; } + /** + * Returns if remote store is enabled for this index. + */ + public boolean isRemoteStoreSSEnabled() { + return isRemoteStoreSSEnabled; + } + public boolean isAssignedOnRemoteNode() { return assignedOnRemoteNode; } @@ -2137,6 +2152,10 @@ public void setRemoteStoreEnabled(boolean isRemoteStoreEnabled) { this.isRemoteStoreEnabled = isRemoteStoreEnabled; } + public void setRemoteStoreSseEnabled(boolean sseEnabled) { + this.isRemoteStoreSSEnabled = sseEnabled; + } + public void setRemoteStoreRepository(String remoteStoreRepository) { this.remoteStoreRepository = remoteStoreRepository; } diff --git a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java index 39a3a73c9529c..aaa2c9c029974 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FieldMapper.java @@ -642,7 +642,7 @@ protected void canDeriveSourceInternal() { /** * Validates if doc values is enabled for a field or not */ - void checkDocValuesForDerivedSource() { + protected void checkDocValuesForDerivedSource() { if (!mappedFieldType.hasDocValues()) { throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with doc values disabled"); } @@ -651,7 +651,7 @@ void checkDocValuesForDerivedSource() { /** * Validates if stored field is enabled for a field or not */ - void checkStoredForDerivedSource() { + protected void checkStoredForDerivedSource() { if (!mappedFieldType.isStored()) { throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with store disabled"); } @@ -660,7 +660,7 @@ void checkStoredForDerivedSource() { /** * Validates if doc_values or stored field is enabled for a field or not */ - void checkStoredAndDocValuesForDerivedSource() { + protected void checkStoredAndDocValuesForDerivedSource() { if (!mappedFieldType.isStored() && !mappedFieldType.hasDocValues()) { throw new UnsupportedOperationException("Unable to derive source for [" + name() + "] with stored and " + "docValues disabled"); } diff --git a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java index 46c5a40457ce7..a2f71a7064903 100644 --- a/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/BoolQueryBuilder.java @@ -49,8 +49,6 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.NumberFieldMapper; import java.io.IOException; import java.util.ArrayList; @@ -402,9 +400,6 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return any.get(); } - changed |= rewriteMustNotRangeClausesToShould(newBuilder, queryRewriteContext); - changed |= rewriteMustClausesToFilter(newBuilder, queryRewriteContext); - if (changed) { newBuilder.adjustPureNegative = adjustPureNegative; if (minimumShouldMatch != null) { @@ -559,53 +554,4 @@ private boolean checkAllDocsHaveOneValue(List contexts, Strin } return true; } - - private boolean rewriteMustClausesToFilter(BoolQueryBuilder newBuilder, QueryRewriteContext queryRewriteContext) { - // If we have must clauses which return the same score for all matching documents, like numeric term queries or ranges, - // moving them from must clauses to filter clauses improves performance in some cases. - // This works because it can let Lucene use MaxScoreCache to skip non-competitive docs. - boolean changed = false; - Set mustClausesToMove = new HashSet<>(); - - QueryShardContext shardContext; - if (queryRewriteContext == null) { - shardContext = null; - } else { - shardContext = queryRewriteContext.convertToShardContext(); // can still be null - } - - for (QueryBuilder clause : mustClauses) { - if (isClauseIrrelevantToScoring(clause, shardContext)) { - mustClausesToMove.add(clause); - changed = true; - } - } - - newBuilder.mustClauses.removeAll(mustClausesToMove); - newBuilder.filterClauses.addAll(mustClausesToMove); - return changed; - } - - private boolean isClauseIrrelevantToScoring(QueryBuilder clause, QueryShardContext context) { - // This is an incomplete list of clauses this might apply for; it can be expanded in future. - - // If a clause is purely numeric, for example a date range, its score is unimportant as - // it'll be the same for all returned docs - if (clause instanceof RangeQueryBuilder) return true; - if (clause instanceof GeoBoundingBoxQueryBuilder) return true; - - // Further optimizations depend on knowing whether the field is numeric. - // QueryBuilder.doRewrite() is called several times in the search flow, and the shard context telling us this - // is only available the last time, when it's called from SearchService.executeQueryPhase(). - // Skip moving these clauses if we don't have the shard context. - if (context == null) return false; - if (!(clause instanceof WithFieldName wfn)) return false; - MappedFieldType fieldType = context.fieldMapper(wfn.fieldName()); - if (!(fieldType instanceof NumberFieldMapper.NumberFieldType)) return false; - - if (clause instanceof MatchQueryBuilder) return true; - if (clause instanceof TermQueryBuilder) return true; - if (clause instanceof TermsQueryBuilder) return true; - return false; - } } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java index 18b6d6184d1b0..7ea9448d9fd7d 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java @@ -81,6 +81,9 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener { private BlobStoreRepository translogRepository; private BlobStoreRepository segmentRepository; + private BlobStoreRepository translogSSERepository; + private BlobStoreRepository segmentSSERepository; + public RemoteIndexPathUploader( ThreadPool threadPool, Settings settings, @@ -174,11 +177,24 @@ private void writeIndexPathAsync(IndexMetadata idxMD, CountDownLatch latch, List if (isTranslogSegmentRepoSame) { // If the repositories are same, then we need to upload a single file containing paths for both translog and segments. writePathToRemoteStore(idxMD, translogRepository, latch, exceptionList, COMBINED_PATH); + + if (translogSSERepository != null) { + writePathToRemoteStore(idxMD, translogSSERepository, latch, exceptionList, COMBINED_PATH); + } + } else { // If the repositories are different, then we need to upload one file per segment and translog containing their individual // paths. writePathToRemoteStore(idxMD, translogRepository, latch, exceptionList, TRANSLOG_PATH); writePathToRemoteStore(idxMD, segmentRepository, latch, exceptionList, SEGMENT_PATH); + + if (translogSSERepository != null) { + writePathToRemoteStore(idxMD, translogSSERepository, latch, exceptionList, TRANSLOG_PATH); + } + + if (segmentSSERepository != null) { + writePathToRemoteStore(idxMD, segmentSSERepository, latch, exceptionList, SEGMENT_PATH); + } } } @@ -234,15 +250,22 @@ public void start() { return; } - translogRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings)); - segmentRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings)); + translogRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(false)); + segmentRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(false)); + + if (RemoteStoreNodeAttribute.isRemoteStoreServerSideEncryptionEnabled()) { + translogSSERepository = (BlobStoreRepository) validateAndGetRepository( + RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(true) + ); + segmentSSERepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(true)); + } } private boolean isTranslogSegmentRepoSame() { // TODO - The current comparison checks the repository name. But it is also possible that the repository are same // by attributes, but different by name. We need to handle this. - String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings); - String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings); + String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(false); + String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(false); return Objects.equals(translogRepoName, segmentRepoName); } diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java index 1f9ffca4460b7..8440e774c085a 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java @@ -30,7 +30,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStoreCustomMetadataDuringMigration; -import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName; /** * Utils for checking and mutating cluster state during remote migration @@ -72,13 +71,12 @@ public void maybeAddRemoteIndexSettings(IndexMetadata.Builder indexMetadataBuild "Index {} does not have remote store based index settings but all primary shards and STARTED replica shards have moved to remote enabled nodes. Applying remote store settings to the index", index ); - Map remoteRepoNames = getRemoteStoreRepoName(discoveryNodes); - String segmentRepoName = RemoteStoreNodeAttribute.getSegmentRepoName(remoteRepoNames); - String tlogRepoName = RemoteStoreNodeAttribute.getTranslogRepoName(remoteRepoNames); + String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(currentIndexSettings); + String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(currentIndexSettings); - assert Objects.nonNull(segmentRepoName) && Objects.nonNull(tlogRepoName) : "Remote repo names cannot be null"; + assert Objects.nonNull(segmentRepoName) && Objects.nonNull(translogRepoName) : "Remote repo names cannot be null"; Settings.Builder indexSettingsBuilder = Settings.builder().put(currentIndexSettings); - updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, tlogRepoName); + updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, translogRepoName); indexMetadataBuilder.settings(indexSettingsBuilder); indexMetadataBuilder.settingsVersion(1 + indexMetadata.getVersion()); } else { diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java index e8a0dda5a699e..fe6d52115ed5b 100644 --- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java +++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreCustomMetadataResolver.java @@ -14,6 +14,7 @@ import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm; import org.opensearch.index.remote.RemoteStoreEnums.PathType; import org.opensearch.indices.RemoteStoreSettings; +import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.Repository; import org.opensearch.repositories.RepositoryMissingException; @@ -21,8 +22,6 @@ import java.util.function.Supplier; -import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo; - /** * Determines the {@link RemoteStorePathStrategy} at the time of index metadata creation. * @@ -61,7 +60,7 @@ public RemoteStorePathStrategy getPathStrategy() { public boolean isTranslogMetadataEnabled() { Repository repository; try { - repository = repositoriesServiceSupplier.get().repository(getRemoteStoreTranslogRepo(settings)); + repository = repositoriesServiceSupplier.get().repository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(false)); } catch (RepositoryMissingException ex) { throw new IllegalArgumentException("Repository should be created before creating index with remote_store enabled setting", ex); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 609a6290d36ce..aac1d81cc0117 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -5311,6 +5311,7 @@ public void syncTranslogFilesFromGivenRemoteTranslog( boolean isTranslogMetadataEnabled, long timestamp ) throws IOException { + boolean sseEnabled = indexSettings.isRemoteStoreSSEnabled(); RemoteFsTranslog.download( repository, shardId, diff --git a/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java index ad42b6d677b41..bcbf50b1fe00f 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteStoreFileDownloader.java @@ -114,6 +114,15 @@ private void downloadInternal( Runnable onFileCompletion, ActionListener listener ) { + try { + logger.info("[pranikum]: Going to download segment file. Stack trace is below "); + if (listener != null) { + logger.info("Listener class is " + listener.getClass().getName()); + } + throw new Exception(); + } catch (Exception e) { + e.printStackTrace(); + } final Queue queue = new ConcurrentLinkedQueue<>(toDownloadSegments); // Choose the minimum of: // - number of files to download diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java index 6e2a7db0cfeb0..b17f217aa3bef 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java @@ -41,7 +41,8 @@ public Translog newTranslog( globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -64,7 +65,8 @@ public Translog newTranslog( globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer, - translogOperationHelper + translogOperationHelper, + null ); } } diff --git a/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java b/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java index 6b45ccb867520..1c4b89be40fa1 100644 --- a/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/LocalTranslog.java @@ -50,7 +50,8 @@ public LocalTranslog( final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier, final LongConsumer persistedSequenceNumberConsumer, - final TranslogOperationHelper translogOperationHelper + final TranslogOperationHelper translogOperationHelper, + final ChannelFactory channelFactory ) throws IOException { super( config, @@ -59,7 +60,8 @@ public LocalTranslog( globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer, - translogOperationHelper + translogOperationHelper, + channelFactory ); try { final Checkpoint checkpoint = readCheckpoint(location); @@ -113,6 +115,30 @@ public LocalTranslog( } } + /** + * Secondary constructor that does not accept ChannelFactory parameter. + */ + public LocalTranslog( + final TranslogConfig config, + final String translogUUID, + TranslogDeletionPolicy deletionPolicy, + final LongSupplier globalCheckpointSupplier, + final LongSupplier primaryTermSupplier, + final LongConsumer persistedSequenceNumberConsumer, + final TranslogOperationHelper translogOperationHelper + ) throws IOException { + this( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + translogOperationHelper, + null + ); + } + /** * Ensures that the given location has be synced / written to the underlying storage. * diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java index 63433a489cbab..d022e38d42203 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java @@ -91,7 +91,6 @@ public Translog newTranslog( BooleanSupplier startedPrimarySupplier, TranslogOperationHelper translogOperationHelper ) throws IOException { - assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository"; BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository); if (RemoteStoreSettings.isPinnedTimestampsEnabled()) { @@ -122,7 +121,8 @@ public Translog newTranslog( startedPrimarySupplier, remoteTranslogTransferTracker, remoteStoreSettings, - translogOperationHelper + translogOperationHelper, + null ); } } diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java index 920d26356bbb3..7fd915ba2c297 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslog.java @@ -90,7 +90,8 @@ public RemoteFsTimestampAwareTranslog( startedPrimarySupplier, remoteTranslogTransferTracker, remoteStoreSettings, - translogOperationHelper + translogOperationHelper, + null ); logger = Loggers.getLogger(getClass(), shardId); this.metadataFilePinnedTimestampMap = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java index cda5085d750d0..bbe8b739e2da4 100644 --- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java +++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java @@ -108,7 +108,8 @@ public RemoteFsTranslog( BooleanSupplier startedPrimarySupplier, RemoteTranslogTransferTracker remoteTranslogTransferTracker, RemoteStoreSettings remoteStoreSettings, - TranslogOperationHelper translogOperationHelper + TranslogOperationHelper translogOperationHelper, + ChannelFactory channelFactory ) throws IOException { super( config, @@ -117,7 +118,8 @@ public RemoteFsTranslog( globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer, - translogOperationHelper + translogOperationHelper, + channelFactory ); logger = Loggers.getLogger(getClass(), shardId); this.startedPrimarySupplier = startedPrimarySupplier; diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 1bd0120586ed3..7f949f85a64ab 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -154,6 +154,7 @@ public abstract class Translog extends AbstractIndexShardComponent implements In protected final TranslogDeletionPolicy deletionPolicy; protected final LongConsumer persistedSequenceNumberConsumer; protected final TranslogOperationHelper translogOperationHelper; + protected final ChannelFactory channelFactory; /** * Creates a new Translog instance. This method will create a new transaction log unless the given {@link TranslogGeneration} is @@ -182,7 +183,8 @@ public Translog( final LongSupplier globalCheckpointSupplier, final LongSupplier primaryTermSupplier, final LongConsumer persistedSequenceNumberConsumer, - final TranslogOperationHelper translogOperationHelper + final TranslogOperationHelper translogOperationHelper, + final ChannelFactory channelFactory ) throws IOException { super(config.getShardId(), config.getIndexSettings()); this.config = config; @@ -198,6 +200,31 @@ public Translog( this.location = config.getTranslogPath(); Files.createDirectories(this.location); this.translogOperationHelper = translogOperationHelper; + this.channelFactory = channelFactory != null ? channelFactory : FileChannel::open; + } + + /** + * Constructor that does not accept channelFactory parameter but accepts translogOperationHelper + */ + public Translog( + final TranslogConfig config, + final String translogUUID, + TranslogDeletionPolicy deletionPolicy, + final LongSupplier globalCheckpointSupplier, + final LongSupplier primaryTermSupplier, + final LongConsumer persistedSequenceNumberConsumer, + final TranslogOperationHelper translogOperationHelper + ) throws IOException { + this( + config, + translogUUID, + deletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer, + translogOperationHelper, + null + ); } /** @@ -218,7 +245,8 @@ public Translog( globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + FileChannel::open ); assert config.getIndexSettings().isDerivedSourceEnabled() == false; // For derived source supported index, it is incorrect to use // this constructor @@ -324,7 +352,7 @@ protected void copyCheckpointTo(Path targetPath) throws IOException { } TranslogReader openReader(Path path, Checkpoint checkpoint) throws IOException { - FileChannel channel = FileChannel.open(path, StandardOpenOption.READ); + FileChannel channel = getChannelFactory().open(path, StandardOpenOption.READ); try { assert Translog.parseIdFromFileName(path) == checkpoint.generation : "expected generation: " + Translog.parseIdFromFileName(path) @@ -1931,7 +1959,7 @@ protected void ensureOpen() { } ChannelFactory getChannelFactory() { - return FileChannel::open; + return this.channelFactory; } /** diff --git a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java index 2e515cb72fd9f..eb592822c17a4 100644 --- a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java +++ b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java @@ -222,7 +222,8 @@ public long minTranslogGenRequired(List readers, TranslogWriter () -> translogGlobalCheckpoint, () -> primaryTerm, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); Translog.Snapshot snapshot = translog.newSnapshot(0, Long.MAX_VALUE) ) { diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 3ae849df07a13..83c1d2f35d607 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -714,7 +714,7 @@ private static BiFunction getTrans return new RemoteBlobStoreInternalTranslogFactory( repositoriesServiceSupplier, threadPool, - RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexSettings.getNodeSettings()), + RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(false), remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()), remoteStoreSettings ); diff --git a/server/src/main/java/org/opensearch/node/remotestore/CompositeRemoteRepository.java b/server/src/main/java/org/opensearch/node/remotestore/CompositeRemoteRepository.java new file mode 100644 index 0000000000000..e1cb226402c39 --- /dev/null +++ b/server/src/main/java/org/opensearch/node/remotestore/CompositeRemoteRepository.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.node.remotestore; + +import org.opensearch.cluster.metadata.RepositoryMetadata; + +import java.util.HashMap; +import java.util.Map; + +/** + * Composite Repository for the ServerSideEncryption support. + */ +public class CompositeRemoteRepository { + + private final Map> repositoryEncryptionTypeMap; + + public CompositeRemoteRepository() { + repositoryEncryptionTypeMap = new HashMap<>(); + } + + public void registerCompositeRepository( + final RemoteStoreRepositoryType repositoryType, + final CompositeRepositoryEncryptionType type, + final RepositoryMetadata metadata + ) { + Map encryptionTypeMap = repositoryEncryptionTypeMap.get(repositoryType); + if (encryptionTypeMap == null) { + encryptionTypeMap = new HashMap<>(); + } + encryptionTypeMap.put(type, metadata); + + repositoryEncryptionTypeMap.put(repositoryType, encryptionTypeMap); + } + + public RepositoryMetadata getRepository(RemoteStoreRepositoryType repositoryType, CompositeRepositoryEncryptionType encryptionType) { + return repositoryEncryptionTypeMap.get(repositoryType).get(encryptionType); + } + + @Override + public String toString() { + return "CompositeRemoteRepository{" + "repositoryEncryptionTypeMap=" + repositoryEncryptionTypeMap + '}'; + } + + public boolean isServerSideEncryptionEnabled() { + return repositoryEncryptionTypeMap.get(RemoteStoreRepositoryType.SEGMENT).containsKey(CompositeRepositoryEncryptionType.SERVER); + } + + /** + * Enum for Remote store repo types + */ + public enum RemoteStoreRepositoryType { + SEGMENT, + TRANSLOG + } + + /** + * Enum for composite repo types + */ + public enum CompositeRepositoryEncryptionType { + CLIENT, + SERVER + } +} diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java index 56c3af3410643..53c8436652b8e 100644 --- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java +++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java @@ -9,6 +9,7 @@ package org.opensearch.node.remotestore; import org.opensearch.cluster.metadata.CryptoMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -37,12 +38,17 @@ */ public class RemoteStoreNodeAttribute { + private static final String REMOTE_STORE_TRANSLOG_REPO_PREFIX = "translog"; + private static final String REMOTE_STORE_SEGMENT_REPO_PREFIX = "segment"; + public static final List REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX = List.of("remote_store", "remote_publication"); // TO-DO the string constants are used only for tests and can be moved to test package public static final String REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.state.repository"; public static final String REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.segment.repository"; + public static final String REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.translog.repository"; + public static final String REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY = "remote_store.routing_table.repository"; public static final List REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() @@ -52,9 +58,11 @@ public class RemoteStoreNodeAttribute { public static final List REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() .map(prefix -> prefix + ".routing_table.repository") .collect(Collectors.toList()); + public static final List REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() .map(prefix -> prefix + ".segment.repository") .collect(Collectors.toList()); + public static final List REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS = REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX.stream() .map(prefix -> prefix + ".translog.repository") .collect(Collectors.toList()); @@ -74,6 +82,8 @@ public class RemoteStoreNodeAttribute { + CryptoMetadata.SETTINGS_KEY; public static final String REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "%s.repository.%s.settings."; + public static final String REPOSITORY_SETTINGS_SSE_ENABLED_ATTRIBUTE_KEY = "sse_enabled"; + private final RepositoriesMetadata repositoriesMetadata; public static List> SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = Arrays.asList( @@ -82,41 +92,46 @@ public class RemoteStoreNodeAttribute { ); public static final String REMOTE_STORE_MODE_KEY = "remote_store.mode"; + public static final String REMOTE_STORE_SSE_REPO_SUFFIX = "-sse"; + + private static CompositeRemoteRepository compositeRemoteRepository; + private static Map repositoryMetadataMap; /** * Creates a new {@link RemoteStoreNodeAttribute} */ public RemoteStoreNodeAttribute(DiscoveryNode node) { + repositoryMetadataMap = new HashMap<>(); + compositeRemoteRepository = new CompositeRemoteRepository(); this.repositoriesMetadata = buildRepositoriesMetadata(node); } - private String validateAttributeNonNull(DiscoveryNode node, String attributeKey) { + private String getAndValidateNodeAttribute(DiscoveryNode node, String attributeKey) { String attributeValue = node.getAttributes().get(attributeKey); if (attributeValue == null || attributeValue.isEmpty()) { throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKey + "]"); } - return attributeValue; } - private Tuple validateAttributeNonNull(DiscoveryNode node, List attributeKeys) { + private Tuple getAndValidateNodeAttributeEntries(DiscoveryNode node, List attributeKeys) { Tuple attributeValue = getValue(node.getAttributes(), attributeKeys); if (attributeValue == null || attributeValue.v1() == null || attributeValue.v1().isEmpty()) { throw new IllegalStateException("joining node [" + node + "] doesn't have the node attribute [" + attributeKeys.get(0) + "]"); } - return attributeValue; } private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repositoryName, String prefix) { String metadataKey = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_ATTRIBUTE_KEY_FORMAT, prefix, repositoryName); boolean isRepoEncrypted = node.getAttributes().keySet().stream().anyMatch(key -> key.startsWith(metadataKey)); - if (isRepoEncrypted == false) { + + if (!isRepoEncrypted) { return null; } - String keyProviderName = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_NAME_KEY); - String keyProviderType = validateAttributeNonNull(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_TYPE_KEY); + String keyProviderName = getAndValidateNodeAttribute(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_NAME_KEY); + String keyProviderType = getAndValidateNodeAttribute(node, metadataKey + "." + CryptoMetadata.KEY_PROVIDER_TYPE_KEY); String settingsAttributeKeyPrefix = String.format(Locale.getDefault(), REPOSITORY_CRYPTO_SETTINGS_PREFIX, prefix, repositoryName); @@ -132,7 +147,7 @@ private CryptoMetadata buildCryptoMetadata(DiscoveryNode node, String repository return new CryptoMetadata(keyProviderName, keyProviderType, settings.build()); } - private Map validateSettingsAttributesNonNull(DiscoveryNode node, String repositoryName, String prefix) { + private Map getSettingAttribute(DiscoveryNode node, String repositoryName, String prefix) { String settingsAttributeKeyPrefix = String.format( Locale.getDefault(), REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, @@ -143,23 +158,22 @@ private Map validateSettingsAttributesNonNull(DiscoveryNode node .keySet() .stream() .filter(key -> key.startsWith(settingsAttributeKeyPrefix)) - .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> validateAttributeNonNull(node, key))); + .collect(Collectors.toMap(key -> key.replace(settingsAttributeKeyPrefix, ""), key -> getAndValidateNodeAttribute(node, key))); if (settingsMap.isEmpty()) { throw new IllegalStateException( "joining node [" + node + "] doesn't have settings attribute for [" + repositoryName + "] repository" ); } - return settingsMap; } private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String name, String prefix) { - String type = validateAttributeNonNull( + String type = getAndValidateNodeAttribute( node, String.format(Locale.getDefault(), REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, prefix, name) ); - Map settingsMap = validateSettingsAttributesNonNull(node, name, prefix); + Map settingsMap = getSettingAttribute(node, name, prefix); Settings.Builder settings = Settings.builder(); settingsMap.forEach(settings::put); @@ -168,21 +182,65 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na // Repository metadata built here will always be for a system repository. settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); - return new RepositoryMetadata(name, type, settings.build(), cryptoMetadata); } private RepositoriesMetadata buildRepositoriesMetadata(DiscoveryNode node) { - Map repositoryNamesWithPrefix = getValidatedRepositoryNames(node); + Map remoteStoryTypeToRepoNameMap = new HashMap<>(); + Map repositoryNamesWithPrefix = getValidatedRepositoryNames(node, remoteStoryTypeToRepoNameMap); + List repositoryMetadataList = new ArrayList<>(); for (Map.Entry repository : repositoryNamesWithPrefix.entrySet()) { - repositoryMetadataList.add(buildRepositoryMetadata(node, repository.getKey(), repository.getValue())); + RepositoryMetadata repositoryMetadata = buildRepositoryMetadata(node, repository.getKey(), repository.getValue()); + repositoryMetadataList.add(repositoryMetadata); + repositoryMetadataMap.put(repositoryMetadata.name(), repositoryMetadata); + + if (isCompositeRepository(repositoryMetadata)) { + RepositoryMetadata sseRepoMetadata = new RepositoryMetadata( + repositoryMetadata.name() + REMOTE_STORE_SSE_REPO_SUFFIX, + repositoryMetadata.type(), + repositoryMetadata.settings() + ); + repositoryMetadataMap.put(sseRepoMetadata.name(), sseRepoMetadata); + repositoryMetadataList.add(sseRepoMetadata); + } } + // Let's Iterate over repo's and build Composite Repository structure + for (Map.Entry repositoryTypeToNameEntry : remoteStoryTypeToRepoNameMap.entrySet()) { + CompositeRemoteRepository.CompositeRepositoryEncryptionType encryptionType = + CompositeRemoteRepository.CompositeRepositoryEncryptionType.CLIENT; + CompositeRemoteRepository.RemoteStoreRepositoryType remoteStoreRepositoryType = + CompositeRemoteRepository.RemoteStoreRepositoryType.SEGMENT; + if (repositoryTypeToNameEntry.getKey().contains(REMOTE_STORE_TRANSLOG_REPO_PREFIX)) { + remoteStoreRepositoryType = CompositeRemoteRepository.RemoteStoreRepositoryType.TRANSLOG; + } + + String repositoryName = repositoryTypeToNameEntry.getValue(); + compositeRemoteRepository.registerCompositeRepository( + remoteStoreRepositoryType, + encryptionType, + repositoryMetadataMap.get(repositoryName) + ); + + String sseRepositoryName = repositoryTypeToNameEntry.getValue() + REMOTE_STORE_SSE_REPO_SUFFIX; + if (repositoryMetadataMap.containsKey(sseRepositoryName)) { + encryptionType = CompositeRemoteRepository.CompositeRepositoryEncryptionType.SERVER; + compositeRemoteRepository.registerCompositeRepository( + remoteStoreRepositoryType, + encryptionType, + repositoryMetadataMap.get(sseRepositoryName) + ); + } + } return new RepositoriesMetadata(repositoryMetadataList); } + private boolean isCompositeRepository(RepositoryMetadata repositoryMetadata) { + return repositoryMetadata.settings().hasValue(REPOSITORY_SETTINGS_SSE_ENABLED_ATTRIBUTE_KEY); + } + private static Tuple getValue(Map attributes, List keys) { for (String key : keys) { if (attributes.containsKey(key)) { @@ -197,7 +255,7 @@ private enum RemoteStoreMode { DEFAULT } - private Map getValidatedRepositoryNames(DiscoveryNode node) { + private Map getValidatedRepositoryNames(DiscoveryNode node, Map remoteStoryTypeToRepoNameMap) { Set> repositoryNames = new HashSet<>(); RemoteStoreMode remoteStoreMode = RemoteStoreMode.DEFAULT; if (containsKey(node.getAttributes(), List.of(REMOTE_STORE_MODE_KEY))) { @@ -208,15 +266,35 @@ private Map getValidatedRepositoryNames(DiscoveryNode node) { throw new IllegalStateException("Unknown remote store mode [" + mode + "] for node [" + node + "]"); } } + if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + addRepositoryNames( + node, + remoteStoryTypeToRepoNameMap, + repositoryNames, + REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS, + REMOTE_STORE_SEGMENT_REPO_PREFIX + ); } else if (containsKey(node.getAttributes(), REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) || containsKey(node.getAttributes(), REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS)); - repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + addRepositoryNames( + node, + remoteStoryTypeToRepoNameMap, + repositoryNames, + REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS, + REMOTE_STORE_SEGMENT_REPO_PREFIX + ); + addRepositoryNames( + node, + remoteStoryTypeToRepoNameMap, + repositoryNames, + REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS, + REMOTE_STORE_TRANSLOG_REPO_PREFIX + ); + + repositoryNames.add(getAndValidateNodeAttributeEntries(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); } else if (containsKey(node.getAttributes(), REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { - repositoryNames.add(validateAttributeNonNull(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(getAndValidateNodeAttributeEntries(node, REMOTE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); } if (containsKey(node.getAttributes(), REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)) { if (remoteStoreMode == RemoteStoreMode.SEGMENTS_ONLY) { @@ -228,7 +306,7 @@ private Map getValidatedRepositoryNames(DiscoveryNode node) { + "]" ); } - repositoryNames.add(validateAttributeNonNull(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); + repositoryNames.add(getAndValidateNodeAttributeEntries(node, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS)); } Map repoNamesWithPrefix = new HashMap<>(); @@ -240,6 +318,18 @@ private Map getValidatedRepositoryNames(DiscoveryNode node) { return repoNamesWithPrefix; } + private void addRepositoryNames( + DiscoveryNode node, + Map remoteStoryTypeToRepoNameMap, + Set> repositoryNames, + List attributeKeys, + String remoteStoreRepoPrefix + ) { + Tuple remoteStoreAttributeKeyMap = getAndValidateNodeAttributeEntries(node, attributeKeys); + remoteStoryTypeToRepoNameMap.put(remoteStoreRepoPrefix, remoteStoreAttributeKeyMap.v1()); + repositoryNames.add(remoteStoreAttributeKeyMap); + } + public static boolean isRemoteStoreAttributePresent(Settings settings) { for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) { if (settings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + prefix).isEmpty() == false) { @@ -280,22 +370,8 @@ public static boolean isRemoteClusterStateConfigured(Settings settings) { return false; } - public static String getRemoteStoreSegmentRepo(Settings settings) { - for (String prefix : REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS) { - if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); - } - } - return null; - } - - public static String getRemoteStoreTranslogRepo(Settings settings) { - for (String prefix : REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS) { - if (settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix) != null) { - return settings.get(Node.NODE_ATTRIBUTES.getKey() + prefix); - } - } - return null; + public static boolean isRemoteStoreServerSideEncryptionEnabled() { + return compositeRemoteRepository.isServerSideEncryptionEnabled(); } public static boolean isRemoteStoreClusterStateEnabled(Settings settings) { @@ -355,12 +431,40 @@ public static String getRoutingTableRepoName(Map repos) { return getValueFromAnyKey(repos, REMOTE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEYS); } - public static String getSegmentRepoName(Map repos) { - return getValueFromAnyKey(repos, REMOTE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEYS); + public static String getRemoteStoreSegmentRepo(boolean serverSideEncryptionEnabled) { + if (compositeRemoteRepository == null) { + return null; + } + + CompositeRemoteRepository.RemoteStoreRepositoryType repositoryType = CompositeRemoteRepository.RemoteStoreRepositoryType.SEGMENT; + CompositeRemoteRepository.CompositeRepositoryEncryptionType encryptionType = + CompositeRemoteRepository.CompositeRepositoryEncryptionType.CLIENT; + if (serverSideEncryptionEnabled) { + encryptionType = CompositeRemoteRepository.CompositeRepositoryEncryptionType.SERVER; + } + return compositeRemoteRepository.getRepository(repositoryType, encryptionType).name(); + } + + public static String getRemoteStoreSegmentRepo(Settings indexSettings) { + return getRemoteStoreSegmentRepo(indexSettings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, false)); + } + + public static String getRemoteStoreTranslogRepo(boolean serverSideEncryptionEnabled) { + if (compositeRemoteRepository == null) { + return null; + } + + CompositeRemoteRepository.RemoteStoreRepositoryType repositoryType = CompositeRemoteRepository.RemoteStoreRepositoryType.TRANSLOG; + CompositeRemoteRepository.CompositeRepositoryEncryptionType encryptionType = + CompositeRemoteRepository.CompositeRepositoryEncryptionType.CLIENT; + if (serverSideEncryptionEnabled) { + encryptionType = CompositeRemoteRepository.CompositeRepositoryEncryptionType.SERVER; + } + return compositeRemoteRepository.getRepository(repositoryType, encryptionType).name(); } - public static String getTranslogRepoName(Map repos) { - return getValueFromAnyKey(repos, REMOTE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEYS); + public static String getRemoteStoreTranslogRepo(Settings indexSettings) { + return getRemoteStoreTranslogRepo(indexSettings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_SSE_ENABLED, false)); } private static String getValueFromAnyKey(Map repos, List keys) { diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index beecab73ffeab..c6fe57188eff1 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -136,6 +136,7 @@ import org.opensearch.search.profile.Profilers; import org.opensearch.search.profile.SearchProfileShardResults; import org.opensearch.search.query.QueryPhase; +import org.opensearch.search.query.QueryRewriterRegistry; import org.opensearch.search.query.QuerySearchRequest; import org.opensearch.search.query.QuerySearchResult; import org.opensearch.search.query.ScrollQuerySearchResult; @@ -276,6 +277,27 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv Property.Deprecated ); + public static final Setting QUERY_REWRITING_ENABLED_SETTING = Setting.boolSetting( + "search.query_rewriting.enabled", + true, + Property.Dynamic, + Property.NodeScope + ); + + /** + * Controls the threshold for the number of term queries on the same field that triggers + * the TermsMergingRewriter to combine them into a single terms query. For example, + * if set to 16 (default), when 16 or more term queries target the same field within + * a boolean clause, they will be merged into a single terms query for better performance. + */ + public static final Setting QUERY_REWRITING_TERMS_THRESHOLD_SETTING = Setting.intSetting( + "search.query_rewriting.terms_threshold", + 16, + 2, // minimum value + Property.Dynamic, + Property.NodeScope + ); + // Allow concurrent segment search for all requests public static final String CONCURRENT_SEGMENT_SEARCH_MODE_ALL = "all"; @@ -507,6 +529,10 @@ public SearchService( this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories; this.pluginProfilers = pluginProfilers; + + // Initialize QueryRewriterRegistry with cluster settings so TermsMergingRewriter + // can register its settings update consumer + QueryRewriterRegistry.INSTANCE.initialize(settings, clusterService.getClusterSettings()); } private void validateKeepAlives(TimeValue defaultKeepAlive, TimeValue maxKeepAlive) { @@ -1488,8 +1514,13 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.size(source.size()); Map innerHitBuilders = new HashMap<>(); if (source.query() != null) { - InnerHitContextBuilder.extractInnerHits(source.query(), innerHitBuilders); - context.parsedQuery(queryShardContext.toQuery(source.query())); + QueryBuilder query = source.query(); + + // Apply query rewriting optimizations + query = QueryRewriterRegistry.INSTANCE.rewrite(query, queryShardContext); + + InnerHitContextBuilder.extractInnerHits(query, innerHitBuilders); + context.parsedQuery(queryShardContext.toQuery(query)); } if (source.postFilter() != null) { InnerHitContextBuilder.extractInnerHits(source.postFilter(), innerHitBuilders); diff --git a/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java b/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java index 9c9bef2a23e53..e9adfcf9c0760 100644 --- a/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java +++ b/server/src/main/java/org/opensearch/search/profile/fetch/FlatFetchProfileTree.java @@ -12,11 +12,12 @@ import org.opensearch.search.profile.Timer; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; /** * Simplified profiling tree for fetch phase operations. Each fetch phase is @@ -59,13 +60,13 @@ private static class Node { } private final List roots = new ArrayList<>(); - private final Map rootsMap = new HashMap<>(); - private final Map phaseMap = new HashMap<>(); + private final ConcurrentMap rootsMap = new ConcurrentHashMap<>(); + private final ConcurrentMap phaseMap = new ConcurrentHashMap<>(); /** Start profiling a new fetch phase and return its breakdown. */ FetchProfileBreakdown startFetchPhase(String element) { // Make phase name unique for concurrent slices by including thread info - String uniqueElement = element + "_" + Thread.currentThread().getId(); + String uniqueElement = element + "_" + Thread.currentThread().threadId(); Node node = rootsMap.get(uniqueElement); if (node == null) { @@ -81,8 +82,8 @@ FetchProfileBreakdown startFetchPhase(String element) { /** Start profiling a fetch sub-phase under the specified parent phase. */ FetchProfileBreakdown startSubPhase(String element, String parentElement) { // Make phase names unique for concurrent slices - String uniqueParentElement = parentElement + "_" + Thread.currentThread().getId(); - String uniqueElement = element + "_" + Thread.currentThread().getId(); + String uniqueParentElement = parentElement + "_" + Thread.currentThread().threadId(); + String uniqueElement = element + "_" + Thread.currentThread().threadId(); Node parent = phaseMap.get(uniqueParentElement); if (parent == null) { @@ -107,7 +108,7 @@ FetchProfileBreakdown startSubPhase(String element, String parentElement) { */ void endFetchPhase(String element) { // Make phase name unique for concurrent slices - String uniqueElement = element + "_" + Thread.currentThread().getId(); + String uniqueElement = element + "_" + Thread.currentThread().threadId(); Node node = phaseMap.get(uniqueElement); if (node == null) { diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java index b08bebb840343..6bcad535e5274 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecFactory.java @@ -8,6 +8,7 @@ package org.opensearch.search.query; +import org.apache.lucene.search.Query; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.search.internal.SearchContext; @@ -21,12 +22,14 @@ public interface QueryCollectorContextSpecFactory { /** * @param searchContext context needed to create collector context spec + * @param query required to create collector context spec * @param queryCollectorArguments arguments to create collector context spec * @return QueryCollectorContextSpec * @throws IOException */ Optional createQueryCollectorContextSpec( SearchContext searchContext, + Query query, QueryCollectorArguments queryCollectorArguments ) throws IOException; } diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java index 413cd63b97856..384f8f031373b 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java +++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContextSpecRegistry.java @@ -8,6 +8,7 @@ package org.opensearch.search.query; +import org.apache.lucene.search.Query; import org.opensearch.search.internal.SearchContext; import java.io.IOException; @@ -43,18 +44,24 @@ public static void registerFactory(QueryCollectorContextSpecFactory factory) { /** * Get collector context spec * @param searchContext search context + * @param query required to create collectorContext spec * @param queryCollectorArguments query collector arguments * @return collector context spec * @throws IOException */ public static Optional getQueryCollectorContextSpec( final SearchContext searchContext, + final Query query, final QueryCollectorArguments queryCollectorArguments ) throws IOException { Iterator iterator = registry.iterator(); while (iterator.hasNext()) { QueryCollectorContextSpecFactory factory = iterator.next(); - Optional spec = factory.createQueryCollectorContextSpec(searchContext, queryCollectorArguments); + Optional spec = factory.createQueryCollectorContextSpec( + searchContext, + query, + queryCollectorArguments + ); if (spec.isEmpty() == false) { return spec; } diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java index ebf8ed0ce3362..f8427440a6c13 100644 --- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java @@ -446,14 +446,16 @@ protected boolean searchWithCollector( boolean hasFilterCollector, boolean hasTimeout ) throws IOException { - QueryCollectorContext queryCollectorContext = getQueryCollectorContext(searchContext, hasFilterCollector); + QueryCollectorContext queryCollectorContext = getQueryCollectorContext(searchContext, query, hasFilterCollector); return searchWithCollector(searchContext, searcher, query, collectors, queryCollectorContext, hasFilterCollector, hasTimeout); } - private QueryCollectorContext getQueryCollectorContext(SearchContext searchContext, boolean hasFilterCollector) throws IOException { + private QueryCollectorContext getQueryCollectorContext(SearchContext searchContext, Query query, boolean hasFilterCollector) + throws IOException { // create the top docs collector last when the other collectors are known final Optional queryCollectorContextOpt = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( searchContext, + query, new QueryCollectorArguments.Builder().hasFilterCollector(hasFilterCollector).build() ).map(queryCollectorContextSpec -> new QueryCollectorContext(queryCollectorContextSpec.getContextName()) { @Override diff --git a/server/src/main/java/org/opensearch/search/query/QueryRewriter.java b/server/src/main/java/org/opensearch/search/query/QueryRewriter.java new file mode 100644 index 0000000000000..32854c5881d61 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryRewriter.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; + +/** + * Interface for query rewriting implementations that optimize query structure + * before conversion to Lucene queries. + * + * @opensearch.experimental + */ +@ExperimentalApi +public interface QueryRewriter { + + /** + * Rewrites the given query builder to a more optimal form. + * + * @param query The query to rewrite + * @param context The search execution context + * @return The rewritten query (may be the same instance if no rewrite needed) + */ + QueryBuilder rewrite(QueryBuilder query, QueryShardContext context); + + /** + * Returns the priority of this rewriter. Lower values execute first. + * This allows control over rewrite ordering when multiple rewriters + * may interact. + * + * @return The priority value + */ + default int priority() { + return 1000; + } + + /** + * Returns the name of this rewriter for debugging and profiling. + * + * @return The rewriter name + */ + String name(); +} diff --git a/server/src/main/java/org/opensearch/search/query/QueryRewriterRegistry.java b/server/src/main/java/org/opensearch/search/query/QueryRewriterRegistry.java new file mode 100644 index 0000000000000..7de5a5c8e554e --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/QueryRewriterRegistry.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.SearchService; +import org.opensearch.search.query.rewriters.BooleanFlatteningRewriter; +import org.opensearch.search.query.rewriters.MatchAllRemovalRewriter; +import org.opensearch.search.query.rewriters.MustNotToShouldRewriter; +import org.opensearch.search.query.rewriters.MustToFilterRewriter; +import org.opensearch.search.query.rewriters.TermsMergingRewriter; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Registry for query rewriters + * + * @opensearch.internal + */ +public final class QueryRewriterRegistry { + + private static final Logger logger = LogManager.getLogger(QueryRewriterRegistry.class); + + public static final QueryRewriterRegistry INSTANCE = new QueryRewriterRegistry(); + + /** + * Default rewriters. + * CopyOnWriteArrayList is used for thread-safety during registration. + */ + private final CopyOnWriteArrayList rewriters; + + /** + * Whether query rewriting is enabled. + */ + private volatile boolean enabled; + + private QueryRewriterRegistry() { + this.rewriters = new CopyOnWriteArrayList<>(); + + // Register default rewriters using singletons + registerRewriter(BooleanFlatteningRewriter.INSTANCE); + registerRewriter(MustToFilterRewriter.INSTANCE); + registerRewriter(MustNotToShouldRewriter.INSTANCE); + registerRewriter(MatchAllRemovalRewriter.INSTANCE); + registerRewriter(TermsMergingRewriter.INSTANCE); + } + + /** + * Register a custom query rewriter. + * + * @param rewriter The rewriter to register + */ + public void registerRewriter(QueryRewriter rewriter) { + if (rewriter != null) { + rewriters.add(rewriter); + logger.info("Registered query rewriter: {}", rewriter.name()); + } + } + + /** + * Initialize the registry with cluster settings. + * This must be called once during system startup to properly configure + * the TermsMergingRewriter with settings and update consumers. + * + * @param settings Initial cluster settings + * @param clusterSettings Cluster settings for registering update consumers + */ + public void initialize(Settings settings, ClusterSettings clusterSettings) { + TermsMergingRewriter.INSTANCE.initialize(settings, clusterSettings); + this.enabled = SearchService.QUERY_REWRITING_ENABLED_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer( + SearchService.QUERY_REWRITING_ENABLED_SETTING, + (Boolean enabled) -> this.enabled = enabled + ); + } + + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (!enabled || query == null) { + return query; + } + + List sortedRewriters = new ArrayList<>(rewriters); + sortedRewriters.sort(Comparator.comparingInt(QueryRewriter::priority)); + + QueryBuilder current = query; + for (QueryRewriter rewriter : sortedRewriters) { + try { + QueryBuilder rewritten = rewriter.rewrite(current, context); + if (rewritten != current) { + current = rewritten; + } + } catch (Exception e) { + logger.warn("Query rewriter {} failed: {}", rewriter.name(), e.getMessage()); + } + } + + return current; + } +} diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java index 82a8d3507cb10..5b82b0df68ca6 100644 --- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java @@ -331,8 +331,7 @@ protected ReduceableSearchResult reduceWith(final Collection + * {"bool": {"filter": [{"bool": {"filter": [{"term": {"field": "value"}}]}}]}} + * + * becomes: + *
+ * {"bool": {"filter": [{"term": {"field": "value"}}]}}
+ * 
+ * + * Note: While Lucene's BooleanQuery does flatten pure disjunctions (SHOULD-only clauses) + * for WAND optimization, it does NOT flatten other nested structures like filter-in-filter + * or must-in-must. This rewriter handles those additional patterns that are common in + * user-generated and template-based queries but not optimized by Lucene. + * + * @opensearch.internal + */ +public class BooleanFlatteningRewriter implements QueryRewriter { + + public static final BooleanFlatteningRewriter INSTANCE = new BooleanFlatteningRewriter(); + + private BooleanFlatteningRewriter() { + // Singleton + } + + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (!(query instanceof BoolQueryBuilder)) { + return query; + } + + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // First check if flattening is needed + if (!needsFlattening(boolQuery)) { + return query; + } + + return flattenBoolQuery(boolQuery); + } + + private boolean needsFlattening(BoolQueryBuilder boolQuery) { + // Check all clause types for nested bool queries that can be flattened + if (hasFlattenableBool(boolQuery.must(), ClauseType.MUST) + || hasFlattenableBool(boolQuery.filter(), ClauseType.FILTER) + || hasFlattenableBool(boolQuery.should(), ClauseType.SHOULD) + || hasFlattenableBool(boolQuery.mustNot(), ClauseType.MUST_NOT)) { + return true; + } + + // Check if any nested bool queries need flattening + return hasNestedBoolThatNeedsFlattening(boolQuery); + } + + private boolean hasFlattenableBool(List clauses, ClauseType parentType) { + for (QueryBuilder clause : clauses) { + if (clause instanceof BoolQueryBuilder) { + BoolQueryBuilder nestedBool = (BoolQueryBuilder) clause; + // Can flatten if nested bool only has one type of clause matching parent + if (canFlatten(nestedBool, parentType)) { + return true; + } + } + } + return false; + } + + private boolean hasNestedBoolThatNeedsFlattening(BoolQueryBuilder boolQuery) { + for (QueryBuilder clause : boolQuery.must()) { + if (clause instanceof BoolQueryBuilder && needsFlattening((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.filter()) { + if (clause instanceof BoolQueryBuilder && needsFlattening((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.should()) { + if (clause instanceof BoolQueryBuilder && needsFlattening((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.mustNot()) { + if (clause instanceof BoolQueryBuilder && needsFlattening((BoolQueryBuilder) clause)) { + return true; + } + } + return false; + } + + private BoolQueryBuilder flattenBoolQuery(BoolQueryBuilder original) { + BoolQueryBuilder flattened = new BoolQueryBuilder(); + + flattened.boost(original.boost()); + flattened.queryName(original.queryName()); + flattened.minimumShouldMatch(original.minimumShouldMatch()); + flattened.adjustPureNegative(original.adjustPureNegative()); + + flattenClauses(original.must(), flattened, ClauseType.MUST); + flattenClauses(original.filter(), flattened, ClauseType.FILTER); + flattenClauses(original.should(), flattened, ClauseType.SHOULD); + flattenClauses(original.mustNot(), flattened, ClauseType.MUST_NOT); + + return flattened; + } + + private void flattenClauses(List clauses, BoolQueryBuilder target, ClauseType clauseType) { + for (QueryBuilder clause : clauses) { + if (clause instanceof BoolQueryBuilder) { + BoolQueryBuilder nestedBool = (BoolQueryBuilder) clause; + + if (canFlatten(nestedBool, clauseType)) { + // Flatten the nested bool query by extracting its clauses + List nestedClauses = getClausesForType(nestedBool, clauseType); + for (QueryBuilder nestedClause : nestedClauses) { + // Recursively flatten if needed + if (nestedClause instanceof BoolQueryBuilder) { + nestedClause = flattenBoolQuery((BoolQueryBuilder) nestedClause); + } + addClauseBasedOnType(target, nestedClause, clauseType); + } + } else { + // Can't flatten this bool, but recursively flatten its contents + BoolQueryBuilder flattenedNested = flattenBoolQuery(nestedBool); + addClauseBasedOnType(target, flattenedNested, clauseType); + } + } else { + // Non-boolean clause, add as-is + addClauseBasedOnType(target, clause, clauseType); + } + } + } + + private boolean canFlatten(BoolQueryBuilder nestedBool, ClauseType parentType) { + // Can only flatten if: + // 1. The nested bool has the same properties as default (boost=1, no queryName, etc.) + // 2. The nested bool only has clauses of the same type as the parent + + if (nestedBool.boost() != 1.0f || nestedBool.queryName() != null) { + return false; + } + + // Check if only has clauses matching parent type + switch (parentType) { + case MUST: + return !nestedBool.must().isEmpty() + && nestedBool.filter().isEmpty() + && nestedBool.should().isEmpty() + && nestedBool.mustNot().isEmpty(); + case FILTER: + return nestedBool.must().isEmpty() + && !nestedBool.filter().isEmpty() + && nestedBool.should().isEmpty() + && nestedBool.mustNot().isEmpty(); + case SHOULD: + return nestedBool.must().isEmpty() + && nestedBool.filter().isEmpty() + && !nestedBool.should().isEmpty() + && nestedBool.mustNot().isEmpty() + && nestedBool.minimumShouldMatch() == null; + case MUST_NOT: + return nestedBool.must().isEmpty() + && nestedBool.filter().isEmpty() + && nestedBool.should().isEmpty() + && !nestedBool.mustNot().isEmpty(); + default: + return false; + } + } + + private List getClausesForType(BoolQueryBuilder bool, ClauseType type) { + switch (type) { + case MUST: + return bool.must(); + case FILTER: + return bool.filter(); + case SHOULD: + return bool.should(); + case MUST_NOT: + return bool.mustNot(); + default: + return new ArrayList<>(); + } + } + + private void addClauseBasedOnType(BoolQueryBuilder target, QueryBuilder clause, ClauseType type) { + switch (type) { + case MUST: + target.must(clause); + break; + case FILTER: + target.filter(clause); + break; + case SHOULD: + target.should(clause); + break; + case MUST_NOT: + target.mustNot(clause); + break; + } + } + + @Override + public int priority() { + return 100; + } + + @Override + public String name() { + return "boolean_flattening"; + } + + private enum ClauseType { + MUST, + FILTER, + SHOULD, + MUST_NOT + } +} diff --git a/server/src/main/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriter.java b/server/src/main/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriter.java new file mode 100644 index 0000000000000..39d2257e483bc --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriter.java @@ -0,0 +1,239 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.query.QueryRewriter; + +import java.util.List; + +/** + * Removes unnecessary match_all queries from boolean contexts where they have no effect. + * + * @opensearch.internal + */ +public class MatchAllRemovalRewriter implements QueryRewriter { + + public static final MatchAllRemovalRewriter INSTANCE = new MatchAllRemovalRewriter(); + + private MatchAllRemovalRewriter() { + // Singleton + } + + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (query instanceof BoolQueryBuilder) { + return rewriteBoolQuery((BoolQueryBuilder) query); + } + return query; + } + + private QueryBuilder rewriteBoolQuery(BoolQueryBuilder original) { + // Special case: bool query with only match_all queries and no should/mustNot + if (original.should().isEmpty() && original.mustNot().isEmpty()) { + boolean onlyMatchAll = true; + int matchAllCount = 0; + int matchAllInMust = 0; + + for (QueryBuilder q : original.must()) { + if (q instanceof MatchAllQueryBuilder) { + matchAllCount++; + matchAllInMust++; + } else { + // Don't treat constant score queries or any other queries as match_all + onlyMatchAll = false; + break; + } + } + + if (onlyMatchAll) { + for (QueryBuilder q : original.filter()) { + if (q instanceof MatchAllQueryBuilder) { + matchAllCount++; + } else { + onlyMatchAll = false; + break; + } + } + } + + // Only convert to single match_all if there are no must clauses + // (to preserve scoring) or if there's only one match_all total + if (onlyMatchAll && matchAllCount > 0 && (matchAllInMust == 0 || matchAllCount == 1)) { + // Convert to single match_all, preserving boost + MatchAllQueryBuilder matchAll = new MatchAllQueryBuilder(); + if (original.boost() != 1.0f) { + matchAll.boost(original.boost()); + } + return matchAll; + } + } + + // Check if we need rewriting + boolean needsRewrite = shouldRewrite(original); + + if (!needsRewrite) { + return original; + } + + // Clone the query structure + BoolQueryBuilder rewritten = new BoolQueryBuilder(); + rewritten.boost(original.boost()); + rewritten.queryName(original.queryName()); + rewritten.minimumShouldMatch(original.minimumShouldMatch()); + rewritten.adjustPureNegative(original.adjustPureNegative()); + + // Process each clause type with different match_all removal logic: + // - must: Remove match_all only if other queries exist (preserves scoring semantics) + // - filter: Always remove match_all (it's redundant in non-scoring context) + // - should: Keep match_all (changes OR semantics if removed) + // - mustNot: Keep match_all (excluding all docs is meaningful) + processClausesWithContext(original.must(), rewritten::must, true, original, true); + processClauses(original.filter(), rewritten::filter, true, original); + processClauses(original.should(), rewritten::should, false, original); + processClauses(original.mustNot(), rewritten::mustNot, false, original); + + return rewritten; + } + + private boolean shouldRewrite(BoolQueryBuilder bool) { + // Check if any must/filter has match_all + if (hasMatchAll(bool.must()) || hasMatchAll(bool.filter())) { + return true; + } + + // Check nested bool queries + return hasNestedBoolThatNeedsRewrite(bool); + } + + private boolean hasMatchAll(List clauses) { + for (QueryBuilder q : clauses) { + if (q instanceof MatchAllQueryBuilder) { + return true; + } + } + return false; + } + + private boolean hasNestedBoolThatNeedsRewrite(BoolQueryBuilder bool) { + for (QueryBuilder q : bool.must()) { + if (q instanceof BoolQueryBuilder && shouldRewrite((BoolQueryBuilder) q)) { + return true; + } + } + for (QueryBuilder q : bool.filter()) { + if (q instanceof BoolQueryBuilder && shouldRewrite((BoolQueryBuilder) q)) { + return true; + } + } + for (QueryBuilder q : bool.should()) { + if (q instanceof BoolQueryBuilder && shouldRewrite((BoolQueryBuilder) q)) { + return true; + } + } + for (QueryBuilder q : bool.mustNot()) { + if (q instanceof BoolQueryBuilder && shouldRewrite((BoolQueryBuilder) q)) { + return true; + } + } + return false; + } + + private void processClausesWithContext( + List clauses, + ClauseAdder adder, + boolean removeMatchAll, + BoolQueryBuilder original, + boolean isMustClause + ) { + if (!removeMatchAll) { + processClauses(clauses, adder, false, original); + return; + } + + // For must clauses, only remove match_all if there are other non-match_all queries + if (isMustClause) { + boolean hasNonMatchAll = clauses.stream().anyMatch(q -> !(q instanceof MatchAllQueryBuilder)); + + // Also check if we're in a scoring context (no filter/should/mustNot clauses) + boolean isScoringContext = original.filter().isEmpty() && original.should().isEmpty() && original.mustNot().isEmpty(); + + if (!hasNonMatchAll || isScoringContext) { + // All queries are match_all or we're in a scoring context, don't remove any to preserve scoring + processClauses(clauses, adder, false, original); + return; + } + } + + // Otherwise, use normal processing + processClauses(clauses, adder, removeMatchAll, original); + } + + private void processClauses(List clauses, ClauseAdder adder, boolean removeMatchAll, BoolQueryBuilder original) { + if (!removeMatchAll) { + // For should/mustNot, don't remove match_all + for (QueryBuilder clause : clauses) { + if (clause instanceof BoolQueryBuilder) { + adder.addClause(rewriteBoolQuery((BoolQueryBuilder) clause)); + } else { + adder.addClause(clause); + } + } + return; + } + + // For must/filter, remove match_all if: + // 1. There are other non-match_all clauses in the same list OR + // 2. There are clauses in other lists (must, filter, should, mustNot) + boolean hasOtherClauses = hasNonMatchAllInSameList(clauses) || hasClausesInOtherLists(original); + + for (QueryBuilder clause : clauses) { + if (clause instanceof BoolQueryBuilder) { + adder.addClause(rewriteBoolQuery((BoolQueryBuilder) clause)); + } else if (clause instanceof MatchAllQueryBuilder && hasOtherClauses) { + // Skip match_all + continue; + } else { + adder.addClause(clause); + } + } + } + + private boolean hasNonMatchAllInSameList(List clauses) { + for (QueryBuilder q : clauses) { + if (!(q instanceof MatchAllQueryBuilder)) { + return true; + } + } + return false; + } + + private boolean hasClausesInOtherLists(BoolQueryBuilder bool) { + // Check if there are any clauses in any list + return !bool.must().isEmpty() || !bool.filter().isEmpty() || !bool.should().isEmpty() || !bool.mustNot().isEmpty(); + } + + @Override + public int priority() { + return 300; + } + + @Override + public String name() { + return "match_all_removal"; + } + + @FunctionalInterface + private interface ClauseAdder { + void addClause(QueryBuilder clause); + } +} diff --git a/server/src/main/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriter.java b/server/src/main/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriter.java new file mode 100644 index 0000000000000..ddcbea48b4b12 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriter.java @@ -0,0 +1,251 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.ComplementAwareQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.WithFieldName; +import org.opensearch.search.query.QueryRewriter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Rewrites must_not clauses to should clauses when possible. + * This improves performance by transforming negative queries into positive ones. + * + * For example: + *
+ * {"bool": {"must_not": [{"range": {"age": {"gte": 18, "lte": 65}}}]}}
+ * 
+ * becomes: + *
+ * {"bool": {"must": [{"bool": {"should": [
+ *   {"range": {"age": {"lt": 18}}},
+ *   {"range": {"age": {"gt": 65}}}
+ * ], "minimum_should_match": 1}}]}}
+ * 
+ * + * This optimization applies to: + * - RangeQueryBuilder + * - TermQueryBuilder (on numeric fields) + * - TermsQueryBuilder (on numeric fields) + * - MatchQueryBuilder (on numeric fields) + * + * @opensearch.internal + */ +public class MustNotToShouldRewriter implements QueryRewriter { + + public static final MustNotToShouldRewriter INSTANCE = new MustNotToShouldRewriter(); + + private MustNotToShouldRewriter() { + // Singleton + } + + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (!(query instanceof BoolQueryBuilder)) { + return query; + } + + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // We need LeafReaderContexts to verify single-valued fields (only for must_not rewriting) + List leafReaderContexts = null; + List mustNotClausesToRewrite = new ArrayList<>(); + + // Only process must_not clauses if they exist + if (!boolQuery.mustNot().isEmpty()) { + leafReaderContexts = getLeafReaderContexts(context); + if (leafReaderContexts != null && !leafReaderContexts.isEmpty()) { + Map fieldCounts = new HashMap<>(); + + // Find complement-aware queries that can be rewritten + for (QueryBuilder clause : boolQuery.mustNot()) { + if (clause instanceof ComplementAwareQueryBuilder && clause instanceof WithFieldName) { + WithFieldName wfn = (WithFieldName) clause; + fieldCounts.merge(wfn.fieldName(), 1, Integer::sum); + } + } + + // For now, only handle the case where there's exactly 1 complement-aware query per field + for (QueryBuilder clause : boolQuery.mustNot()) { + if (clause instanceof ComplementAwareQueryBuilder && clause instanceof WithFieldName) { + WithFieldName wfn = (WithFieldName) clause; + String fieldName = wfn.fieldName(); + + if (fieldCounts.getOrDefault(fieldName, 0) == 1) { + // Check that all docs on this field have exactly 1 value + if (checkAllDocsHaveOneValue(leafReaderContexts, fieldName)) { + mustNotClausesToRewrite.add(clause); + } + } + } + } + } + } + + // Create a new BoolQueryBuilder with rewritten clauses + BoolQueryBuilder rewritten = new BoolQueryBuilder(); + + // Copy all properties + rewritten.boost(boolQuery.boost()); + rewritten.queryName(boolQuery.queryName()); + rewritten.minimumShouldMatch(boolQuery.minimumShouldMatch()); + rewritten.adjustPureNegative(boolQuery.adjustPureNegative()); + + // Copy must clauses (rewrite nested queries first) + for (QueryBuilder mustClause : boolQuery.must()) { + rewritten.must(rewrite(mustClause, context)); + } + + // Copy filter clauses (rewrite nested queries first) + for (QueryBuilder filterClause : boolQuery.filter()) { + rewritten.filter(rewrite(filterClause, context)); + } + + // Copy should clauses (rewrite nested queries first) + for (QueryBuilder shouldClause : boolQuery.should()) { + rewritten.should(rewrite(shouldClause, context)); + } + + // Process must_not clauses + boolean changed = false; + for (QueryBuilder mustNotClause : boolQuery.mustNot()) { + if (mustNotClausesToRewrite.contains(mustNotClause)) { + // Rewrite this clause + ComplementAwareQueryBuilder caq = (ComplementAwareQueryBuilder) mustNotClause; + List complement = caq.getComplement(context); + + if (complement != null && !complement.isEmpty()) { + BoolQueryBuilder nestedBoolQuery = new BoolQueryBuilder(); + nestedBoolQuery.minimumShouldMatch(1); + for (QueryBuilder complementComponent : complement) { + nestedBoolQuery.should(complementComponent); + } + rewritten.must(nestedBoolQuery); + changed = true; + } else { + // If complement couldn't be determined, keep original + rewritten.mustNot(mustNotClause); + } + } else { + // Keep clauses we're not rewriting + rewritten.mustNot(rewrite(mustNotClause, context)); + } + } + + // Handle minimumShouldMatch adjustment + if (changed && boolQuery.minimumShouldMatch() == null) { + if (!boolQuery.should().isEmpty() && boolQuery.must().isEmpty() && boolQuery.filter().isEmpty()) { + // If there were originally should clauses and no must/filter clauses, + // null minimumShouldMatch defaults to 1 in Lucene. + // But if there was originally a must or filter clause, the default is 0. + // If we added a must clause due to this rewrite, we should respect the original default. + rewritten.minimumShouldMatch(1); + } + } + + // Check if any nested queries were rewritten + boolean nestedQueriesChanged = false; + for (QueryBuilder mustClause : boolQuery.must()) { + if (mustClause instanceof BoolQueryBuilder && rewritten.must().contains(mustClause) == false) { + nestedQueriesChanged = true; + break; + } + } + if (!nestedQueriesChanged) { + for (QueryBuilder filterClause : boolQuery.filter()) { + if (filterClause instanceof BoolQueryBuilder && rewritten.filter().contains(filterClause) == false) { + nestedQueriesChanged = true; + break; + } + } + } + if (!nestedQueriesChanged) { + for (QueryBuilder shouldClause : boolQuery.should()) { + if (shouldClause instanceof BoolQueryBuilder && rewritten.should().contains(shouldClause) == false) { + nestedQueriesChanged = true; + break; + } + } + } + if (!nestedQueriesChanged) { + for (QueryBuilder mustNotClause : boolQuery.mustNot()) { + if (mustNotClause instanceof BoolQueryBuilder && rewritten.mustNot().contains(mustNotClause) == false) { + nestedQueriesChanged = true; + break; + } + } + } + + return (changed || nestedQueriesChanged) ? rewritten : query; + } + + private List getLeafReaderContexts(QueryShardContext context) { + if (context == null) { + return null; + } + try { + return context.getIndexReader().leaves(); + } catch (Exception e) { + return null; + } + } + + private boolean checkAllDocsHaveOneValue(List leafReaderContexts, String fieldName) { + try { + for (LeafReaderContext leafReaderContext : leafReaderContexts) { + PointValues pointValues = leafReaderContext.reader().getPointValues(fieldName); + if (pointValues != null) { + int docCount = pointValues.getDocCount(); + long valueCount = pointValues.size(); + // Check if all documents have exactly one value + if (docCount != valueCount) { + return false; + } + // Also check if all documents in the segment have a value for this field + // If some documents are missing the field, we can't do the optimization + // because the semantics change (missing values won't match positive queries) + int maxDoc = leafReaderContext.reader().maxDoc(); + if (docCount != maxDoc) { + return false; + } + } else { + // If there are no point values but there are documents, some docs are missing the field + if (leafReaderContext.reader().maxDoc() > 0) { + return false; + } + } + } + return true; + } catch (IOException e) { + return false; + } + } + + @Override + public int priority() { + // Run after boolean flattening (100) and must-to-filter (150) + // but before terms merging (200) + return 175; + } + + @Override + public String name() { + return "must_not_to_should"; + } +} diff --git a/server/src/main/java/org/opensearch/search/query/rewriters/MustToFilterRewriter.java b/server/src/main/java/org/opensearch/search/query/rewriters/MustToFilterRewriter.java new file mode 100644 index 0000000000000..4960302d0c812 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/rewriters/MustToFilterRewriter.java @@ -0,0 +1,177 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.GeoBoundingBoxQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.index.query.WithFieldName; +import org.opensearch.search.query.QueryRewriter; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rewrites must clauses to filter clauses when they don't affect scoring. + * This improves performance by avoiding unnecessary scoring calculations. + * + * For example: + *
+ * {"bool": {"must": [
+ *   {"range": {"date": {"gte": "2024-01-01"}}},
+ *   {"term": {"status": "active"}}
+ * ]}}
+ * 
+ * becomes: + *
+ * {"bool": {
+ *   "filter": [{"range": {"date": {"gte": "2024-01-01"}}}],
+ *   "must": [{"term": {"status": "active"}}]
+ * }}
+ * 
+ * + * @opensearch.internal + */ +public class MustToFilterRewriter implements QueryRewriter { + + public static final MustToFilterRewriter INSTANCE = new MustToFilterRewriter(); + + private MustToFilterRewriter() { + // Singleton + } + + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (!(query instanceof BoolQueryBuilder)) { + return query; + } + + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // If there are no must clauses, nothing to rewrite + if (boolQuery.must().isEmpty()) { + return query; + } + + // First, rewrite all clauses recursively + List rewrittenMustClauses = new ArrayList<>(); + List mustClausesToMove = new ArrayList<>(); + + for (QueryBuilder clause : boolQuery.must()) { + QueryBuilder rewrittenClause = rewriteIfNeeded(clause, context); + rewrittenMustClauses.add(rewrittenClause); + + if (isClauseIrrelevantToScoring(rewrittenClause, context)) { + mustClausesToMove.add(rewrittenClause); + } + } + + // Check if anything changed - either clauses to move or nested rewrites + boolean hasChanges = !mustClausesToMove.isEmpty(); + for (int i = 0; i < boolQuery.must().size(); i++) { + if (boolQuery.must().get(i) != rewrittenMustClauses.get(i)) { + hasChanges = true; + break; + } + } + + if (!hasChanges) { + return query; + } + + // Create a new BoolQueryBuilder with moved clauses + BoolQueryBuilder rewritten = new BoolQueryBuilder(); + + // Copy all properties + rewritten.boost(boolQuery.boost()); + rewritten.queryName(boolQuery.queryName()); + rewritten.minimumShouldMatch(boolQuery.minimumShouldMatch()); + rewritten.adjustPureNegative(boolQuery.adjustPureNegative()); + + // Copy must clauses except the ones we're moving + for (QueryBuilder rewrittenClause : rewrittenMustClauses) { + if (!mustClausesToMove.contains(rewrittenClause)) { + rewritten.must(rewrittenClause); + } + } + + // Add the moved clauses to filter + for (QueryBuilder movedClause : mustClausesToMove) { + rewritten.filter(movedClause); + } + + // Copy existing filter clauses + for (QueryBuilder filterClause : boolQuery.filter()) { + rewritten.filter(rewriteIfNeeded(filterClause, context)); + } + + // Copy should and mustNot clauses + for (QueryBuilder shouldClause : boolQuery.should()) { + rewritten.should(rewriteIfNeeded(shouldClause, context)); + } + for (QueryBuilder mustNotClause : boolQuery.mustNot()) { + rewritten.mustNot(rewriteIfNeeded(mustNotClause, context)); + } + + return rewritten; + } + + private QueryBuilder rewriteIfNeeded(QueryBuilder query, QueryShardContext context) { + // Recursively rewrite nested boolean queries + if (query instanceof BoolQueryBuilder) { + return rewrite(query, context); + } + return query; + } + + private boolean isClauseIrrelevantToScoring(QueryBuilder clause, QueryShardContext context) { + // This is an incomplete list of clauses this might apply for; it can be expanded in future. + + // If a clause is purely numeric, for example a date range, its score is unimportant as + // it'll be the same for all returned docs + if (clause instanceof RangeQueryBuilder) return true; + if (clause instanceof GeoBoundingBoxQueryBuilder) return true; + + // Further optimizations depend on knowing whether the field is numeric. + // Skip moving these clauses if we don't have the shard context. + if (context == null) return false; + + if (!(clause instanceof WithFieldName)) return false; + + WithFieldName wfn = (WithFieldName) clause; + MappedFieldType fieldType = context.fieldMapper(wfn.fieldName()); + + if (!(fieldType instanceof NumberFieldMapper.NumberFieldType)) return false; + + // Numeric field queries have constant scores + if (clause instanceof MatchQueryBuilder) return true; + if (clause instanceof TermQueryBuilder) return true; + if (clause instanceof TermsQueryBuilder) return true; + + return false; + } + + @Override + public int priority() { + // Run after boolean flattening (100) but before terms merging (200) + return 150; + } + + @Override + public String name() { + return "must_to_filter"; + } +} diff --git a/server/src/main/java/org/opensearch/search/query/rewriters/TermsMergingRewriter.java b/server/src/main/java/org/opensearch/search/query/rewriters/TermsMergingRewriter.java new file mode 100644 index 0000000000000..9c4d863602091 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/rewriters/TermsMergingRewriter.java @@ -0,0 +1,314 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.SearchService; +import org.opensearch.search.query.QueryRewriter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Rewrites multiple term queries on the same field into a single terms query. + * For example: + *
+ * {"bool": {"filter": [
+ *   {"term": {"status": "active"}},
+ *   {"term": {"status": "pending"}},
+ *   {"term": {"category": "A"}}
+ * ]}}
+ * 
+ * becomes: + *
+ * {"bool": {"filter": [
+ *   {"terms": {"status": ["active", "pending"]}},
+ *   {"term": {"category": "A"}}
+ * ]}}
+ * 
+ * + * Note: Terms are only merged when there are enough terms to benefit from + * the terms query's bit set optimization (default threshold: 16 terms). + * This avoids performance regressions for small numbers of terms where + * individual term queries may perform better. + * + * @opensearch.internal + */ +public class TermsMergingRewriter implements QueryRewriter { + + public static final TermsMergingRewriter INSTANCE = new TermsMergingRewriter(); + + /** + * Default minimum number of terms to merge. Below this threshold, individual + * term queries may perform better than a terms query. + * Based on Lucene's TermInSetQuery optimization characteristics. + */ + private static final int DEFAULT_MINIMUM_TERMS_TO_MERGE = 16; + + /** + * The minimum number of terms to merge. + */ + private volatile int minimumTermsToMerge = DEFAULT_MINIMUM_TERMS_TO_MERGE; + + /** + * Creates a new rewriter. + */ + private TermsMergingRewriter() { + // Singleton + } + + /** + * Initialize this rewriter with cluster settings. + * This registers an update consumer to keep the threshold in sync with the cluster setting. + * + * @param settings Initial settings + * @param clusterSettings Cluster settings to register update consumer + */ + public void initialize(Settings settings, ClusterSettings clusterSettings) { + this.minimumTermsToMerge = SearchService.QUERY_REWRITING_TERMS_THRESHOLD_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer( + SearchService.QUERY_REWRITING_TERMS_THRESHOLD_SETTING, + threshold -> this.minimumTermsToMerge = threshold + ); + } + + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (!(query instanceof BoolQueryBuilder)) { + return query; + } + + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + + // First check if merging is needed + if (!needsMerging(boolQuery)) { + return query; + } + + BoolQueryBuilder rewritten = new BoolQueryBuilder(); + + rewritten.boost(boolQuery.boost()); + rewritten.queryName(boolQuery.queryName()); + rewritten.minimumShouldMatch(boolQuery.minimumShouldMatch()); + rewritten.adjustPureNegative(boolQuery.adjustPureNegative()); + + // Only merge terms in contexts where it's semantically safe + rewriteClausesNoMerge(boolQuery.must(), rewritten::must); // Don't merge in must + rewriteClauses(boolQuery.filter(), rewritten::filter); // Safe to merge + rewriteClauses(boolQuery.should(), rewritten::should); // Safe to merge + rewriteClausesNoMerge(boolQuery.mustNot(), rewritten::mustNot); // Don't merge in mustNot + + return rewritten; + } + + private boolean needsMerging(BoolQueryBuilder boolQuery) { + // Check filter and should clauses for mergeable terms + if (hasMergeableTerms(boolQuery.filter()) || hasMergeableTerms(boolQuery.should())) { + return true; + } + + // Check nested bool queries + return hasNestedBoolThatNeedsMerging(boolQuery); + } + + private boolean hasMergeableTerms(List clauses) { + Map> fieldBoosts = new HashMap<>(); + + for (QueryBuilder clause : clauses) { + if (clause instanceof TermQueryBuilder) { + TermQueryBuilder termQuery = (TermQueryBuilder) clause; + String field = termQuery.fieldName(); + float boost = termQuery.boost(); + + fieldBoosts.computeIfAbsent(field, k -> new ArrayList<>()).add(boost); + + List boosts = fieldBoosts.get(field); + if (boosts.size() >= minimumTermsToMerge) { + // Check if all boosts are the same + float firstBoost = boosts.get(0); + boolean sameBoost = boosts.stream().allMatch(b -> b == firstBoost); + if (sameBoost) { + return true; + } + } + } else if (clause instanceof TermsQueryBuilder) { + // Check if there are enough term queries that can be merged with this terms query + TermsQueryBuilder termsQuery = (TermsQueryBuilder) clause; + String field = termsQuery.fieldName(); + int additionalTerms = 0; + + for (QueryBuilder other : clauses) { + if (other != clause && other instanceof TermQueryBuilder) { + TermQueryBuilder termQuery = (TermQueryBuilder) other; + if (field.equals(termQuery.fieldName()) && termsQuery.boost() == termQuery.boost()) { + additionalTerms++; + } + } + } + + // Only worth merging if the combined size would meet the threshold + if (termsQuery.values().size() + additionalTerms >= minimumTermsToMerge) { + return true; + } + } + } + + return false; + } + + private boolean hasNestedBoolThatNeedsMerging(BoolQueryBuilder boolQuery) { + for (QueryBuilder clause : boolQuery.must()) { + if (clause instanceof BoolQueryBuilder && needsMerging((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.filter()) { + if (clause instanceof BoolQueryBuilder && needsMerging((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.should()) { + if (clause instanceof BoolQueryBuilder && needsMerging((BoolQueryBuilder) clause)) { + return true; + } + } + for (QueryBuilder clause : boolQuery.mustNot()) { + if (clause instanceof BoolQueryBuilder && needsMerging((BoolQueryBuilder) clause)) { + return true; + } + } + return false; + } + + private void rewriteClauses(List clauses, ClauseAdder adder) { + Map termsMap = new HashMap<>(); + List nonTermClauses = new ArrayList<>(); + + // Group term queries by field + for (QueryBuilder clause : clauses) { + if (clause instanceof TermQueryBuilder) { + TermQueryBuilder termQuery = (TermQueryBuilder) clause; + String field = termQuery.fieldName(); + float boost = termQuery.boost(); + + TermsInfo info = termsMap.get(field); + if (info != null && info.boost != boost) { + // Different boost, can't merge - add as single term + nonTermClauses.add(clause); + } else { + termsMap.computeIfAbsent(field, k -> new TermsInfo(boost)).addValue(termQuery.value()); + } + } else if (clause instanceof TermsQueryBuilder) { + // Existing terms query - add to it + TermsQueryBuilder termsQuery = (TermsQueryBuilder) clause; + String field = termsQuery.fieldName(); + float boost = termsQuery.boost(); + + TermsInfo info = termsMap.get(field); + if (info != null && info.boost != boost) { + // Different boost, can't merge + nonTermClauses.add(clause); + } else { + info = termsMap.computeIfAbsent(field, k -> new TermsInfo(boost)); + for (Object value : termsQuery.values()) { + info.addValue(value); + } + } + } else if (clause instanceof BoolQueryBuilder) { + // Recursively rewrite nested bool queries + nonTermClauses.add(rewrite(clause, null)); + } else { + nonTermClauses.add(clause); + } + } + + // Create terms queries for fields with multiple values + for (Map.Entry entry : termsMap.entrySet()) { + String field = entry.getKey(); + TermsInfo info = entry.getValue(); + + if (info.values.size() == 1) { + // Single value, keep as term query + TermQueryBuilder termQuery = new TermQueryBuilder(field, info.values.get(0)); + if (info.boost != 1.0f) { + termQuery.boost(info.boost); + } + adder.addClause(termQuery); + } else if (info.values.size() >= minimumTermsToMerge) { + // Many values, merge into terms query for better performance + TermsQueryBuilder termsQuery = new TermsQueryBuilder(field, info.values); + if (info.boost != 1.0f) { + termsQuery.boost(info.boost); + } + adder.addClause(termsQuery); + } else { + // Few values, keep as individual term queries for better performance + for (Object value : info.values) { + TermQueryBuilder termQuery = new TermQueryBuilder(field, value); + if (info.boost != 1.0f) { + termQuery.boost(info.boost); + } + adder.addClause(termQuery); + } + } + } + + // Add non-term clauses + for (QueryBuilder clause : nonTermClauses) { + adder.addClause(clause); + } + } + + private void rewriteClausesNoMerge(List clauses, ClauseAdder adder) { + for (QueryBuilder clause : clauses) { + if (clause instanceof BoolQueryBuilder) { + // Recursively rewrite nested bool queries + adder.addClause(rewrite(clause, null)); + } else { + adder.addClause(clause); + } + } + } + + @Override + public int priority() { + return 200; + } + + @Override + public String name() { + return "terms_merging"; + } + + @FunctionalInterface + private interface ClauseAdder { + void addClause(QueryBuilder clause); + } + + private static class TermsInfo { + final float boost; + final List values = new ArrayList<>(); + + TermsInfo(float boost) { + this.boost = boost; + } + + void addValue(Object value) { + values.add(value); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/query/rewriters/package-info.java b/server/src/main/java/org/opensearch/search/query/rewriters/package-info.java new file mode 100644 index 0000000000000..167d97e097185 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/rewriters/package-info.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Query rewriting optimizations for improving search performance. + * + *

This package contains various query rewriters that transform queries + * into more efficient forms while maintaining semantic equivalence. + * + *

The rewriters include: + *

    + *
  • {@link org.opensearch.search.query.rewriters.BooleanFlatteningRewriter} - + * Flattens nested boolean queries with single clauses
  • + *
  • {@link org.opensearch.search.query.rewriters.MatchAllRemovalRewriter} - + * Removes redundant match_all queries from boolean clauses
  • + *
  • {@link org.opensearch.search.query.rewriters.TermsMergingRewriter} - + * Merges multiple term queries on the same field into a single terms query
  • + *
  • {@link org.opensearch.search.query.rewriters.MustNotToShouldRewriter} - + * Transforms must_not queries to should queries for better performance
  • + *
  • {@link org.opensearch.search.query.rewriters.MustToFilterRewriter} - + * Moves scoring-irrelevant queries from must to filter clauses
  • + *
+ * + * @opensearch.internal + */ +package org.opensearch.search.query.rewriters; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 0b1cac07b0a10..e66f8fe14dd04 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -382,7 +382,7 @@ public ClusterState execute(ClusterState currentState) { IndexId snapshotIndexId = repositoryData.resolveIndexId(index); - final Settings overrideSettingsInternal = getOverrideSettingsInternal(); + final Settings overrideSettingsInternal = getOverrideSettingsInternal(metadata.index(index)); final String[] ignoreSettingsInternal = getIgnoreSettingsInternal(); IndexMetadata snapshotIndexMetadata = updateIndexSettings( @@ -688,7 +688,7 @@ private String[] getIgnoreSettingsInternal() { return indexSettingsToBeIgnored; } - private Settings getOverrideSettingsInternal() { + private Settings getOverrideSettingsInternal(IndexMetadata indexMetadata) { final Settings.Builder settingsBuilder = Settings.builder(); // We will use whatever replication strategy provided by user or from snapshot metadata unless @@ -712,7 +712,8 @@ private Settings getOverrideSettingsInternal() { clusterService.state(), clusterSettings, clusterService.getSettings(), - String.join(",", request.indices()) + String.join(",", request.indices()), + indexMetadata ); return settingsBuilder.build(); } diff --git a/server/src/main/java/org/opensearch/transport/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/transport/client/IndicesAdminClient.java index 6b8d168ecbbda..46b270e7eec7c 100644 --- a/server/src/main/java/org/opensearch/transport/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/transport/client/IndicesAdminClient.java @@ -142,6 +142,9 @@ import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.action.ActionListener; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; + /** * Administrative actions/operations against indices. * @@ -899,4 +902,306 @@ public interface IndicesAdminClient extends OpenSearchClient { * @return The request builder configured with the specified scaling direction */ ScaleIndexRequestBuilder prepareScaleSearchOnly(String index, boolean searchOnly); + + /** Indices Exists - CompletionStage version */ + default CompletionStage existsAsync(IndicesExistsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + exists(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Indices stats - CompletionStage version */ + default CompletionStage statsAsync(IndicesStatsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + stats(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Recoveries - CompletionStage version */ + default CompletionStage recoveriesAsync(RecoveryRequest request) { + CompletableFuture future = new CompletableFuture<>(); + recoveries(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Segment replication stats - CompletionStage version */ + default CompletionStage segmentReplicationStatsAsync(SegmentReplicationStatsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + segmentReplicationStats(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Segments - CompletionStage version */ + default CompletionStage segmentsAsync(IndicesSegmentsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + segments(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Shard stores - CompletionStage version */ + default CompletionStage shardStoresAsync(IndicesShardStoresRequest request) { + CompletableFuture future = new CompletableFuture<>(); + shardStores(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Create index - CompletionStage version */ + default CompletionStage createAsync(CreateIndexRequest request) { + CompletableFuture future = new CompletableFuture<>(); + create(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Delete index - CompletionStage version */ + default CompletionStage deleteAsync(DeleteIndexRequest request) { + CompletableFuture future = new CompletableFuture<>(); + delete(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Close index - CompletionStage version */ + default CompletionStage closeAsync(CloseIndexRequest request) { + CompletableFuture future = new CompletableFuture<>(); + close(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Open index - CompletionStage version */ + default CompletionStage openAsync(OpenIndexRequest request) { + CompletableFuture future = new CompletableFuture<>(); + open(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Add index block - CompletionStage version */ + default CompletionStage addBlockAsync(AddIndexBlockRequest request) { + CompletableFuture future = new CompletableFuture<>(); + addBlock(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Refresh - CompletionStage version */ + default CompletionStage refreshAsync(RefreshRequest request) { + CompletableFuture future = new CompletableFuture<>(); + refresh(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Flush - CompletionStage version */ + default CompletionStage flushAsync(FlushRequest request) { + CompletableFuture future = new CompletableFuture<>(); + flush(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Force-merge - CompletionStage version */ + default CompletionStage forceMergeAsync(ForceMergeRequest request) { + CompletableFuture future = new CompletableFuture<>(); + forceMerge(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Upgrade - CompletionStage version */ + default CompletionStage upgradeAsync(UpgradeRequest request) { + CompletableFuture future = new CompletableFuture<>(); + upgrade(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Upgrade status - CompletionStage version */ + default CompletionStage upgradeStatusAsync(UpgradeStatusRequest request) { + CompletableFuture future = new CompletableFuture<>(); + upgradeStatus(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get mappings - CompletionStage version */ + default CompletionStage getMappingsAsync(GetMappingsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getMappings(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get field mappings - CompletionStage version */ + default CompletionStage getFieldMappingsAsync(GetFieldMappingsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getFieldMappings(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Put mapping - CompletionStage version */ + default CompletionStage putMappingAsync(PutMappingRequest request) { + CompletableFuture future = new CompletableFuture<>(); + putMapping(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Aliases - CompletionStage version */ + default CompletionStage aliasesAsync(IndicesAliasesRequest request) { + CompletableFuture future = new CompletableFuture<>(); + aliases(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get aliases - CompletionStage version */ + default CompletionStage getAliasesAsync(GetAliasesRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getAliases(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get index - CompletionStage version */ + default CompletionStage getIndexAsync(GetIndexRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getIndex(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Clear cache - CompletionStage version */ + default CompletionStage clearCacheAsync(ClearIndicesCacheRequest request) { + CompletableFuture future = new CompletableFuture<>(); + clearCache(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Update settings - CompletionStage version */ + default CompletionStage updateSettingsAsync(UpdateSettingsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + updateSettings(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get settings - CompletionStage version */ + default CompletionStage getSettingsAsync(GetSettingsRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getSettings(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Analyze - CompletionStage version */ + default CompletionStage analyzeAsync(AnalyzeAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + analyze(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Put template - CompletionStage version */ + default CompletionStage putTemplateAsync(PutIndexTemplateRequest request) { + CompletableFuture future = new CompletableFuture<>(); + putTemplate(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Delete template - CompletionStage version */ + default CompletionStage deleteTemplateAsync(DeleteIndexTemplateRequest request) { + CompletableFuture future = new CompletableFuture<>(); + deleteTemplate(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get templates - CompletionStage version */ + default CompletionStage getTemplatesAsync(GetIndexTemplatesRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getTemplates(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Validate query - CompletionStage version */ + default CompletionStage validateQueryAsync(ValidateQueryRequest request) { + CompletableFuture future = new CompletableFuture<>(); + validateQuery(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Resize index - CompletionStage version */ + default CompletionStage resizeIndexAsync(ResizeRequest request) { + CompletableFuture future = new CompletableFuture<>(); + resizeIndex(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Rollover index - CompletionStage version */ + default CompletionStage rolloverIndexAsync(RolloverRequest request) { + CompletableFuture future = new CompletableFuture<>(); + rolloverIndex(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Create data stream - CompletionStage version */ + default CompletionStage createDataStreamAsync(CreateDataStreamAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + createDataStream(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Delete data stream - CompletionStage version */ + default CompletionStage deleteDataStreamAsync(DeleteDataStreamAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + deleteDataStream(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get data streams - CompletionStage version */ + default CompletionStage getDataStreamsAsync(GetDataStreamAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + getDataStreams(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Resolve index - CompletionStage version */ + default CompletionStage resolveIndexAsync(ResolveIndexAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + resolveIndex(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Create view - CompletionStage version */ + default CompletionStage createViewAsync(CreateViewAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + createView(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get view - CompletionStage version */ + default CompletionStage getViewAsync(GetViewAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + getView(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Delete view - CompletionStage version */ + default CompletionStage deleteViewAsync(DeleteViewAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + deleteView(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Update view - CompletionStage version */ + default CompletionStage updateViewAsync(CreateViewAction.Request request) { + CompletableFuture future = new CompletableFuture<>(); + updateView(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Pause ingestion - CompletionStage version */ + default CompletionStage pauseIngestionAsync(PauseIngestionRequest request) { + CompletableFuture future = new CompletableFuture<>(); + pauseIngestion(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Resume ingestion - CompletionStage version */ + default CompletionStage resumeIngestionAsync(ResumeIngestionRequest request) { + CompletableFuture future = new CompletableFuture<>(); + resumeIngestion(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + + /** Get ingestion state - CompletionStage version */ + default CompletionStage getIngestionStateAsync(GetIngestionStateRequest request) { + CompletableFuture future = new CompletableFuture<>(); + getIngestionState(request, ActionListener.wrap(future::complete, future::completeExceptionally)); + return future; + } + } diff --git a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java index 9115ccadf2998..a4eb8f7548be5 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchPhaseControllerTests.java @@ -1826,7 +1826,13 @@ private static void consumeShardLevelQueryPhaseResultsAsync(int expectedNumResul result.setShardIndex(index); result.size(1); - consumer.consumeResult(result, latch::countDown); + try { + consumer.consumeResult(result, latch::countDown); + } catch (Exception e) { + // Ensure latch counts down even on cancellation + latch.countDown(); + // Don't rethrow - let the thread complete normally + } }); threads[index].start(); } diff --git a/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java b/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java index 92f8e132b691b..4ad7cc324c618 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java +++ b/server/src/test/java/org/opensearch/action/termvectors/AbstractTermVectorsTestCase.java @@ -223,9 +223,8 @@ public String toString() { if (requestPayloads) { requested += "payload,"; } - Locale aLocale = new Locale("en", "US"); return String.format( - aLocale, + Locale.US, "(doc: %s\n requested: %s, fields: %s)", doc, requested, diff --git a/server/src/test/java/org/opensearch/action/termvectors/GetTermVectorsTests.java b/server/src/test/java/org/opensearch/action/termvectors/GetTermVectorsTests.java index 7dd73966bb079..88ecd0f94e1a2 100644 --- a/server/src/test/java/org/opensearch/action/termvectors/GetTermVectorsTests.java +++ b/server/src/test/java/org/opensearch/action/termvectors/GetTermVectorsTests.java @@ -184,7 +184,7 @@ public void testRandomPayloadWithDelimitedPayloadTokenFilter() throws IOExceptio .put("index.analysis.filter.my_delimited_payload.encoding", encodingString) .put("index.analysis.filter.my_delimited_payload.type", "mock_payload_filter") .build(); - createIndex("test", setting, "type1", mapping); + createIndex("test", setting, mapping); client().prepareIndex("test") .setId(Integer.toString(1)) diff --git a/server/src/test/java/org/opensearch/bootstrap/SecurityTests.java b/server/src/test/java/org/opensearch/bootstrap/SecurityTests.java index 9bd5e46fe50a5..738fc80c34ce6 100644 --- a/server/src/test/java/org/opensearch/bootstrap/SecurityTests.java +++ b/server/src/test/java/org/opensearch/bootstrap/SecurityTests.java @@ -35,6 +35,7 @@ import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -80,7 +81,7 @@ public void testEnsureRegularFile() throws IOException { public void testProcessExecution() throws Exception { assumeTrue("test requires security manager", System.getSecurityManager() != null); try { - Runtime.getRuntime().exec("ls"); + Runtime.getRuntime().exec(new String[] { "ls" }); fail("didn't get expected exception"); } catch (SecurityException expected) {} } @@ -89,15 +90,15 @@ public void testProcessExecution() throws Exception { public void testReadPolicyWithCodebases() throws IOException { final Map codebases = Map.of( "test-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar", - new URL("file://test-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar"), + URI.create("file://test-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar").toURL(), "test-kafka-server-common-3.6.1.jar", - new URL("file://test-kafka-server-common-3.6.1.jar"), + URI.create("file://test-kafka-server-common-3.6.1.jar").toURL(), "test-kafka-server-common-3.6.1-test.jar", - new URL("file://test-kafka-server-common-3.6.1-test.jar"), + URI.create("file://test-kafka-server-common-3.6.1-test.jar").toURL(), "test-lucene-core-9.11.0-snapshot-8a555eb.jar", - new URL("file://test-lucene-core-9.11.0-snapshot-8a555eb.jar"), + URI.create("file://test-lucene-core-9.11.0-snapshot-8a555eb.jar").toURL(), "test-zstd-jni-1.5.6-1.jar", - new URL("file://test-zstd-jni-1.5.6-1.jar") + URI.create("file://test-zstd-jni-1.5.6-1.jar").toURL() ); AccessController.doPrivileged( diff --git a/server/src/test/java/org/opensearch/common/cache/CacheTests.java b/server/src/test/java/org/opensearch/common/cache/CacheTests.java index f6277a7139c7e..65aa5931f144c 100644 --- a/server/src/test/java/org/opensearch/common/cache/CacheTests.java +++ b/server/src/test/java/org/opensearch/common/cache/CacheTests.java @@ -774,7 +774,7 @@ public int hashCode() { // start a watchdog service ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); scheduler.scheduleAtFixedRate(() -> { - Set ids = threads.stream().map(t -> t.getId()).collect(Collectors.toSet()); + Set ids = threads.stream().map(Thread::threadId).collect(Collectors.toSet()); ThreadMXBean mxBean = ManagementFactory.getThreadMXBean(); long[] deadlockedThreads = mxBean.findDeadlockedThreads(); if (!deadlock.get() && deadlockedThreads != null) { diff --git a/server/src/test/java/org/opensearch/env/EnvironmentTests.java b/server/src/test/java/org/opensearch/env/EnvironmentTests.java index 0e343a6e43ba7..ebbd17fc636a2 100644 --- a/server/src/test/java/org/opensearch/env/EnvironmentTests.java +++ b/server/src/test/java/org/opensearch/env/EnvironmentTests.java @@ -38,7 +38,7 @@ import java.io.FileNotFoundException; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.nio.file.Path; import java.util.List; @@ -86,17 +86,20 @@ public void testRepositoryResolution() throws IOException { assertThat(environment.resolveRepoFile("/somethingeles/repos/repo1"), nullValue()); assertThat(environment.resolveRepoFile("/test/other/repo"), notNullValue()); - assertThat(environment.resolveRepoURL(new URL("file:///test/repos/repo1")), notNullValue()); - assertThat(environment.resolveRepoURL(new URL("file:/test/repos/repo1")), notNullValue()); - assertThat(environment.resolveRepoURL(new URL("file://test/repos/repo1")), nullValue()); - assertThat(environment.resolveRepoURL(new URL("file:///test/repos/../repo1")), nullValue()); - assertThat(environment.resolveRepoURL(new URL("http://localhost/test/")), nullValue()); - - assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/repo1!/repo/")), notNullValue()); - assertThat(environment.resolveRepoURL(new URL("jar:file:/test/repos/repo1!/repo/")), notNullValue()); - assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/repo1!/repo/")).toString(), endsWith("repo1!/repo/")); - assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/../repo1!/repo/")), nullValue()); - assertThat(environment.resolveRepoURL(new URL("jar:http://localhost/test/../repo1?blah!/repo/")), nullValue()); + assertThat(environment.resolveRepoURL(URI.create("file:///test/repos/repo1").toURL()), notNullValue()); + assertThat(environment.resolveRepoURL(URI.create("file:/test/repos/repo1").toURL()), notNullValue()); + assertThat(environment.resolveRepoURL(URI.create("file://test/repos/repo1").toURL()), nullValue()); + assertThat(environment.resolveRepoURL(URI.create("file:///test/repos/../repo1").toURL()), nullValue()); + assertThat(environment.resolveRepoURL(URI.create("http://localhost/test/").toURL()), nullValue()); + + assertThat(environment.resolveRepoURL(URI.create("jar:file:///test/repos/repo1!/repo/").toURL()), notNullValue()); + assertThat(environment.resolveRepoURL(URI.create("jar:file:/test/repos/repo1!/repo/").toURL()), notNullValue()); + assertThat( + environment.resolveRepoURL(URI.create("jar:file:///test/repos/repo1!/repo/").toURL()).toString(), + endsWith("repo1!/repo/") + ); + assertThat(environment.resolveRepoURL(URI.create("jar:file:///test/repos/../repo1!/repo/").toURL()), nullValue()); + assertThat(environment.resolveRepoURL(URI.create("jar:http://localhost/test/../repo1?blah!/repo/").toURL()), nullValue()); } public void testPathDataWhenNotSet() { diff --git a/server/src/test/java/org/opensearch/index/analysis/PreBuiltAnalyzerTests.java b/server/src/test/java/org/opensearch/index/analysis/PreBuiltAnalyzerTests.java index 6f9a662caff46..d8fb8603484b6 100644 --- a/server/src/test/java/org/opensearch/index/analysis/PreBuiltAnalyzerTests.java +++ b/server/src/test/java/org/opensearch/index/analysis/PreBuiltAnalyzerTests.java @@ -127,7 +127,7 @@ public void testThatAnalyzersAreUsedInMapping() throws IOException { .endObject() .endObject() .endObject(); - MapperService mapperService = createIndex("test", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("test", indexSettings, mapping).mapperService(); MappedFieldType fieldType = mapperService.fieldType("field"); assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer(), instanceOf(NamedAnalyzer.class)); diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index d004d5aa90eac..bb5a1eb568108 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -4182,7 +4182,8 @@ public void testRecoverFromForeignTranslog() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); translog.add(new Translog.Index("SomeBogusId", 0, primaryTerm.get(), "{}".getBytes(Charset.forName("UTF-8")))); assertEquals(generation.translogFileGeneration, translog.currentFileGeneration()); diff --git a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java index ec670ec969bad..9fcadcfb36b69 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java @@ -156,7 +156,7 @@ public void testDerivedValueFetching() throws IOException { } private ConstantKeywordFieldMapper getMapper(FieldMapper.CopyTo copyTo) { - indexService = createIndex("test-index", Settings.EMPTY, "constant_keyword", "field", "type=constant_keyword,value=default_value"); + indexService = createIndexWithSimpleMappings("test-index", Settings.EMPTY, "field", "type=constant_keyword,value=default_value"); ConstantKeywordFieldMapper mapper = (ConstantKeywordFieldMapper) indexService.mapperService() .documentMapper() .mappers() diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java index bc0fa4e96d011..24f9f7c78372c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java @@ -235,10 +235,10 @@ public void testIndexSortWithNestedFieldsWithOlderVersion() throws IOException { Settings settings = settings(Version.V_3_0_0).put("index.sort.field", "foo").build(); IllegalArgumentException invalidNestedException = expectThrows( IllegalArgumentException.class, - () -> createIndex("test", settings, "t", "nested_field", "type=nested", "foo", "type=keyword") + () -> createIndexWithSimpleMappings("test", settings, "nested_field", "type=nested", "foo", "type=keyword") ); assertThat(invalidNestedException.getMessage(), containsString("cannot have nested fields when index sort is activated")); - IndexService indexService = createIndex("test", settings, "t", "foo", "type=keyword"); + IndexService indexService = createIndexWithSimpleMappings("test", settings, "foo", "type=keyword"); CompressedXContent nestedFieldMapping = new CompressedXContent( BytesReference.bytes( XContentFactory.jsonBuilder() diff --git a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java index 9a0d34c916f5c..cf668178b3df0 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NestedObjectMapperTests.java @@ -881,7 +881,6 @@ public void testParentObjectMapperAreNested() throws Exception { MapperService mapperService = createIndex( "index1", Settings.EMPTY, - "_doc", jsonBuilder().startObject() .startObject("properties") .startObject("comments") @@ -901,7 +900,6 @@ public void testParentObjectMapperAreNested() throws Exception { mapperService = createIndex( "index2", Settings.EMPTY, - "_doc", jsonBuilder().startObject() .startObject("properties") .startObject("comments") @@ -1107,7 +1105,6 @@ public void testMergeNestedMappings() throws IOException { MapperService mapperService = createIndex( "index1", Settings.EMPTY, - MapperService.SINGLE_MAPPING_NAME, jsonBuilder().startObject() .startObject("properties") .startObject("nested1") diff --git a/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java b/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java index 7e40354eb7f29..b5d6922a12fcc 100644 --- a/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/UpdateMappingTests.java @@ -83,7 +83,7 @@ public void testConflictFieldsMapping(String fieldName) throws Exception { } protected void testConflictWhileMergingAndMappingUnchanged(XContentBuilder mapping, XContentBuilder mappingUpdate) throws IOException { - IndexService indexService = createIndex("test", Settings.builder().build(), MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("test", Settings.builder().build(), mapping); CompressedXContent mappingBeforeUpdate = indexService.mapperService().documentMapper().mappingSource(); // simulate like in MetadataMappingService#putMapping try { @@ -111,8 +111,7 @@ public void testConflictSameType() throws Exception { .endObject() .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.builder().build(), MapperService.SINGLE_MAPPING_NAME, mapping) - .mapperService(); + MapperService mapperService = createIndex("test", Settings.builder().build(), mapping).mapperService(); XContentBuilder update = XContentFactory.jsonBuilder() .startObject() @@ -158,8 +157,7 @@ public void testConflictNewType() throws Exception { .endObject() .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.builder().build(), MapperService.SINGLE_MAPPING_NAME, mapping) - .mapperService(); + MapperService mapperService = createIndex("test", Settings.builder().build(), mapping).mapperService(); XContentBuilder update = XContentFactory.jsonBuilder() .startObject() diff --git a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java index cb83b8e1986b9..85e1d0f00c661 100644 --- a/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/BoolQueryBuilderTests.java @@ -73,7 +73,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class BoolQueryBuilderTests extends AbstractQueryTestCase { @Override @@ -517,63 +516,6 @@ public void testVisit() { } - public void testOneMustNotRangeRewritten() throws Exception { - int from = 10; - int to = 20; - Directory dir = newDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new StandardAnalyzer())); - addDocument(w, INT_FIELD_NAME, 1); - DirectoryReader reader = DirectoryReader.open(w); - IndexSearcher searcher = getIndexSearcher(reader); - - for (boolean includeLower : new boolean[] { true, false }) { - for (boolean includeUpper : new boolean[] { true, false }) { - BoolQueryBuilder qb = new BoolQueryBuilder(); - QueryBuilder rq = getRangeQueryBuilder(INT_FIELD_NAME, from, to, includeLower, includeUpper); - qb.mustNot(rq); - - BoolQueryBuilder rewritten = (BoolQueryBuilder) Rewriteable.rewrite(qb, createShardContext(searcher)); - assertFalse(rewritten.mustNot().contains(rq)); - - QueryBuilder expectedLowerQuery = getRangeQueryBuilder(INT_FIELD_NAME, null, from, false, !includeLower); - QueryBuilder expectedUpperQuery = getRangeQueryBuilder(INT_FIELD_NAME, to, null, !includeUpper, true); - assertEquals(1, rewritten.must().size()); - - BoolQueryBuilder nestedBoolQuery = (BoolQueryBuilder) rewritten.must().get(0); - assertEquals(2, nestedBoolQuery.should().size()); - assertEquals("1", nestedBoolQuery.minimumShouldMatch()); - assertTrue(nestedBoolQuery.should().contains(expectedLowerQuery)); - assertTrue(nestedBoolQuery.should().contains(expectedUpperQuery)); - } - } - IOUtils.close(w, reader, dir); - } - - public void testOneSingleEndedMustNotRangeRewritten() throws Exception { - // Test a must_not range query with only one endpoint is rewritten correctly - int from = 10; - Directory dir = newDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new StandardAnalyzer())); - addDocument(w, INT_FIELD_NAME, 1); - DirectoryReader reader = DirectoryReader.open(w); - IndexSearcher searcher = getIndexSearcher(reader); - - BoolQueryBuilder qb = new BoolQueryBuilder(); - QueryBuilder rq = getRangeQueryBuilder(INT_FIELD_NAME, from, null, false, false); - qb.mustNot(rq); - BoolQueryBuilder rewritten = (BoolQueryBuilder) Rewriteable.rewrite(qb, createShardContext(searcher)); - assertFalse(rewritten.mustNot().contains(rq)); - - QueryBuilder expectedQuery = getRangeQueryBuilder(INT_FIELD_NAME, null, from, false, true); - assertEquals(1, rewritten.must().size()); - BoolQueryBuilder nestedBoolQuery = (BoolQueryBuilder) rewritten.must().get(0); - assertEquals(1, nestedBoolQuery.should().size()); - assertTrue(nestedBoolQuery.should().contains(expectedQuery)); - assertEquals("1", nestedBoolQuery.minimumShouldMatch()); - - IOUtils.close(w, reader, dir); - } - public void testMultipleComplementAwareOnSameFieldNotRewritten() throws Exception { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new StandardAnalyzer())); @@ -641,100 +583,6 @@ public void testMustNotRewriteDisabledWithoutExactlyOneValuePerDoc() throws Exce IOUtils.close(w, reader, dir); } - public void testOneMustNotNumericMatchQueryRewritten() throws Exception { - Directory dir = newDirectory(); - IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new StandardAnalyzer())); - addDocument(w, INT_FIELD_NAME, 1); - DirectoryReader reader = DirectoryReader.open(w); - IndexSearcher searcher = getIndexSearcher(reader); - - BoolQueryBuilder qb = new BoolQueryBuilder(); - int excludedValue = 200; - QueryBuilder matchQuery = new MatchQueryBuilder(INT_FIELD_NAME, excludedValue); - qb.mustNot(matchQuery); - - BoolQueryBuilder rewritten = (BoolQueryBuilder) Rewriteable.rewrite(qb, createShardContext(searcher)); - assertFalse(rewritten.mustNot().contains(matchQuery)); - - QueryBuilder expectedLowerQuery = getRangeQueryBuilder(INT_FIELD_NAME, null, excludedValue, true, false); - QueryBuilder expectedUpperQuery = getRangeQueryBuilder(INT_FIELD_NAME, excludedValue, null, false, true); - assertEquals(1, rewritten.must().size()); - - BoolQueryBuilder nestedBoolQuery = (BoolQueryBuilder) rewritten.must().get(0); - assertEquals(2, nestedBoolQuery.should().size()); - assertEquals("1", nestedBoolQuery.minimumShouldMatch()); - assertTrue(nestedBoolQuery.should().contains(expectedLowerQuery)); - assertTrue(nestedBoolQuery.should().contains(expectedUpperQuery)); - - // When the QueryShardContext is null, we should not rewrite any match queries as we can't confirm if they're on numeric fields. - QueryRewriteContext nullContext = mock(QueryRewriteContext.class); - when(nullContext.convertToShardContext()).thenReturn(null); - BoolQueryBuilder rewrittenNoContext = (BoolQueryBuilder) Rewriteable.rewrite(qb, nullContext); - assertTrue(rewrittenNoContext.mustNot().contains(matchQuery)); - assertTrue(rewrittenNoContext.should().isEmpty()); - - IOUtils.close(w, reader, dir); - } - - public void testMustClausesRewritten() throws Exception { - BoolQueryBuilder qb = new BoolQueryBuilder(); - - // Should be moved - QueryBuilder intTermQuery = new TermQueryBuilder(INT_FIELD_NAME, 200); - QueryBuilder rangeQuery = new RangeQueryBuilder(INT_FIELD_NAME).gt(10).lt(20); - // Should be moved to filter clause, the boost applies equally to all matched docs - QueryBuilder rangeQueryWithBoost = new RangeQueryBuilder(DATE_FIELD_NAME).gt(10).lt(20).boost(2); - QueryBuilder intTermsQuery = new TermsQueryBuilder(INT_FIELD_NAME, new int[] { 1, 4, 100 }); - QueryBuilder boundingBoxQuery = new GeoBoundingBoxQueryBuilder(GEO_POINT_FIELD_NAME); - QueryBuilder doubleMatchQuery = new MatchQueryBuilder(DOUBLE_FIELD_NAME, 5.5); - - // Should not be moved - QueryBuilder textTermQuery = new TermQueryBuilder(TEXT_FIELD_NAME, "bar"); - QueryBuilder textTermsQuery = new TermsQueryBuilder(TEXT_FIELD_NAME, "foo", "bar"); - QueryBuilder textMatchQuery = new MatchQueryBuilder(TEXT_FIELD_NAME, "baz"); - - qb.must(intTermQuery); - qb.must(rangeQuery); - qb.must(rangeQueryWithBoost); - qb.must(intTermsQuery); - qb.must(boundingBoxQuery); - qb.must(doubleMatchQuery); - - qb.must(textTermQuery); - qb.must(textTermsQuery); - qb.must(textMatchQuery); - - BoolQueryBuilder rewritten = (BoolQueryBuilder) Rewriteable.rewrite(qb, createShardContext()); - for (QueryBuilder clause : List.of( - intTermQuery, - rangeQuery, - rangeQueryWithBoost, - intTermsQuery, - boundingBoxQuery, - doubleMatchQuery - )) { - assertFalse(rewritten.must().contains(clause)); - assertTrue(rewritten.filter().contains(clause)); - } - for (QueryBuilder clause : List.of(textTermQuery, textTermsQuery, textMatchQuery)) { - assertTrue(rewritten.must().contains(clause)); - assertFalse(rewritten.filter().contains(clause)); - } - - // If we have null QueryShardContext, match/term/terms queries should not be moved as we can't determine if they're numeric. - QueryRewriteContext nullContext = mock(QueryRewriteContext.class); - when(nullContext.convertToShardContext()).thenReturn(null); - rewritten = (BoolQueryBuilder) Rewriteable.rewrite(qb, nullContext); - for (QueryBuilder clause : List.of(rangeQuery, rangeQueryWithBoost, boundingBoxQuery)) { - assertFalse(rewritten.must().contains(clause)); - assertTrue(rewritten.filter().contains(clause)); - } - for (QueryBuilder clause : List.of(textTermQuery, textTermsQuery, textMatchQuery, intTermQuery, intTermsQuery, doubleMatchQuery)) { - assertTrue(rewritten.must().contains(clause)); - assertFalse(rewritten.filter().contains(clause)); - } - } - private QueryBuilder getRangeQueryBuilder(String fieldName, Integer lower, Integer upper, boolean includeLower, boolean includeUpper) { RangeQueryBuilder rq = new RangeQueryBuilder(fieldName); if (lower != null) { diff --git a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java index cc77a19755f5d..5d8469241831b 100644 --- a/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/opensearch/index/search/NestedHelperTests.java @@ -117,7 +117,7 @@ public void setUp() throws Exception { .endObject() .endObject() .endObject(); - indexService = createIndex("index", Settings.EMPTY, "type", mapping); + indexService = createIndex("index", Settings.EMPTY, mapping); mapperService = indexService.mapperService(); } diff --git a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java index f50dcfde112f2..b66fcf55b5e4d 100644 --- a/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/opensearch/index/search/nested/NestedSortingTests.java @@ -466,7 +466,7 @@ public void testMultiLevelNestedSorting() throws IOException { mapping.endObject(); } mapping.endObject(); - IndexService indexService = createIndex("nested_sorting", Settings.EMPTY, "_doc", mapping); + IndexService indexService = createIndex("nested_sorting", Settings.EMPTY, mapping); List> books = new ArrayList<>(); { diff --git a/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java index 1fafa4739b8b4..ccaeeef190684 100644 --- a/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/opensearch/index/similarity/SimilarityTests.java @@ -102,7 +102,7 @@ public void testResolveLegacySimilarity() throws IOException { .endObject() .endObject(); - MapperService mapperService = createIndex("foo", settings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", settings, mapping).mapperService(); assertThat(mapperService.fieldType("dummy").getTextSearchInfo().getSimilarity().get(), instanceOf(LegacyBM25Similarity.class)); } @@ -136,7 +136,7 @@ public void testResolveSimilaritiesFromMapping_bm25() throws IOException { .put("index.similarity.my_similarity.b", 0.5f) .put("index.similarity.my_similarity.discount_overlaps", false) .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(BM25Similarity.class)); BM25Similarity similarity = (BM25Similarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); @@ -156,7 +156,7 @@ public void testResolveSimilaritiesFromMapping_boolean() throws IOException { .endObject() .endObject(); - MapperService mapperService = createIndex("foo", Settings.EMPTY, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", Settings.EMPTY, mapping).mapperService(); assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(BooleanSimilarity.class)); } @@ -178,7 +178,7 @@ public void testResolveSimilaritiesFromMapping_DFR() throws IOException { .put("index.similarity.my_similarity.normalization", "h2") .put("index.similarity.my_similarity.normalization.h2.c", 3f) .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(DFRSimilarity.class)); DFRSimilarity similarity = (DFRSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); @@ -206,7 +206,7 @@ public void testResolveSimilaritiesFromMapping_IB() throws IOException { .put("index.similarity.my_similarity.normalization", "h2") .put("index.similarity.my_similarity.normalization.h2.c", 3f) .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(IBSimilarity.class)); IBSimilarity similarity = (IBSimilarity) mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(); @@ -231,7 +231,7 @@ public void testResolveSimilaritiesFromMapping_DFI() throws IOException { .put("index.similarity.my_similarity.type", "DFI") .put("index.similarity.my_similarity.independence_measure", "chisquared") .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); MappedFieldType fieldType = mapperService.fieldType("field1"); assertThat(fieldType.getTextSearchInfo().getSimilarity().get(), instanceOf(DFISimilarity.class)); @@ -255,7 +255,7 @@ public void testResolveSimilaritiesFromMapping_LMDirichlet() throws IOException .put("index.similarity.my_similarity.mu", 3000f) .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); assertThat(mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LMDirichletSimilarity.class)); LMDirichletSimilarity similarity = (LMDirichletSimilarity) mapperService.fieldType("field1") @@ -280,7 +280,7 @@ public void testResolveSimilaritiesFromMapping_LMJelinekMercer() throws IOExcept .put("index.similarity.my_similarity.type", "LMJelinekMercer") .put("index.similarity.my_similarity.lambda", 0.7f) .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + MapperService mapperService = createIndex("foo", indexSettings, mapping).mapperService(); assertThat( mapperService.fieldType("field1").getTextSearchInfo().getSimilarity().get(), instanceOf(LMJelinekMercerSimilarity.class) diff --git a/server/src/test/java/org/opensearch/index/termvectors/TermVectorsServiceTests.java b/server/src/test/java/org/opensearch/index/termvectors/TermVectorsServiceTests.java index 37b672dc064c4..f6f50d6cdf2a2 100644 --- a/server/src/test/java/org/opensearch/index/termvectors/TermVectorsServiceTests.java +++ b/server/src/test/java/org/opensearch/index/termvectors/TermVectorsServiceTests.java @@ -67,7 +67,7 @@ public void testTook() throws Exception { .endObject() .endObject() .endObject(); - createIndex("test", Settings.EMPTY, "type1", mapping); + createIndex("test", Settings.EMPTY, mapping); ensureGreen(); client().prepareIndex("test").setId("0").setSource("field", "foo bar").setRefreshPolicy(IMMEDIATE).get(); @@ -96,7 +96,7 @@ public void testDocFreqs() throws IOException { .endObject() .endObject(); Settings settings = Settings.builder().put("number_of_shards", 1).build(); - createIndex("test", settings, "_doc", mapping); + createIndex("test", settings, mapping); ensureGreen(); int max = between(3, 10); @@ -135,7 +135,7 @@ public void testWithIndexedPhrases() throws IOException { .endObject() .endObject(); Settings settings = Settings.builder().put("number_of_shards", 1).build(); - createIndex("test", settings, "_doc", mapping); + createIndex("test", settings, mapping); ensureGreen(); int max = between(3, 10); diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java index 4c9bc2ae622ba..ec015a3049d38 100644 --- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java @@ -223,7 +223,8 @@ protected Translog createTranslog(TranslogConfig config) throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, getPersistedSeqNoConsumer(), - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -235,7 +236,8 @@ protected Translog openTranslog(TranslogConfig config, String translogUUID) thro () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, getPersistedSeqNoConsumer(), - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -272,7 +274,8 @@ private Translog create(Path path) throws IOException { () -> globalCheckpoint.get(), primaryTerm::get, getPersistedSeqNoConsumer(), - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -1508,13 +1511,9 @@ public int write(ByteBuffer src) throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, persistedSeqNos::add, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { TranslogWriter writer = translog.getCurrent(); int initialWriteCalls = writeCalls.get(); @@ -1614,13 +1613,9 @@ public void force(boolean metaData) throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, persistedSeqNos::add, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { TranslogWriter writer = translog.getCurrent(); byte[] bytes = new byte[256]; @@ -1712,13 +1707,9 @@ public void force(boolean metaData) throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, persistedSeqNos::add, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { TranslogWriter writer = translog.getCurrent(); @@ -1819,7 +1810,8 @@ public void testBasicRecovery() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); assertEquals( "lastCommitted must be 1 less than current", @@ -1879,7 +1871,8 @@ public void testRecoveryUncommitted() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertNotNull(translogGeneration); @@ -1907,7 +1900,8 @@ public void testRecoveryUncommitted() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertNotNull(translogGeneration); @@ -1970,7 +1964,8 @@ public void testRecoveryUncommittedFileExists() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertNotNull(translogGeneration); @@ -1999,7 +1994,8 @@ public void testRecoveryUncommittedFileExists() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertNotNull(translogGeneration); @@ -2064,7 +2060,8 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ); assertThat( @@ -2091,7 +2088,8 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertNotNull(translogGeneration); @@ -2390,7 +2388,8 @@ public void testOpenForeignTranslog() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); fail("translog doesn't belong to this UUID"); } catch (TranslogCorruptedException ex) { @@ -2403,7 +2402,8 @@ public void testOpenForeignTranslog() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); try (Translog.Snapshot snapshot = this.translog.newSnapshot(randomLongBetween(0, firstUncommitted), Long.MAX_VALUE)) { for (int i = firstUncommitted; i < translogOperations; i++) { @@ -2634,7 +2634,8 @@ public void testFailFlush() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertEquals( @@ -2792,7 +2793,8 @@ protected void afterAdd() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); Translog.Snapshot snapshot = tlog.newSnapshot() ) { @@ -2856,7 +2858,8 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); assertThat(translog.getMinFileGeneration(), equalTo(1L)); // no trimming done yet, just recovered @@ -2926,7 +2929,8 @@ public void testRecoveryFromFailureOnTrimming() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { // we don't know when things broke exactly @@ -3003,13 +3007,9 @@ private Translog getFailableTranslog( () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + channelFactory ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - @Override void deleteReaderFiles(TranslogReader reader) { if (fail.fail()) { @@ -3151,7 +3151,8 @@ public void testFailWhileCreateWriteWithRecoveredTLogs() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) { @Override protected TranslogWriter createWriter( @@ -3220,7 +3221,8 @@ public void testRecoverWithUnbackedNextGenInIllegalState() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ); assertEquals(ex.getMessage(), "failed to create new translog file"); @@ -3248,7 +3250,8 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { assertFalse(tlog.syncNeeded()); @@ -3271,7 +3274,8 @@ public void testRecoverWithUnbackedNextGenAndFutureFile() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ); assertEquals(ex.getMessage(), "failed to create new translog file"); @@ -3364,8 +3368,8 @@ public void testWithRandomException() throws IOException { localCheckpointOfSafeCommit = failableTLog.getDeletionPolicy().getLocalCheckpointOfSafeCommit(); IOUtils.closeWhileHandlingException(failableTLog); } - } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) { - // failed - that's ok, we didn't even create it + } catch (TranslogException | MockDirectoryWrapper.FakeIOException | TranslogCorruptedException ex) { + // failed - that's ok, we didn't even create it or it was corrupted from partial writes } catch (IOException ex) { assertEquals(ex.getMessage(), "__FAKE__ no space left on device"); } @@ -3375,8 +3379,8 @@ public void testWithRandomException() throws IOException { TranslogDeletionPolicy deletionPolicy = createTranslogDeletionPolicy(); deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit); IOUtils.close(getFailableTranslog(fail, config, randomBoolean(), false, generationUUID, deletionPolicy)); - } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) { - // failed - that's ok, we didn't even create it + } catch (TranslogException | MockDirectoryWrapper.FakeIOException | TranslogCorruptedException ex) { + // failed - that's ok, we didn't even create it or it was corrupted from partial writes } catch (IOException ex) { assertEquals(ex.getMessage(), "__FAKE__ no space left on device"); } @@ -3402,7 +3406,8 @@ public void testWithRandomException() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); Translog.Snapshot snapshot = translog.newSnapshot(localCheckpointOfSafeCommit + 1, Long.MAX_VALUE) ) { @@ -3498,7 +3503,8 @@ public void testPendingDelete() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); translog.add(new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 2 })); translog.rollGeneration(); @@ -3513,7 +3519,8 @@ public void testPendingDelete() throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -3881,7 +3888,8 @@ class MisbehavingTranslog extends LocalTranslog { globalCheckpointSupplier, primaryTermSupplier, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -3989,7 +3997,8 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { recoveredTranslog.rollGeneration(); @@ -4024,7 +4033,8 @@ public void testSyncConcurrently() throws Exception { globalCheckpointSupplier, primaryTerm::get, persistedSeqNos::add, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ) ) { Thread[] threads = new Thread[between(2, 8)]; @@ -4106,13 +4116,9 @@ public void force(boolean metaData) throws IOException { () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + channelFactory ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - @Override void syncBeforeRollGeneration() { // make it a noop like the old versions diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java index 9682c0ba45a06..6c89cf2adf988 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTimestampAwareTranslogTests.java @@ -621,13 +621,9 @@ public void testExtraGenToKeep() throws Exception { () -> Boolean.TRUE, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); addToTranslogAndListAndUpload(translog, ops, new Translog.Index("2", 1, primaryTerm.get(), new byte[] { 1 })); diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java index 7da3bba9448a4..edcdca3f7b3de 100644 --- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java @@ -201,7 +201,8 @@ protected RemoteFsTranslog createTranslogInstance( primaryMode::get, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } @@ -475,13 +476,9 @@ public void testExtraGenToKeep() throws Exception { () -> Boolean.TRUE, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + null + ) ) { addToTranslogAndListAndUpload(translog, ops, new Translog.Index("1", 0, primaryTerm.get(), new byte[] { 1 })); @@ -1525,13 +1522,9 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException { () -> Boolean.TRUE, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { TranslogWriter writer = translog.getCurrent(); int initialWriteCalls = writeCalls.get(); @@ -1636,13 +1629,9 @@ public void force(boolean metaData) throws IOException { () -> Boolean.TRUE, new RemoteTranslogTransferTracker(shardId, 10), DefaultRemoteStoreSettings.INSTANCE, - TranslogOperationHelper.DEFAULT - ) { - @Override - ChannelFactory getChannelFactory() { - return channelFactory; - } - } + TranslogOperationHelper.DEFAULT, + channelFactory + ) ) { TranslogWriter writer = translog.getCurrent(); byte[] bytes = new byte[256]; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java index 22aa2e88e665f..10529bb155845 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java @@ -95,7 +95,8 @@ protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSup () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier, seqNo -> {}, - TranslogOperationHelper.DEFAULT + TranslogOperationHelper.DEFAULT, + null ); } diff --git a/server/src/test/java/org/opensearch/lucene/grouping/CollapsingTopDocsCollectorTests.java b/server/src/test/java/org/opensearch/lucene/grouping/CollapsingTopDocsCollectorTests.java index de1ff6afa903c..38bc575037924 100644 --- a/server/src/test/java/org/opensearch/lucene/grouping/CollapsingTopDocsCollectorTests.java +++ b/server/src/test/java/org/opensearch/lucene/grouping/CollapsingTopDocsCollectorTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -224,7 +225,7 @@ private > void assertSearchCollapse( subSearcher.search(weight, c); shardHits[shardIDX] = c.getTopDocs(); } - CollapseTopFieldDocs mergedFieldDocs = CollapseTopFieldDocs.merge(sort, 0, expectedNumGroups, shardHits, true); + CollapseTopFieldDocs mergedFieldDocs = CollapseTopFieldDocs.merge(sort, 0, expectedNumGroups, shardHits); assertTopDocsEquals(query, mergedFieldDocs, collapseTopFieldDocs); w.close(); reader.close(); @@ -455,4 +456,42 @@ public void testEmptySortedSegment() throws Exception { reader.close(); dir.close(); } + + public void testInconsistentShardIndicesException() { + Sort sort = Sort.RELEVANCE; + + // Create TopDocs with mixed shardIndex values - some set, some -1 + ScoreDoc[] shard1Docs = { + new FieldDoc(1, 9.0f, new Object[] { 9.0f }, 0), // shardIndex = 0 + new FieldDoc(2, 8.0f, new Object[] { 8.0f }, 0) // shardIndex = 0 + }; + + ScoreDoc[] shard2Docs = { + new FieldDoc(3, 7.0f, new Object[] { 7.0f }, -1), // shardIndex = -1 (inconsistent!) + new FieldDoc(4, 6.0f, new Object[] { 6.0f }, -1) // shardIndex = -1 + }; + + CollapseTopFieldDocs[] shardHits = { + new CollapseTopFieldDocs( + "field", + new TotalHits(2, TotalHits.Relation.EQUAL_TO), + shard1Docs, + sort.getSort(), + new Object[] { "val1", "val2" } + ), + new CollapseTopFieldDocs( + "field", + new TotalHits(2, TotalHits.Relation.EQUAL_TO), + shard2Docs, + sort.getSort(), + new Object[] { "val3", "val4" } + ) }; + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + CollapseTopFieldDocs.merge(sort, 0, 10, shardHits); + }); + + assertEquals("Inconsistent order of shard indices", exception.getMessage()); + } + } diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java index ce96623ea06df..34c1f5ab7f218 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorBaseTests.java @@ -144,7 +144,7 @@ private ValuesSourceConfig getVSConfig( } public void testShortcutIsApplicable() throws IOException { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bytes", "type=keyword"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "bytes", "type=keyword"); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); diff --git a/server/src/test/java/org/opensearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/opensearch/search/aggregations/support/ValuesSourceConfigTests.java index 568c3c950f588..bcd35d6418652 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -42,7 +42,6 @@ import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; import org.opensearch.index.fielddata.SortedBinaryDocValues; -import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryShardContext; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -50,7 +49,7 @@ public class ValuesSourceConfigTests extends OpenSearchSingleNodeTestCase { public void testKeyword() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bytes", "type=keyword"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "bytes", "type=keyword"); client().prepareIndex("index").setId("1").setSource("bytes", "abc").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -76,7 +75,7 @@ public void testKeyword() throws Exception { } public void testEmptyKeyword() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bytes", "type=keyword"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "bytes", "type=keyword"); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -107,7 +106,7 @@ public void testEmptyKeyword() throws Exception { } public void testUnmappedKeyword() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + IndexService indexService = createIndex("index", Settings.EMPTY); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -137,7 +136,7 @@ public void testUnmappedKeyword() throws Exception { } public void testLong() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "long", "type=long"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "long", "type=long"); client().prepareIndex("index").setId("1").setSource("long", 42).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -163,7 +162,7 @@ public void testLong() throws Exception { } public void testEmptyLong() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "long", "type=long"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "long", "type=long"); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -194,7 +193,7 @@ public void testEmptyLong() throws Exception { } public void testUnmappedLong() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + IndexService indexService = createIndex("index", Settings.EMPTY); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -225,7 +224,7 @@ public void testUnmappedLong() throws Exception { } public void testBoolean() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bool", "type=boolean"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "bool", "type=boolean"); client().prepareIndex("index").setId("1").setSource("bool", true).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -251,7 +250,7 @@ public void testBoolean() throws Exception { } public void testEmptyBoolean() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bool", "type=boolean"); + IndexService indexService = createIndexWithSimpleMappings("index", Settings.EMPTY, "bool", "type=boolean"); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -282,7 +281,7 @@ public void testEmptyBoolean() throws Exception { } public void testUnmappedBoolean() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + IndexService indexService = createIndex("index", Settings.EMPTY); client().prepareIndex("index").setId("1").setSource().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -313,7 +312,14 @@ public void testUnmappedBoolean() throws Exception { } public void testFieldAlias() throws Exception { - IndexService indexService = createIndex("index", Settings.EMPTY, "type", "field", "type=keyword", "alias", "type=alias,path=field"); + IndexService indexService = createIndexWithSimpleMappings( + "index", + Settings.EMPTY, + "field", + "type=keyword", + "alias", + "type=alias,path=field" + ); client().prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { @@ -354,7 +360,7 @@ public void testDerivedField() throws Exception { .endObject() .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); client().prepareIndex("index").setId("1").setSource("field", "value").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java index 1c8a93f6483ae..a738b386f277e 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java @@ -277,7 +277,7 @@ public void testIgnoreAbove() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); XContentBuilder source = XContentFactory.jsonBuilder() @@ -307,7 +307,7 @@ public void testFieldAliases() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject(); @@ -341,7 +341,7 @@ public void testMultiFields() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject().field("field", 42).endObject(); @@ -374,7 +374,7 @@ public void testCopyTo() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); XContentBuilder source = XContentFactory.jsonBuilder() @@ -420,7 +420,7 @@ public void testTextSubFields() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject().array("field", "some text").endObject(); @@ -484,7 +484,7 @@ public MapperService createMapperService() throws IOException { .endObject() .endObject(); - IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("index", Settings.EMPTY, mapping); return indexService.mapperService(); } diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java index f106aaa13dc48..409f123d6e108 100644 --- a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java @@ -124,7 +124,7 @@ public void testDerivedFieldFromIndexMapping() throws IOException { .endObject(); int docId = 0; - IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("test_index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); try ( @@ -260,7 +260,7 @@ public void testDerivedFieldFromSearchMapping() throws IOException { // Create index and mapper service // We are not defining derived fields in index mapping here XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().endObject(); - IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping); + IndexService indexService = createIndex("test_index", Settings.EMPTY, mapping); MapperService mapperService = indexService.mapperService(); try ( diff --git a/server/src/test/java/org/opensearch/search/geo/GeoShapeQueryTests.java b/server/src/test/java/org/opensearch/search/geo/GeoShapeQueryTests.java index 4f78d9166b414..3fb6fef08452f 100644 --- a/server/src/test/java/org/opensearch/search/geo/GeoShapeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/geo/GeoShapeQueryTests.java @@ -365,7 +365,7 @@ public void testEnvelopeSpanningDateline() throws Exception { public void testGeometryCollectionRelations() throws Exception { XContentBuilder mapping = createDefaultMapping(); - createIndex("test", Settings.builder().put("index.number_of_shards", 1).build(), "doc", mapping); + createIndex("test", Settings.builder().put("index.number_of_shards", 1).build(), mapping); EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(-10, 10), new Coordinate(10, -10)); @@ -490,7 +490,7 @@ public void testEdgeCases() throws Exception { public void testIndexedShapeReferenceSourceDisabled() throws Exception { XContentBuilder mapping = createDefaultMapping(); client().admin().indices().prepareCreate("test").setMapping(mapping).get(); - createIndex("shapes", Settings.EMPTY, "shape_type", "_source", "enabled=false"); + createIndexWithSimpleMappings("shapes", Settings.EMPTY, "_source", "enabled=false"); ensureGreen(); EnvelopeBuilder shape = new EnvelopeBuilder(new Coordinate(-45, 45), new Coordinate(45, -45)); @@ -749,7 +749,7 @@ public void testFieldAlias() throws IOException { .endObject() .endObject(); - createIndex("test", Settings.EMPTY, "type", mapping); + createIndex("test", Settings.EMPTY, mapping); ShapeBuilder shape = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.MULTIPOINT); client().prepareIndex("test") diff --git a/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java b/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java index 03fedc3534e82..e6bf421c5b1c9 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryCollectorContextSpecRegistryTests.java @@ -49,11 +49,14 @@ public void testGetQueryCollectorContextSpec_WithValidSpec() throws IOException QueryCollectorArguments mockArguments = new QueryCollectorArguments.Builder().build(); // Given QueryCollectorContextSpecRegistry.registerFactory(mockFactory1); - when(mockFactory1.createQueryCollectorContextSpec(mockSearchContext, mockArguments)).thenReturn(Optional.of(mockSpec)); + when(mockFactory1.createQueryCollectorContextSpec(mockSearchContext, mockSearchContext.query(), mockArguments)).thenReturn( + Optional.of(mockSpec) + ); // When Optional result = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( mockSearchContext, + mockSearchContext.query(), mockArguments ); @@ -69,6 +72,7 @@ public void testGetQueryCollectorContextSpec_NoFactories() throws IOException { // When Optional result = QueryCollectorContextSpecRegistry.getQueryCollectorContextSpec( mockSearchContext, + mockSearchContext.query(), mockArguments ); diff --git a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java index ae32ebd0a6f7a..f00111b6160a2 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryPhaseTests.java @@ -1123,6 +1123,9 @@ public void testCollapseQuerySearchResults() throws Exception { assertEquals(2, context.queryResult().topDocs().topDocs.scoreDocs.length); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs, instanceOf(CollapseTopFieldDocs.class)); + for (ScoreDoc scoreDoc : context.queryResult().topDocs().topDocs.scoreDocs) { + assertEquals(-1, scoreDoc.shardIndex); + } CollapseTopFieldDocs topDocs = (CollapseTopFieldDocs) context.queryResult().topDocs().topDocs; assertThat(topDocs.collapseValues.length, equalTo(2)); @@ -1135,6 +1138,9 @@ public void testCollapseQuerySearchResults() throws Exception { assertEquals(2, context.queryResult().topDocs().topDocs.scoreDocs.length); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs, instanceOf(CollapseTopFieldDocs.class)); + for (ScoreDoc scoreDoc : context.queryResult().topDocs().topDocs.scoreDocs) { + assertEquals(-1, scoreDoc.shardIndex); + } topDocs = (CollapseTopFieldDocs) context.queryResult().topDocs().topDocs; assertThat(topDocs.collapseValues.length, equalTo(2)); @@ -1147,6 +1153,9 @@ public void testCollapseQuerySearchResults() throws Exception { assertEquals(2, context.queryResult().topDocs().topDocs.scoreDocs.length); assertThat(context.queryResult().topDocs().topDocs.totalHits.value(), equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs, instanceOf(CollapseTopFieldDocs.class)); + for (ScoreDoc scoreDoc : context.queryResult().topDocs().topDocs.scoreDocs) { + assertEquals(-1, scoreDoc.shardIndex); + } topDocs = (CollapseTopFieldDocs) context.queryResult().topDocs().topDocs; assertThat(topDocs.collapseValues.length, equalTo(2)); diff --git a/server/src/test/java/org/opensearch/search/query/QueryRewriterRegistryTests.java b/server/src/test/java/org/opensearch/search/query/QueryRewriterRegistryTests.java new file mode 100644 index 0000000000000..64cb2ff5efb27 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/QueryRewriterRegistryTests.java @@ -0,0 +1,328 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.search.SearchService; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; + +public class QueryRewriterRegistryTests extends OpenSearchTestCase { + + private final QueryShardContext context = mock(QueryShardContext.class); + + @Override + public void setUp() throws Exception { + super.setUp(); + // Initialize registry with default settings + Settings settings = Settings.builder() + .put(SearchService.QUERY_REWRITING_ENABLED_SETTING.getKey(), true) + .put(SearchService.QUERY_REWRITING_TERMS_THRESHOLD_SETTING.getKey(), 16) + .build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + QueryRewriterRegistry.INSTANCE.initialize(settings, clusterSettings); + } + + public void testCompleteRewritingPipeline() { + // Test that all rewriters work together correctly + QueryBuilder nestedBool = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.termQuery("status", "active")) + .must(QueryBuilders.termQuery("status", "pending")); + + QueryBuilder query = QueryBuilders.boolQuery() + .must(nestedBool) + .filter(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.termQuery("type", "product")) + .filter(QueryBuilders.termQuery("type", "service")); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have: + // - Flattened nested boolean + // - Terms in must clauses are NOT merged (semantically different) + // - Removed match_all queries + assertThat(rewrittenBool.must().size(), equalTo(2)); // two term queries for status + assertThat(rewrittenBool.must().get(0), instanceOf(TermQueryBuilder.class)); + assertThat(rewrittenBool.must().get(1), instanceOf(TermQueryBuilder.class)); + + assertThat(rewrittenBool.filter().size(), equalTo(2)); // two term queries for type (below threshold) + assertThat(rewrittenBool.filter().get(0), instanceOf(TermQueryBuilder.class)); + assertThat(rewrittenBool.filter().get(1), instanceOf(TermQueryBuilder.class)); + } + + public void testDisabledRewriting() { + // Test disabled rewriting via settings + Settings settings = Settings.builder().put(SearchService.QUERY_REWRITING_ENABLED_SETTING.getKey(), false).build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + // Initialize with disabled setting + QueryRewriterRegistry.INSTANCE.initialize(settings, clusterSettings); + + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertSame(query, rewritten); + + // Enable via settings update + clusterSettings.applySettings(Settings.builder().put(SearchService.QUERY_REWRITING_ENABLED_SETTING.getKey(), true).build()); + + // Now it should rewrite + QueryBuilder rewritten2 = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertNotSame(query, rewritten2); + } + + public void testNullQuery() { + // Null query should return null + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(null, context); + assertNull(rewritten); + } + + public void testRewriterPriorityOrder() { + // Test that rewriters are applied in correct order + // Create a query that will be affected by multiple rewriters + QueryBuilder deeplyNested = QueryBuilders.boolQuery() + .must( + QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.termQuery("field", "value1")) + .must(QueryBuilders.termQuery("field", "value2")) + ); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(deeplyNested, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should be flattened first, match_all kept in must (scoring context), but terms NOT merged in must context + assertThat(rewrittenBool.must().size(), equalTo(3)); // match_all + 2 term queries + // Check that we have one match_all and two term queries (order may vary) + long matchAllCount = rewrittenBool.must().stream().filter(q -> q instanceof MatchAllQueryBuilder).count(); + long termCount = rewrittenBool.must().stream().filter(q -> q instanceof TermQueryBuilder).count(); + assertThat(matchAllCount, equalTo(1L)); + assertThat(termCount, equalTo(2L)); + } + + public void testComplexRealWorldQuery() { + // Test a complex real-world-like query + QueryBuilder query = QueryBuilders.boolQuery() + .must( + QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.termQuery("category", "electronics")) + .filter(QueryBuilders.termQuery("category", "computers")) + ) + .filter( + QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("brand", "apple")) + .should(QueryBuilders.termQuery("brand", "dell")) + .should(QueryBuilders.termQuery("brand", "hp")) + ) + .must(QueryBuilders.rangeQuery("price").gte(500).lte(2000)) + .mustNot(QueryBuilders.termQuery("status", "discontinued")); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // After rewriting: + // - The nested bool in must clause should be flattened + // - match_all should be removed + // - term queries should be merged into terms query + // - The filter bool with brand terms should be preserved + // - The range query should be moved from must to filter by MustToFilterRewriter + + // Check must clauses (should have terms query for category only - range moved to filter) + assertThat(rewrittenBool.must().size(), equalTo(1)); + + // Check filter clauses (should have the brand bool query AND the range query) + assertThat(rewrittenBool.filter().size(), equalTo(2)); + // One should be the brand bool query + boolean hasBoolFilter = false; + boolean hasRangeFilter = false; + for (QueryBuilder filter : rewrittenBool.filter()) { + if (filter instanceof BoolQueryBuilder) { + hasBoolFilter = true; + } else if (filter instanceof RangeQueryBuilder) { + hasRangeFilter = true; + } + } + assertTrue(hasBoolFilter); + assertTrue(hasRangeFilter); + + // Must not should be preserved + assertThat(rewrittenBool.mustNot().size(), equalTo(1)); + } + + public void testPerformanceMetrics() { + // Test that we log performance metrics in debug mode + // This is more of a sanity check that the timing code doesn't throw exceptions + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("field1", "value1")) + .must(QueryBuilders.termQuery("field1", "value2")) + .filter(QueryBuilders.matchAllQuery()); + + // Should not throw any exceptions + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertNotNull(rewritten); + } + + public void testRewriterErrorHandling() { + // Test that if a rewriter throws an exception, others still run + // This is handled internally by QueryRewriterRegistry + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("field", "value")) + .filter(QueryBuilders.matchAllQuery()); + + // Even if one rewriter fails, others should still be applied + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertNotNull(rewritten); + } + + public void testVeryComplexMixedQuery() { + // Test a very complex query with all optimizations applicable + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .must( + QueryBuilders.boolQuery() + .must( + QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("status", "active")) + .filter(QueryBuilders.termQuery("status", "pending")) + .filter(QueryBuilders.termQuery("status", "approved")) + ) + .must( + QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.termQuery("type", "A")) + .filter(QueryBuilders.termQuery("type", "B")) + ) + ) + .filter(QueryBuilders.matchAllQuery()) + .filter( + QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("priority", "high")) + .should(QueryBuilders.termQuery("priority", "medium")) + .should(QueryBuilders.termQuery("priority", "low")) + ) + .should(QueryBuilders.termQuery("category", "urgent")) + .should(QueryBuilders.termQuery("category", "important")) + .mustNot(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("archived", "true"))) + .minimumShouldMatch(1); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder result = (BoolQueryBuilder) rewritten; + + // Check that minimum should match is preserved + assertThat(result.minimumShouldMatch(), equalTo("1")); + + // Verify optimizations were applied + assertNotSame(query, rewritten); + + // Should have flattened structure and merged terms + assertTrue(result.must().size() >= 1); + assertTrue(result.filter().size() >= 1); + } + + public void testCustomRewriterRegistration() { + // Create a custom rewriter for testing + QueryRewriter customRewriter = new QueryRewriter() { + @Override + public QueryBuilder rewrite(QueryBuilder query, QueryShardContext context) { + if (query instanceof TermQueryBuilder) { + TermQueryBuilder termQuery = (TermQueryBuilder) query; + if ("test_field".equals(termQuery.fieldName()) && "test_value".equals(termQuery.value())) { + // Replace with a different query + return QueryBuilders.termQuery("custom_field", "custom_value"); + } + } else if (query instanceof BoolQueryBuilder) { + // Recursively apply to nested queries + BoolQueryBuilder boolQuery = (BoolQueryBuilder) query; + BoolQueryBuilder rewritten = new BoolQueryBuilder(); + + // Copy settings + rewritten.boost(boolQuery.boost()); + rewritten.queryName(boolQuery.queryName()); + rewritten.minimumShouldMatch(boolQuery.minimumShouldMatch()); + rewritten.adjustPureNegative(boolQuery.adjustPureNegative()); + + // Recursively rewrite clauses + boolean changed = false; + for (QueryBuilder must : boolQuery.must()) { + QueryBuilder rewrittenClause = rewrite(must, context); + rewritten.must(rewrittenClause); + if (rewrittenClause != must) changed = true; + } + for (QueryBuilder filter : boolQuery.filter()) { + QueryBuilder rewrittenClause = rewrite(filter, context); + rewritten.filter(rewrittenClause); + if (rewrittenClause != filter) changed = true; + } + for (QueryBuilder should : boolQuery.should()) { + QueryBuilder rewrittenClause = rewrite(should, context); + rewritten.should(rewrittenClause); + if (rewrittenClause != should) changed = true; + } + for (QueryBuilder mustNot : boolQuery.mustNot()) { + QueryBuilder rewrittenClause = rewrite(mustNot, context); + rewritten.mustNot(rewrittenClause); + if (rewrittenClause != mustNot) changed = true; + } + + return changed ? rewritten : query; + } + return query; + } + + @Override + public int priority() { + return 1000; // High priority to ensure it runs last + } + + @Override + public String name() { + return "test_custom_rewriter"; + } + }; + + // Register the custom rewriter + QueryRewriterRegistry.INSTANCE.registerRewriter(customRewriter); + + // Test that it's applied + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("test_field", "test_value")) + .filter(QueryBuilders.termQuery("other_field", "other_value")); + + QueryBuilder rewritten = QueryRewriterRegistry.INSTANCE.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // The custom rewriter should have replaced the term query + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(TermQueryBuilder.class)); + TermQueryBuilder mustTerm = (TermQueryBuilder) rewrittenBool.must().get(0); + assertThat(mustTerm.fieldName(), equalTo("custom_field")); + assertThat(mustTerm.value(), equalTo("custom_value")); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/rewriters/BooleanFlatteningRewriterTests.java b/server/src/test/java/org/opensearch/search/query/rewriters/BooleanFlatteningRewriterTests.java new file mode 100644 index 0000000000000..317d753493cf1 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/rewriters/BooleanFlatteningRewriterTests.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.apache.lucene.tests.util.LuceneTestCase.AwaitsFix; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; + +public class BooleanFlatteningRewriterTests extends OpenSearchTestCase { + + private final BooleanFlatteningRewriter rewriter = BooleanFlatteningRewriter.INSTANCE; + private final QueryShardContext context = mock(QueryShardContext.class); + + public void testSimpleBooleanQuery() { + // Simple boolean query should not be modified + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("field1", "value1")) + .filter(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNestedBooleanFlattening() { + // Nested boolean query with single must clause should be flattened + QueryBuilder nestedBool = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("field1", "value1")); + + QueryBuilder query = QueryBuilders.boolQuery().must(nestedBool).filter(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // The nested bool should be flattened + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(QueryBuilders.termQuery("field1", "value1").getClass())); + assertThat(rewrittenBool.filter().size(), equalTo(1)); + } + + public void testMultipleNestedBooleansFlattening() { + // Multiple nested boolean queries should all be flattened + QueryBuilder nested1 = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("field1", "value1")) + .must(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder nested2 = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("field3", "value3")); + + QueryBuilder query = QueryBuilders.boolQuery().must(nested1).filter(nested2); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // All nested clauses should be flattened + assertThat(rewrittenBool.must().size(), equalTo(2)); + assertThat(rewrittenBool.filter().size(), equalTo(1)); + } + + public void testShouldClauseFlattening() { + // Should clauses should also be flattened + QueryBuilder nestedShould = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("field1", "value1")) + .should(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder query = QueryBuilders.boolQuery().should(nestedShould).must(QueryBuilders.termQuery("field3", "value3")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should clauses should be flattened + assertThat(rewrittenBool.should().size(), equalTo(2)); + assertThat(rewrittenBool.must().size(), equalTo(1)); + } + + public void testMustNotClauseNoFlattening() { + // Must_not clauses should NOT be flattened to preserve semantics + QueryBuilder nestedMustNot = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("field1", "value1")); + + QueryBuilder query = QueryBuilders.boolQuery().mustNot(nestedMustNot).must(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Must_not should not be flattened + assertThat(rewrittenBool.mustNot().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().get(0), instanceOf(BoolQueryBuilder.class)); + } + + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/18906") + public void testDeepNesting() { + // TODO: This test expects complete flattening of deeply nested bool queries + // where intermediate bool wrappers are removed entirely. Our current implementation + // only flattens by merging same-type clauses but preserves the bool structure. + // This would require a different optimization strategy. + + // Deep nesting should be flattened at all levels + QueryBuilder deepNested = QueryBuilders.boolQuery() + .must(QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.termQuery("field1", "value1")))); + + QueryBuilder rewritten = rewriter.rewrite(deepNested, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should be flattened to single level bool with term query + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(TermQueryBuilder.class)); + + // Verify the term query details + TermQueryBuilder termQuery = (TermQueryBuilder) rewrittenBool.must().get(0); + assertThat(termQuery.fieldName(), equalTo("field1")); + assertThat(termQuery.value(), equalTo("value1")); + } + + public void testMixedClauseTypes() { + // Mixed clause types with different minimumShouldMatch settings + QueryBuilder nested = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("field1", "value1")) + .should(QueryBuilders.termQuery("field2", "value2")) + .minimumShouldMatch(1); + + QueryBuilder query = QueryBuilders.boolQuery().must(nested).minimumShouldMatch(2); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // Should not flatten due to different minimumShouldMatch + } + + public void testEmptyBooleanQuery() { + // Empty boolean query should not cause issues + QueryBuilder query = QueryBuilders.boolQuery(); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNonBooleanQuery() { + // Non-boolean queries should be returned as-is + QueryBuilder query = QueryBuilders.termQuery("field", "value"); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testVeryDeepNesting() { + // Test with 10 levels of nesting + QueryBuilder innermost = QueryBuilders.termQuery("field", "value"); + for (int i = 0; i < 10; i++) { + innermost = QueryBuilders.boolQuery().must(innermost); + } + + QueryBuilder rewritten = rewriter.rewrite(innermost, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + + // Should be flattened significantly + BoolQueryBuilder result = (BoolQueryBuilder) rewritten; + assertThat(result.must().size(), equalTo(1)); + } + + public void testQueryNamePreservation() { + // Ensure query names are preserved during flattening + QueryBuilder query = QueryBuilders.boolQuery() + .queryName("outer") + .must(QueryBuilders.boolQuery().queryName("inner").must(QueryBuilders.termQuery("field", "value"))); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + BoolQueryBuilder result = (BoolQueryBuilder) rewritten; + assertThat(result.queryName(), equalTo("outer")); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriterTests.java b/server/src/test/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriterTests.java new file mode 100644 index 0000000000000..6f2c6cf93133d --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/rewriters/MatchAllRemovalRewriterTests.java @@ -0,0 +1,167 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; + +public class MatchAllRemovalRewriterTests extends OpenSearchTestCase { + + private final MatchAllRemovalRewriter rewriter = MatchAllRemovalRewriter.INSTANCE; + private final QueryShardContext context = mock(QueryShardContext.class); + + public void testRemoveMatchAllFromMust() { + // match_all in must clause should NOT be removed in scoring context + QueryBuilder query = QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).must(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // match_all should be kept in scoring context + assertThat(rewrittenBool.must().size(), equalTo(2)); + } + + public void testRemoveMatchAllFromFilter() { + // match_all in filter clause should be removed + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.rangeQuery("price").gt(100)) + .must(QueryBuilders.termQuery("category", "electronics")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // match_all should be removed from filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.must().size(), equalTo(1)); + } + + public void testKeepMatchAllInShould() { + // match_all in should clause should be kept + QueryBuilder query = QueryBuilders.boolQuery() + .should(QueryBuilders.matchAllQuery()) + .should(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes for should clause + } + + public void testKeepMatchAllInMustNot() { + // match_all in must_not clause should be kept (it's meaningful) + QueryBuilder query = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes for must_not clause + } + + public void testOnlyMatchAllQuery() { + // Boolean query with only match_all should be simplified to match_all + QueryBuilder query = QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(QueryBuilders.matchAllQuery().getClass())); + } + + public void testMultipleMatchAllQueries() { + // Multiple match_all queries should all be removed + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.matchAllQuery()) + .filter(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // With filter clause present, this is not a pure scoring context + // Since there are non-match_all queries in must, match_all should be removed + assertThat(rewrittenBool.must().size(), equalTo(1)); // only term query remains + assertThat(rewrittenBool.filter().size(), equalTo(0)); + } + + public void testNestedBooleanWithMatchAll() { + // Nested boolean queries should also have match_all removed + QueryBuilder nested = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .must(QueryBuilders.termQuery("field1", "value1")); + + QueryBuilder query = QueryBuilders.boolQuery().must(nested).filter(QueryBuilders.termQuery("field2", "value2")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Nested bool keeps match_all in scoring context + BoolQueryBuilder nestedRewritten = (BoolQueryBuilder) rewrittenBool.must().get(0); + assertThat(nestedRewritten.must().size(), equalTo(2)); // match_all + term + } + + public void testEmptyBoolAfterRemoval() { + // Bool with only match_all in must/filter - keeps match_all in must in scoring context + QueryBuilder query = QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(QueryBuilders.matchAllQuery()); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // match_all in must is kept in scoring context, match_all in filter is removed + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.filter().size(), equalTo(0)); + } + + public void testBoolWithOnlyMustNotAfterRemoval() { + // Bool with only must_not after removal should not be converted to match_all + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.matchAllQuery()) + .mustNot(QueryBuilders.termQuery("status", "deleted")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // must clause keeps match_all in scoring context, must_not preserved + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(1)); + } + + public void testNonBooleanQuery() { + // Non-boolean queries should be returned as-is + QueryBuilder query = QueryBuilders.matchAllQuery(); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testEmptyBooleanQuery() { + // Empty boolean query should not be converted + QueryBuilder query = QueryBuilders.boolQuery(); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testBoostPreservation() { + // When converting bool with only match_all to match_all, preserve boost + QueryBuilder query = QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).boost(2.0f); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(QueryBuilders.matchAllQuery().getClass())); + assertThat(rewritten.boost(), equalTo(2.0f)); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriterTests.java b/server/src/test/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriterTests.java new file mode 100644 index 0000000000000..4fd21d9ad601e --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/rewriters/MustNotToShouldRewriterTests.java @@ -0,0 +1,284 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MustNotToShouldRewriterTests extends OpenSearchTestCase { + + private final MustNotToShouldRewriter rewriter = MustNotToShouldRewriter.INSTANCE; + private QueryShardContext context; + private Directory directory; + private IndexReader reader; + + @Override + public void setUp() throws Exception { + super.setUp(); + context = mock(QueryShardContext.class); + + // Create an index with single-valued numeric fields + directory = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig(Lucene.STANDARD_ANALYZER); + IndexWriter writer = new IndexWriter(directory, config); + + // Add some documents with single-valued numeric fields + for (int i = 0; i < 100; i++) { + Document doc = new Document(); + doc.add(new IntPoint("age", i)); + doc.add(new IntPoint("status", i % 10)); + writer.addDocument(doc); + } + + writer.close(); + reader = DirectoryReader.open(directory); + when(context.getIndexReader()).thenReturn(reader); + + // Setup numeric field types + NumberFieldMapper.NumberFieldType intFieldType = mock(NumberFieldMapper.NumberFieldType.class); + when(intFieldType.numberType()).thenReturn(NumberFieldMapper.NumberType.INTEGER); + // Make parse return the input value as a Number + when(intFieldType.parse(any())).thenAnswer(invocation -> { + Object arg = invocation.getArgument(0); + if (arg instanceof Number) { + return (Number) arg; + } + return Integer.parseInt(arg.toString()); + }); + when(context.fieldMapper("age")).thenReturn(intFieldType); + when(context.fieldMapper("status")).thenReturn(intFieldType); + when(context.fieldMapper("price")).thenReturn(intFieldType); + + // Setup non-numeric field types + MappedFieldType textFieldType = mock(MappedFieldType.class); + when(context.fieldMapper("name")).thenReturn(textFieldType); + when(context.fieldMapper("description")).thenReturn(textFieldType); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + reader.close(); + directory.close(); + } + + public void testRangeQueryRewritten() { + // Test that must_not range query is rewritten to should clauses + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("type", "product")) + .mustNot(QueryBuilders.rangeQuery("age").gte(18).lte(65)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have the original term query plus a new bool query from the rewrite + assertThat(rewrittenBool.must().size(), equalTo(2)); + + // The must_not clause should be removed + assertThat(rewrittenBool.mustNot().size(), equalTo(0)); + + // Find the nested bool query + BoolQueryBuilder nestedBool = null; + for (QueryBuilder must : rewrittenBool.must()) { + if (must instanceof BoolQueryBuilder) { + nestedBool = (BoolQueryBuilder) must; + break; + } + } + + assertNotNull(nestedBool); + assertThat(nestedBool.should().size(), equalTo(2)); // Two range queries for complement + assertThat(nestedBool.minimumShouldMatch(), equalTo("1")); + } + + public void testNumericTermQueryRewritten() { + // Test that must_not term query on numeric field is rewritten + QueryBuilder query = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("status", 5)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have a new bool query from the rewrite + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(0)); + + BoolQueryBuilder nestedBool = (BoolQueryBuilder) rewrittenBool.must().get(0); + assertThat(nestedBool.should().size(), equalTo(2)); // Two range queries for complement + assertThat(nestedBool.minimumShouldMatch(), equalTo("1")); + } + + public void testNumericTermsQueryRewritten() { + // Test that must_not terms query on numeric field is rewritten + QueryBuilder query = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status", new Object[] { 1, 2, 3 })); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have a new bool query from the rewrite + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(0)); + } + + public void testNumericMatchQueryRewritten() { + // Test that must_not match query on numeric field is rewritten + QueryBuilder query = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchQuery("age", 25)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have a new bool query from the rewrite + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(0)); + } + + public void testTextFieldNotRewritten() { + // Test that must_not queries on text fields are not rewritten + QueryBuilder query = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery("name", "test")) + .mustNot(QueryBuilders.matchQuery("description", "product")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes + } + + public void testMultipleQueriesOnSameFieldNotRewritten() { + // Test that multiple must_not queries on the same field are not rewritten + QueryBuilder query = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.rangeQuery("age").gte(18)) + .mustNot(QueryBuilders.rangeQuery("age").lte(65)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes + } + + public void testMinimumShouldMatchHandling() { + // Test that minimumShouldMatch is properly handled + QueryBuilder query = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("category", "A")) + .should(QueryBuilders.termQuery("category", "B")) + .mustNot(QueryBuilders.rangeQuery("age").gte(18)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Since we added a must clause, minimumShouldMatch should be set to 1 + assertThat(rewrittenBool.minimumShouldMatch(), equalTo("1")); + } + + public void testExistingMustClausesPreserved() { + // Test that existing must/filter/should clauses are preserved + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("type", "product")) + .filter(QueryBuilders.rangeQuery("price").gte(100)) + .should(QueryBuilders.termQuery("featured", true)) + .mustNot(QueryBuilders.rangeQuery("age").gte(18)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Original clauses should be preserved + assertThat(rewrittenBool.must().size(), equalTo(2)); // Original + rewritten + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.should().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(0)); + } + + public void testNestedBooleanQueriesRewritten() { + // Test that nested boolean queries are also rewritten + QueryBuilder nested = QueryBuilders.boolQuery().mustNot(QueryBuilders.rangeQuery("age").gte(18)); + + QueryBuilder query = QueryBuilders.boolQuery().must(nested); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // The nested bool should be rewritten + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(BoolQueryBuilder.class)); + + BoolQueryBuilder innerBool = (BoolQueryBuilder) rewrittenBool.must().get(0); + assertThat(innerBool.must().size(), equalTo(1)); // The rewritten clause + assertThat(innerBool.mustNot().size(), equalTo(0)); + } + + public void testNoMustNotClausesNoChanges() { + // Query without must_not clauses should not be changed + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("type", "product")) + .filter(QueryBuilders.rangeQuery("price").gte(100)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNonBoolQueryUnchanged() { + // Non-bool queries should not be changed + QueryBuilder query = QueryBuilders.termQuery("field", "value"); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNullContextNoRewrite() { + // With null context, no rewriting should happen + QueryBuilder query = QueryBuilders.boolQuery().mustNot(QueryBuilders.rangeQuery("age").gte(18)); + + QueryBuilder rewritten = rewriter.rewrite(query, null); + assertSame(query, rewritten); + } + + public void testRewriterPriority() { + // Verify rewriter has correct priority + assertThat(rewriter.priority(), equalTo(175)); + assertThat(rewriter.name(), equalTo("must_not_to_should")); + } + + public void testBoolQueryPropertiesPreserved() { + // All bool query properties should be preserved + QueryBuilder query = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.rangeQuery("age").gte(18)) + .boost(2.0f) + .queryName("my_query") + .adjustPureNegative(false); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Properties should be preserved + assertThat(rewrittenBool.boost(), equalTo(2.0f)); + assertThat(rewrittenBool.queryName(), equalTo("my_query")); + assertThat(rewrittenBool.adjustPureNegative(), equalTo(false)); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/rewriters/MustToFilterRewriterTests.java b/server/src/test/java/org/opensearch/search/query/rewriters/MustToFilterRewriterTests.java new file mode 100644 index 0000000000000..35e289813acff --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/rewriters/MustToFilterRewriterTests.java @@ -0,0 +1,309 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class MustToFilterRewriterTests extends OpenSearchTestCase { + + private final MustToFilterRewriter rewriter = MustToFilterRewriter.INSTANCE; + private QueryShardContext context; + + @Override + public void setUp() throws Exception { + super.setUp(); + context = mock(QueryShardContext.class); + + // Setup numeric field types + NumberFieldMapper.NumberFieldType intFieldType = mock(NumberFieldMapper.NumberFieldType.class); + when(context.fieldMapper("age")).thenReturn(intFieldType); + when(context.fieldMapper("price")).thenReturn(intFieldType); + when(context.fieldMapper("count")).thenReturn(intFieldType); + when(context.fieldMapper("user_id")).thenReturn(intFieldType); + + // Setup non-numeric field types + MappedFieldType textFieldType = mock(MappedFieldType.class); + when(context.fieldMapper("name")).thenReturn(textFieldType); + when(context.fieldMapper("description")).thenReturn(textFieldType); + when(context.fieldMapper("category")).thenReturn(textFieldType); + when(context.fieldMapper("status_code")).thenReturn(textFieldType); + when(context.fieldMapper("active")).thenReturn(textFieldType); + } + + public void testRangeQueryMovedToFilter() { + // Range queries should always be moved to filter + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100).lte(500)) + .must(QueryBuilders.termQuery("category", "electronics")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Range query should be moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.filter().get(0), instanceOf(RangeQueryBuilder.class)); + + // Term query on text field should remain in must + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(TermQueryBuilder.class)); + } + + public void testNumericTermQueryMovedToFilter() { + // Term queries on numeric fields should be moved to filter + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("age", 25)) + .must(QueryBuilders.termQuery("name", "John")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Numeric term query should be moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + TermQueryBuilder filterClause = (TermQueryBuilder) rewrittenBool.filter().get(0); + assertThat(filterClause.fieldName(), equalTo("age")); + + // Text term query should remain in must + assertThat(rewrittenBool.must().size(), equalTo(1)); + TermQueryBuilder mustClause = (TermQueryBuilder) rewrittenBool.must().get(0); + assertThat(mustClause.fieldName(), equalTo("name")); + } + + public void testNumericTermsQueryMovedToFilter() { + // Terms queries on numeric fields should be moved to filter + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termsQuery("count", new Object[] { 1, 2, 3 })) + .must(QueryBuilders.termsQuery("category", "A", "B", "C")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Numeric terms query should be moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + TermsQueryBuilder filterClause = (TermsQueryBuilder) rewrittenBool.filter().get(0); + assertThat(filterClause.fieldName(), equalTo("count")); + + // Text terms query should remain in must + assertThat(rewrittenBool.must().size(), equalTo(1)); + TermsQueryBuilder mustClause = (TermsQueryBuilder) rewrittenBool.must().get(0); + assertThat(mustClause.fieldName(), equalTo("category")); + } + + public void testNumericMatchQueryMovedToFilter() { + // Match queries on numeric fields should be moved to filter + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.matchQuery("price", 99.99)) + .must(QueryBuilders.matchQuery("description", "high quality")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Numeric match query should be moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + MatchQueryBuilder filterClause = (MatchQueryBuilder) rewrittenBool.filter().get(0); + assertThat(filterClause.fieldName(), equalTo("price")); + + // Text match query should remain in must + assertThat(rewrittenBool.must().size(), equalTo(1)); + MatchQueryBuilder mustClause = (MatchQueryBuilder) rewrittenBool.must().get(0); + assertThat(mustClause.fieldName(), equalTo("description")); + } + + public void testExistingFilterClausesPreserved() { + // Existing filter clauses should be preserved + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100)) + .filter(QueryBuilders.termQuery("status", "active")) + .filter(QueryBuilders.existsQuery("description")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have 3 filter clauses: moved range query + 2 existing + assertThat(rewrittenBool.filter().size(), equalTo(3)); + assertThat(rewrittenBool.must().size(), equalTo(0)); + } + + public void testShouldAndMustNotClausesUnchanged() { + // Should and must_not clauses should not be affected + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100)) + .should(QueryBuilders.termQuery("featured", true)) + .mustNot(QueryBuilders.termQuery("deleted", true)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Range query moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.must().size(), equalTo(0)); + + // Should and must_not unchanged + assertThat(rewrittenBool.should().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(1)); + } + + public void testNestedBooleanQueriesRewritten() { + // Nested boolean queries should also be rewritten + QueryBuilder nested = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("age").gte(18)) + .must(QueryBuilders.termQuery("active", true)); + + QueryBuilder query = QueryBuilders.boolQuery().must(nested).must(QueryBuilders.matchQuery("name", "test")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // The nested bool should be rewritten + assertThat(rewrittenBool.must().size(), equalTo(2)); + + // Find the nested bool query + BoolQueryBuilder nestedRewritten = null; + for (QueryBuilder clause : rewrittenBool.must()) { + if (clause instanceof BoolQueryBuilder) { + nestedRewritten = (BoolQueryBuilder) clause; + break; + } + } + + assertNotNull(nestedRewritten); + // The range query in the nested bool should be moved to filter + assertThat(nestedRewritten.filter().size(), equalTo(1)); + assertThat(nestedRewritten.filter().get(0), instanceOf(RangeQueryBuilder.class)); + // The term query should remain in must + assertThat(nestedRewritten.must().size(), equalTo(1)); + assertThat(nestedRewritten.must().get(0), instanceOf(TermQueryBuilder.class)); + } + + public void testBoolQueryPropertiesPreserved() { + // All bool query properties should be preserved + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100)) + .must(QueryBuilders.termQuery("category", "electronics")) + .boost(2.0f) + .queryName("my_query") + .minimumShouldMatch(2) + .adjustPureNegative(false); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Properties should be preserved + assertThat(rewrittenBool.boost(), equalTo(2.0f)); + assertThat(rewrittenBool.queryName(), equalTo("my_query")); + assertThat(rewrittenBool.minimumShouldMatch(), equalTo("2")); + assertThat(rewrittenBool.adjustPureNegative(), equalTo(false)); + } + + public void testNoMustClausesNoChanges() { + // Query without must clauses should not be changed + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.rangeQuery("price").gte(100)) + .should(QueryBuilders.termQuery("featured", true)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNonBoolQueryUnchanged() { + // Non-bool queries should not be changed + QueryBuilder query = QueryBuilders.termQuery("field", "value"); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNullContextStillMovesRangeQueries() { + // With null context, range queries should still be moved + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100)) + .must(QueryBuilders.termQuery("age", 25)); + + QueryBuilder rewritten = rewriter.rewrite(query, null); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Range query should be moved to filter even without context + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.filter().get(0), instanceOf(RangeQueryBuilder.class)); + + // Term query stays in must (can't determine if numeric without context) + assertThat(rewrittenBool.must().size(), equalTo(1)); + assertThat(rewrittenBool.must().get(0), instanceOf(TermQueryBuilder.class)); + } + + public void testAllMustClausesMovedToFilter() { + // If all must clauses can be moved, they should all go to filter + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("price").gte(100)) + .must(QueryBuilders.rangeQuery("age").gte(18)) + .must(QueryBuilders.termQuery("count", 5)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // All clauses should be in filter + assertThat(rewrittenBool.filter().size(), equalTo(3)); + assertThat(rewrittenBool.must().size(), equalTo(0)); + } + + public void testComplexMixedQuery() { + // Complex query with mix of movable and non-movable clauses + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery("created_date").gte("2024-01-01")) + .must(QueryBuilders.termQuery("user_id", 12345)) + .must(QueryBuilders.matchQuery("title", "opensearch")) + .must(QueryBuilders.termsQuery("status_code", new Object[] { 200, 201, 204 })) + .filter(QueryBuilders.existsQuery("description")) + .should(QueryBuilders.matchQuery("tags", "important")) + .mustNot(QueryBuilders.termQuery("deleted", true)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Range and numeric queries moved to filter + assertThat(rewrittenBool.filter().size(), equalTo(3)); // range + exists + user_id (numeric) + + // Text queries remain in must + assertThat(rewrittenBool.must().size(), equalTo(2)); // match title + terms status_code (text field) + + // Should and must_not unchanged + assertThat(rewrittenBool.should().size(), equalTo(1)); + assertThat(rewrittenBool.mustNot().size(), equalTo(1)); + } + + public void testRewriterPriority() { + // Verify rewriter has correct priority + assertThat(rewriter.priority(), equalTo(150)); + assertThat(rewriter.name(), equalTo("must_to_filter")); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/rewriters/TermsMergingRewriterTests.java b/server/src/test/java/org/opensearch/search/query/rewriters/TermsMergingRewriterTests.java new file mode 100644 index 0000000000000..085f2c72c67c9 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/rewriters/TermsMergingRewriterTests.java @@ -0,0 +1,292 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query.rewriters; + +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; + +public class TermsMergingRewriterTests extends OpenSearchTestCase { + + private final TermsMergingRewriter rewriter = TermsMergingRewriter.INSTANCE; + private final QueryShardContext context = mock(QueryShardContext.class); + + public void testSimpleTermMergingBelowThreshold() { + // Few term queries on same field should NOT be merged (below threshold) + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("status", "active")) + .filter(QueryBuilders.termQuery("status", "pending")) + .filter(QueryBuilders.termQuery("status", "approved")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes expected + } + + public void testTermMergingAboveThreshold() { + // Many term queries on same field should be merged (above threshold of 16) + BoolQueryBuilder query = QueryBuilders.boolQuery(); + // Add 20 term queries for the same field + for (int i = 0; i < 20; i++) { + query.filter(QueryBuilders.termQuery("category", "cat_" + i)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have one terms query instead of 20 term queries + assertThat(rewrittenBool.filter().size(), equalTo(1)); + assertThat(rewrittenBool.filter().get(0), instanceOf(TermsQueryBuilder.class)); + + TermsQueryBuilder termsQuery = (TermsQueryBuilder) rewrittenBool.filter().get(0); + assertThat(termsQuery.fieldName(), equalTo("category")); + assertThat(termsQuery.values().size(), equalTo(20)); + } + + public void testMustClauseNoMerging() { + // Term queries in must clauses should NOT be merged (different semantics) + QueryBuilder query = QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery("category", "electronics")) + .must(QueryBuilders.termQuery("category", "computers")) + .must(QueryBuilders.rangeQuery("price").gt(100)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have 3 queries: NO merging in must clause + assertThat(rewrittenBool.must().size(), equalTo(3)); + } + + public void testShouldClauseMergingBelowThreshold() { + // Should clauses with few terms should NOT be merged + QueryBuilder query = QueryBuilders.boolQuery() + .should(QueryBuilders.termQuery("color", "red")) + .should(QueryBuilders.termQuery("color", "blue")) + .should(QueryBuilders.termQuery("size", "large")) + .should(QueryBuilders.termQuery("size", "medium")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes expected + } + + public void testShouldClauseMergingAboveThreshold() { + // Should clauses with many terms should be merged + BoolQueryBuilder query = QueryBuilders.boolQuery(); + + // Add 20 color terms + for (int i = 0; i < 20; i++) { + query.should(QueryBuilders.termQuery("color", "color_" + i)); + } + + // Add 18 size terms + for (int i = 0; i < 18; i++) { + query.should(QueryBuilders.termQuery("size", "size_" + i)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have 2 terms queries: one for color, one for size + assertThat(rewrittenBool.should().size(), equalTo(2)); + assertThat(rewrittenBool.should().get(0), instanceOf(TermsQueryBuilder.class)); + assertThat(rewrittenBool.should().get(1), instanceOf(TermsQueryBuilder.class)); + } + + public void testMixedFieldsNoMerging() { + // Term queries on different fields should not be merged + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("field1", "value1")) + .filter(QueryBuilders.termQuery("field2", "value2")) + .filter(QueryBuilders.termQuery("field3", "value3")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes expected + } + + public void testExistingTermsQueryExpansionBelowThreshold() { + // Existing terms query with few additional terms should NOT be expanded (below threshold) + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.termsQuery("status", "active", "pending")) + .filter(QueryBuilders.termQuery("status", "approved")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes expected + } + + public void testExistingTermsQueryExpansionAboveThreshold() { + // Existing terms query should be expanded when total terms exceed threshold + String[] initialTerms = new String[14]; + for (int i = 0; i < 14; i++) { + initialTerms[i] = "status_" + i; + } + + BoolQueryBuilder query = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery("status", initialTerms)); + + // Add 5 more term queries to exceed threshold + for (int i = 14; i < 19; i++) { + query.filter(QueryBuilders.termQuery("status", "status_" + i)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Should have one terms query with all values + assertThat(rewrittenBool.filter().size(), equalTo(1)); + TermsQueryBuilder termsQuery = (TermsQueryBuilder) rewrittenBool.filter().get(0); + assertThat(termsQuery.values().size(), equalTo(19)); + } + + public void testSingleTermQuery() { + // Single term query should not be converted to terms query + QueryBuilder query = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("field", "value")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testMustNotClauseNoMerging() { + // Must_not clauses should not be merged + QueryBuilder query = QueryBuilders.boolQuery() + .mustNot(QueryBuilders.termQuery("status", "deleted")) + .mustNot(QueryBuilders.termQuery("status", "archived")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes to must_not + } + + public void testNestedBooleanQuery() { + // Should handle nested boolean queries with many terms + BoolQueryBuilder nested = QueryBuilders.boolQuery(); + // Add 20 term queries to exceed threshold + for (int i = 0; i < 20; i++) { + nested.filter(QueryBuilders.termQuery("status", "status_" + i)); + } + + QueryBuilder query = QueryBuilders.boolQuery().must(nested).filter(QueryBuilders.termQuery("type", "product")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + // Nested bool should also be rewritten + assertThat(rewrittenBool.must().size(), equalTo(1)); + BoolQueryBuilder nestedRewritten = (BoolQueryBuilder) rewrittenBool.must().get(0); + assertThat(nestedRewritten.filter().size(), equalTo(1)); + assertThat(nestedRewritten.filter().get(0), instanceOf(TermsQueryBuilder.class)); + + TermsQueryBuilder termsQuery = (TermsQueryBuilder) nestedRewritten.filter().get(0); + assertThat(termsQuery.values().size(), equalTo(20)); + } + + public void testEmptyBooleanQuery() { + // Empty boolean query should not cause issues + QueryBuilder query = QueryBuilders.boolQuery(); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testNonBooleanQuery() { + // Non-boolean queries should be returned as-is + QueryBuilder query = QueryBuilders.termQuery("field", "value"); + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); + } + + public void testBoostPreservation() { + // Boost values should be preserved when merging many terms + BoolQueryBuilder query = QueryBuilders.boolQuery(); + + // Add 20 terms with same boost + for (int i = 0; i < 20; i++) { + query.filter(QueryBuilders.termQuery("status", "status_" + i).boost(2.0f)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertThat(rewritten, instanceOf(BoolQueryBuilder.class)); + BoolQueryBuilder rewrittenBool = (BoolQueryBuilder) rewritten; + + assertThat(rewrittenBool.filter().size(), equalTo(1)); + TermsQueryBuilder termsQuery = (TermsQueryBuilder) rewrittenBool.filter().get(0); + assertThat(termsQuery.boost(), equalTo(2.0f)); + assertThat(termsQuery.values().size(), equalTo(20)); + } + + public void testMixedBoostNoMerging() { + // Different boost values should prevent merging + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("status", "active").boost(1.0f)) + .filter(QueryBuilders.termQuery("status", "pending").boost(2.0f)); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes due to different boosts + } + + public void testLargeTermsMerging() { + // Test merging a large number of term queries + BoolQueryBuilder query = QueryBuilders.boolQuery(); + for (int i = 0; i < 50; i++) { + query.filter(QueryBuilders.termQuery("field", "value" + i)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + BoolQueryBuilder result = (BoolQueryBuilder) rewritten; + + assertThat(result.filter().size(), equalTo(1)); + assertThat(result.filter().get(0), instanceOf(TermsQueryBuilder.class)); + TermsQueryBuilder terms = (TermsQueryBuilder) result.filter().get(0); + assertThat(terms.values().size(), equalTo(50)); + } + + public void testMixedTermsAndTermQueriesBelowThreshold() { + // Mix of existing terms queries and term queries with few values + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.termsQuery("field", "v1", "v2")) + .filter(QueryBuilders.termQuery("field", "v3")) + .filter(QueryBuilders.termsQuery("field", "v4", "v5")) + .filter(QueryBuilders.termQuery("field", "v6")); + + QueryBuilder rewritten = rewriter.rewrite(query, context); + assertSame(query, rewritten); // No changes expected (total 6 values < 16) + } + + public void testMixedTermsAndTermQueriesAboveThreshold() { + // Mix of existing terms queries and term queries with many values + String[] initialValues = new String[10]; + for (int i = 0; i < 10; i++) { + initialValues[i] = "v" + i; + } + + BoolQueryBuilder query = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery("field", initialValues)); + + // Add more term queries to exceed threshold + for (int i = 10; i < 20; i++) { + query.filter(QueryBuilders.termQuery("field", "v" + i)); + } + + QueryBuilder rewritten = rewriter.rewrite(query, context); + BoolQueryBuilder result = (BoolQueryBuilder) rewritten; + + // Should merge all into a single terms query + assertThat(result.filter().size(), equalTo(1)); + assertThat(result.filter().get(0), instanceOf(TermsQueryBuilder.class)); + TermsQueryBuilder merged = (TermsQueryBuilder) result.filter().get(0); + assertThat(merged.values().size(), equalTo(20)); + } + +} diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java index 09a66a2cfcd91..57be372b81359 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/CategoryContextMappingTests.java @@ -787,7 +787,7 @@ public void testUnknownQueryContextParsing() throws Exception { .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.EMPTY, "type1", mapping).mapperService(); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping).mapperService(); CompletionFieldType completionFieldType = (CompletionFieldType) mapperService.fieldType("completion"); Exception e = expectThrows(IllegalArgumentException.class, () -> completionFieldType.getContextMappings().get("brand")); diff --git a/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java b/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java index 07f3526dd2bb0..88898fb8cf963 100644 --- a/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java +++ b/server/src/test/java/org/opensearch/search/suggest/completion/GeoContextMappingTests.java @@ -77,7 +77,7 @@ public void testIndexingWithNoContexts() throws Exception { .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping).mapperService(); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping).mapperService(); MappedFieldType completionFieldType = mapperService.fieldType("completion"); ParsedDocument parsedDocument = mapperService.documentMapper() .parse( @@ -124,7 +124,7 @@ public void testIndexingWithSimpleContexts() throws Exception { .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping).mapperService(); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping).mapperService(); MappedFieldType completionFieldType = mapperService.fieldType("completion"); ParsedDocument parsedDocument = mapperService.documentMapper() .parse( @@ -169,7 +169,7 @@ public void testIndexingWithContextList() throws Exception { .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping).mapperService(); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping).mapperService(); MappedFieldType completionFieldType = mapperService.fieldType("completion"); ParsedDocument parsedDocument = mapperService.documentMapper() .parse( @@ -222,7 +222,7 @@ public void testIndexingWithMultipleContexts() throws Exception { .endObject() .endObject(); - MapperService mapperService = createIndex("test", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping).mapperService(); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping).mapperService(); MappedFieldType completionFieldType = mapperService.fieldType("completion"); XContentBuilder builder = jsonBuilder().startObject() .startArray("completion") @@ -268,10 +268,7 @@ public void testMalformedGeoField() throws Exception { mapping.endObject(); mapping.endObject(); - OpenSearchParseException ex = expectThrows( - OpenSearchParseException.class, - () -> createIndex("test", Settings.EMPTY, "type1", mapping) - ); + OpenSearchParseException ex = expectThrows(OpenSearchParseException.class, () -> createIndex("test", Settings.EMPTY, mapping)); assertThat(ex.getMessage(), equalTo("field [pin] referenced in context [st] must be mapped to geo_point, found [" + type + "]")); } @@ -298,10 +295,7 @@ public void testMissingGeoField() throws Exception { mapping.endObject(); mapping.endObject(); - OpenSearchParseException ex = expectThrows( - OpenSearchParseException.class, - () -> createIndex("test", Settings.EMPTY, "type1", mapping) - ); + OpenSearchParseException ex = expectThrows(OpenSearchParseException.class, () -> createIndex("test", Settings.EMPTY, mapping)); assertThat(ex.getMessage(), equalTo("field [pin] referenced in context [st] is not defined in the mapping")); } diff --git a/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java index f733154c643da..f0090c05634d7 100644 --- a/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java @@ -22,7 +22,6 @@ import org.opensearch.action.admin.indices.close.CloseIndexResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.opensearch.action.admin.indices.open.OpenIndexRequest; import org.opensearch.action.admin.indices.open.OpenIndexResponse; import org.opensearch.action.bulk.BulkRequest; @@ -118,13 +117,12 @@ public static Collection parameters() { public void cleanup() throws InterruptedException { final CountDownLatch allDeleted = new CountDownLatch(3); for (String indexName : new String[] { indexName, renamedIndexName }) { - final StepListener existsIndexResponseStepListener = new StepListener<>(); - client().admin().indices().exists(new IndicesExistsRequest(indexName), existsIndexResponseStepListener); - continueOrDie(existsIndexResponseStepListener, resp -> { + client().admin().indices().existsAsync(new IndicesExistsRequest(indexName)).thenAccept(resp -> { if (resp.isExists()) { - final StepListener deleteIndexResponseStepListener = new StepListener<>(); - client().admin().indices().delete(new DeleteIndexRequest(indexName), deleteIndexResponseStepListener); - continueOrDie(deleteIndexResponseStepListener, ignored -> allDeleted.countDown()); + client().admin() + .indices() + .deleteAsync(new DeleteIndexRequest(indexName)) + .thenAccept(ignored -> { allDeleted.countDown(); }); } else { allDeleted.countDown(); } @@ -218,11 +216,9 @@ public void testRestoreWithRename() throws Exception { final CountDownLatch isRestorable = new CountDownLatch(1); if (!this.exists && !this.renameIndexes) { - final StepListener deleteIndexResponseStepListener = new StepListener<>(); continueOrDie(createSnapshotResponseStepListener, ignored -> { - client().admin().indices().delete(new DeleteIndexRequest(indexName), deleteIndexResponseStepListener); + client().admin().indices().deleteAsync(new DeleteIndexRequest(indexName)).thenAccept(r -> isRestorable.countDown()); }); - continueOrDie(deleteIndexResponseStepListener, ignored -> isRestorable.countDown()); } else { continueOrDie(createSnapshotResponseStepListener, ignored -> isRestorable.countDown()); } @@ -243,7 +239,7 @@ public void testRestoreWithRename() throws Exception { restoreSnapshotResponseStepListener.whenComplete(ignored -> { isRestored.countDown(); - assertTrue("unexpected sucesssful restore", expectSuccess); + assertTrue("unexpected successful restore", expectSuccess); }, e -> { isRestored.countDown(); if (expectSuccess) { diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle index bba7fb9f51857..91dd493a635d1 100644 --- a/test/fixtures/hdfs-fixture/build.gradle +++ b/test/fixtures/hdfs-fixture/build.gradle @@ -90,6 +90,6 @@ dependencies { runtimeOnly("com.squareup.okhttp3:okhttp:5.1.0") { exclude group: "com.squareup.okio" } - runtimeOnly "com.squareup.okio:okio:3.15.0" + runtimeOnly "com.squareup.okio:okio:3.16.0" runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.8" } diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 9e8c3239f5197..fe6e38e1b3e48 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -553,7 +553,8 @@ protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSup () -> SequenceNumbers.NO_OPS_PERFORMED, primaryTermSupplier, seqNo -> {}, - TranslogOperationHelper.create(engine.config()) + TranslogOperationHelper.create(engine.config()), + null ); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 812109c5df6b8..6d537c95725b3 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -193,7 +193,7 @@ import java.lang.annotation.Target; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -2017,9 +2017,9 @@ private ExternalTestCluster buildExternalCluster(String clusterAddresses, String TransportAddress[] transportAddresses = new TransportAddress[stringAddresses.length]; int i = 0; for (String stringAddress : stringAddresses) { - URL url = new URL("http://" + stringAddress); - InetAddress inetAddress = InetAddress.getByName(url.getHost()); - transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, url.getPort())); + URI uri = URI.create("http://" + stringAddress); + InetAddress inetAddress = InetAddress.getByName(uri.getHost()); + transportAddresses[i++] = new TransportAddress(new InetSocketAddress(inetAddress, uri.getPort())); } return new ExternalTestCluster( createTempDir(), diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java index 1d84eeca9c6c2..48a5184349658 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java @@ -339,17 +339,15 @@ protected IndexService createIndex(String index) { * Create a new index on the singleton node with the provided index settings. */ protected IndexService createIndex(String index, Settings settings) { - return createIndex(index, settings, null, (XContentBuilder) null); + return createIndex(index, settings, null); } /** - * Create a new index on the singleton node with the provided index settings. - * @deprecated types are being removed + * Create a new index on the singleton node with the provided index settings and mappings. */ - @Deprecated - protected IndexService createIndex(String index, Settings settings, String type, XContentBuilder mappings) { + protected IndexService createIndex(String index, Settings settings, XContentBuilder mappings) { CreateIndexRequestBuilder createIndexRequestBuilder = client().admin().indices().prepareCreate(index).setSettings(settings); - if (type != null && mappings != null) { + if (mappings != null) { createIndexRequestBuilder.setMapping(mappings); } return createIndex(index, createIndexRequestBuilder); @@ -357,15 +355,20 @@ protected IndexService createIndex(String index, Settings settings, String type, /** * Create a new index on the singleton node with the provided index settings. - * @deprecated types are being removed + * @deprecated types have been removed + */ + @Deprecated + protected IndexService createIndex(String index, Settings settings, String type, XContentBuilder mappings) { + return createIndex(index, settings, mappings); + } + + /** + * Create a new index on the singleton node with the provided index settings. + * @deprecated types have been removed */ @Deprecated protected IndexService createIndex(String index, Settings settings, String type, String... mappings) { - CreateIndexRequestBuilder createIndexRequestBuilder = client().admin().indices().prepareCreate(index).setSettings(settings); - if (mappings != null) { - createIndexRequestBuilder.setMapping(mappings); - } - return createIndex(index, createIndexRequestBuilder); + return createIndexWithSimpleMappings(index, settings, mappings); } /**