diff --git a/.ci/os.ps1 b/.ci/os.ps1 index 54f5ca54ffaa3..0fa43c4b250fa 100644 --- a/.ci/os.ps1 +++ b/.ci/os.ps1 @@ -1,3 +1,5 @@ +param($GradleTasks='destructiveDistroTest') + If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { # Relaunch as an elevated process: @@ -25,7 +27,6 @@ Remove-Item -Recurse -Force \tmp -ErrorAction Ignore New-Item -ItemType directory -Path \tmp $ErrorActionPreference="Continue" -# TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed -& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain destructiveDistroTest +& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks exit $LastExitCode diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java new file mode 100644 index 0000000000000..adef011abfb1b --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java @@ -0,0 +1,172 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.search.aggregations.bucket.terms; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.PageCacheRecycler; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.terms.LongKeyedBucketOrds; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +@Fork(2) +@Warmup(iterations = 10) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@OperationsPerInvocation(1_000_000) +@State(Scope.Benchmark) +public class LongKeyedBucketOrdsBenchmark { + private static final long LIMIT = 1_000_000; + /** + * The number of distinct values to add to the buckets. + */ + private static final long DISTINCT_VALUES = 10; + /** + * The number of buckets to create in the {@link #multiBucket} case. + *

+ * If this is not relatively prime to {@link #DISTINCT_VALUES} then the + * values won't be scattered evenly across the buckets. + */ + private static final long DISTINCT_BUCKETS = 21; + + private final PageCacheRecycler recycler = new PageCacheRecycler(Settings.EMPTY); + private final BigArrays bigArrays = new BigArrays(recycler, null, "REQUEST"); + + /** + * Force loading all of the implementations just for extra paranoia's sake. + * We really don't want the JVM to be able to eliminate one of them just + * because we don't use it in the particular benchmark. That is totally a + * thing it'd do. It is sneaky. + */ + @Setup + public void forceLoadClasses(Blackhole bh) { + bh.consume(LongKeyedBucketOrds.FromSingle.class); + bh.consume(LongKeyedBucketOrds.FromMany.class); + } + + /** + * Emulates a way that we do not use {@link LongKeyedBucketOrds} + * because it is not needed. + */ + @Benchmark + public void singleBucketIntoSingleImmutableMonmorphicInvocation(Blackhole bh) { + try (LongKeyedBucketOrds.FromSingle ords = new LongKeyedBucketOrds.FromSingle(bigArrays)) { + for (long i = 0; i < LIMIT; i++) { + ords.add(0, i % DISTINCT_VALUES); + } + bh.consume(ords); + } + } + + /** + * Emulates the way that most aggregations use {@link LongKeyedBucketOrds}. + */ + @Benchmark + public void singleBucketIntoSingleImmutableBimorphicInvocation(Blackhole bh) { + try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE)) { + for (long i = 0; i < LIMIT; i++) { + ords.add(0, i % DISTINCT_VALUES); + } + bh.consume(ords); + } + } + + /** + * Emulates the way that {@link AutoDateHistogramAggregationBuilder} uses {@link LongKeyedBucketOrds}. + */ + @Benchmark + public void singleBucketIntoSingleMutableMonmorphicInvocation(Blackhole bh) { + LongKeyedBucketOrds.FromSingle ords = new LongKeyedBucketOrds.FromSingle(bigArrays); + for (long i = 0; i < LIMIT; i++) { + if (i % 100_000 == 0) { + ords.close(); + bh.consume(ords); + ords = new LongKeyedBucketOrds.FromSingle(bigArrays); + } + ords.add(0, i % DISTINCT_VALUES); + } + bh.consume(ords); + ords.close(); + } + + /** + * Emulates a way that we do not use {@link LongKeyedBucketOrds} + * because it is significantly slower than the + * {@link #singleBucketIntoSingleMutableMonmorphicInvocation monomorphic invocation}. + */ + @Benchmark + public void singleBucketIntoSingleMutableBimorphicInvocation(Blackhole bh) { + LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE); + for (long i = 0; i < LIMIT; i++) { + if (i % 100_000 == 0) { + ords.close(); + bh.consume(ords); + ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE); + } + ords.add(0, i % DISTINCT_VALUES); + + } + bh.consume(ords); + ords.close(); + } + + /** + * Emulates an aggregation that collects from a single bucket "by accident". + * This can happen if an aggregation is under, say, a {@code terms} + * aggregation and there is only a single value for that term in the index. + */ + @Benchmark + public void singleBucketIntoMulti(Blackhole bh) { + try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.MANY)) { + for (long i = 0; i < LIMIT; i++) { + ords.add(0, i % DISTINCT_VALUES); + } + bh.consume(ords); + } + } + + /** + * Emulates an aggregation that collects from many buckets. + */ + @Benchmark + public void multiBucket(Blackhole bh) { + try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.MANY)) { + for (long i = 0; i < LIMIT; i++) { + ords.add(i % DISTINCT_BUCKETS, i % DISTINCT_VALUES); + } + bh.consume(ords); + } + } +} diff --git a/build.gradle b/build.gradle index a1fa67b65215c..14419937ea9ca 100644 --- a/build.gradle +++ b/build.gradle @@ -174,8 +174,8 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = false -final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/59076" /* place a PR link here when committing bwc changes */ +boolean bwc_tests_enabled = true +final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 1d6c03666d84f..a8d1958218193 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -24,6 +24,7 @@ import groovy.transform.CompileStatic import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.ElasticsearchJavaPlugin import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask +import org.elasticsearch.gradle.RepositoriesSetupPlugin import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin import org.elasticsearch.gradle.precommit.PrecommitTasks @@ -59,9 +60,9 @@ class StandaloneRestTestPlugin implements Plugin { project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin) project.pluginManager.apply(JavaBasePlugin) project.pluginManager.apply(TestClustersPlugin) + project.pluginManager.apply(RepositoriesSetupPlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) - ElasticsearchJavaPlugin.configureRepositories(project) ElasticsearchJavaPlugin.configureTestTasks(project) ElasticsearchJavaPlugin.configureInputNormalization(project) ElasticsearchJavaPlugin.configureCompile(project) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java index f56ea9cb569fa..2dccc95f82508 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java @@ -36,9 +36,6 @@ import org.gradle.api.artifacts.ModuleDependency; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.artifacts.ResolutionStrategy; -import org.gradle.api.artifacts.dsl.RepositoryHandler; -import org.gradle.api.artifacts.repositories.IvyArtifactRepository; -import org.gradle.api.artifacts.repositories.MavenArtifactRepository; import org.gradle.api.execution.TaskActionListener; import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.BasePlugin; @@ -59,16 +56,10 @@ import java.io.File; import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URI; -import java.util.Arrays; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; import static org.elasticsearch.gradle.util.Util.toStringable; @@ -83,11 +74,11 @@ public void apply(Project project) { project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); // apply global test task failure listener project.getRootProject().getPluginManager().apply(TestFailureReportingPlugin.class); - + // common repositories setup + project.getPluginManager().apply(RepositoriesSetupPlugin.class); project.getPluginManager().apply(JavaLibraryPlugin.class); configureConfigurations(project); - configureRepositories(project); configureCompile(project); configureInputNormalization(project); configureTestTasks(project); @@ -149,77 +140,6 @@ public static void configureConfigurations(Project project) { disableTransitiveDeps.accept(JavaPlugin.TEST_IMPLEMENTATION_CONFIGURATION_NAME); } - private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)"); - - /** - * Adds repositories used by ES dependencies - */ - public static void configureRepositories(Project project) { - // ensure all repositories use secure urls - // TODO: remove this with gradle 7.0, which no longer allows insecure urls - project.getRepositories().all(repository -> { - if (repository instanceof MavenArtifactRepository) { - final MavenArtifactRepository maven = (MavenArtifactRepository) repository; - assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl()); - for (URI uri : maven.getArtifactUrls()) { - assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri); - } - } else if (repository instanceof IvyArtifactRepository) { - final IvyArtifactRepository ivy = (IvyArtifactRepository) repository; - assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl()); - } - }); - RepositoryHandler repos = project.getRepositories(); - if (System.getProperty("repos.mavenLocal") != null) { - // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is - // useful for development ie. bwc tests where we install stuff in the local repository - // such that we don't have to pass hardcoded files to gradle - repos.mavenLocal(); - } - repos.jcenter(); - - String luceneVersion = VersionProperties.getLucene(); - if (luceneVersion.contains("-snapshot")) { - // extract the revision number from the version with a regex matcher - Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion); - if (matcher.find() == false) { - throw new GradleException("Malformed lucene snapshot version: " + luceneVersion); - } - String revision = matcher.group(1); - MavenArtifactRepository luceneRepo = repos.maven(repo -> { - repo.setName("lucene-snapshots"); - repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision); - }); - repos.exclusiveContent(exclusiveRepo -> { - exclusiveRepo.filter( - descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision) - ); - exclusiveRepo.forRepositories(luceneRepo); - }); - } - } - - private static final List SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3"); - - private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { - if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) { - String url; - try { - url = uri.toURL().toString(); - } catch (MalformedURLException e) { - throw new IllegalStateException(e); - } - final String message = String.format( - Locale.ROOT, - "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", - repositoryName, - projectPath, - url - ); - throw new GradleException(message); - } - } - /** * Adds compiler settings to the project */ diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java index 18c24db09fee2..efbba73bf5467 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java @@ -218,6 +218,7 @@ private class PropertyListEntry { this.normalization = normalization; } + @Input public PropertyNormalization getNormalization() { return normalization; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java new file mode 100644 index 0000000000000..3ea2fee7d1483 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/RepositoriesSetupPlugin.java @@ -0,0 +1,114 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +import org.gradle.api.GradleException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.repositories.MavenArtifactRepository; + +import java.net.MalformedURLException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RepositoriesSetupPlugin implements Plugin { + + private static final List SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3"); + private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)"); + + @Override + public void apply(Project project) { + configureRepositories(project); + } + + /** + * Adds repositories used by ES projects and dependencies + */ + public static void configureRepositories(Project project) { + // ensure all repositories use secure urls + // TODO: remove this with gradle 7.0, which no longer allows insecure urls + project.getRepositories().all(repository -> { + if (repository instanceof MavenArtifactRepository) { + final MavenArtifactRepository maven = (MavenArtifactRepository) repository; + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl()); + for (URI uri : maven.getArtifactUrls()) { + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri); + } + } else if (repository instanceof IvyArtifactRepository) { + final IvyArtifactRepository ivy = (IvyArtifactRepository) repository; + assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl()); + } + }); + RepositoryHandler repos = project.getRepositories(); + if (System.getProperty("repos.mavenLocal") != null) { + // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is + // useful for development ie. bwc tests where we install stuff in the local repository + // such that we don't have to pass hardcoded files to gradle + repos.mavenLocal(); + } + repos.jcenter(); + + String luceneVersion = VersionProperties.getLucene(); + if (luceneVersion.contains("-snapshot")) { + // extract the revision number from the version with a regex matcher + Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion); + if (matcher.find() == false) { + throw new GradleException("Malformed lucene snapshot version: " + luceneVersion); + } + String revision = matcher.group(1); + MavenArtifactRepository luceneRepo = repos.maven(repo -> { + repo.setName("lucene-snapshots"); + repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision); + }); + repos.exclusiveContent(exclusiveRepo -> { + exclusiveRepo.filter( + descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision) + ); + exclusiveRepo.forRepositories(luceneRepo); + }); + } + } + + private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { + if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) { + String url; + try { + url = uri.toURL().toString(); + } catch (MalformedURLException e) { + throw new IllegalStateException(e); + } + final String message = String.format( + Locale.ROOT, + "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", + repositoryName, + projectPath, + url + ); + throw new GradleException(message); + } + } + +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties new file mode 100644 index 0000000000000..56113f18b6c5d --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.repositories.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.RepositoriesSetupPlugin diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index f380dfabd321f..7561c6601f020 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -98,9 +98,6 @@ testClusters.all { setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.http.ssl.enabled', 'false' setting 'xpack.security.transport.ssl.enabled', 'false' - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.eql_feature_flag_registered', 'true' - } setting 'xpack.eql.enabled', 'true' // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt' diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 4c21067d9519a..44d0cb6994925 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -54,6 +54,8 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.analytics.InferencePipelineAggregationBuilder; +import org.elasticsearch.client.analytics.ParsedInference; import org.elasticsearch.client.analytics.ParsedStringStats; import org.elasticsearch.client.analytics.ParsedTopMetrics; import org.elasticsearch.client.analytics.StringStatsAggregationBuilder; @@ -1957,6 +1959,7 @@ static List getDefaultNamedXContents() { map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c)); map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c)); map.put(TopMetricsAggregationBuilder.NAME, (p, c) -> ParsedTopMetrics.PARSER.parse(p, (String) c)); + map.put(InferencePipelineAggregationBuilder.NAME, (p, c) -> ParsedInference.fromXContent(p, (String ) (c))); List entries = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) .collect(Collectors.toList()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/InferencePipelineAggregationBuilder.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/InferencePipelineAggregationBuilder.java new file mode 100644 index 0000000000000..05a24a08e4c59 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/InferencePipelineAggregationBuilder.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.analytics; + +import org.elasticsearch.client.ml.inference.trainedmodel.InferenceConfig; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * For building inference pipeline aggregations + * + * NOTE: This extends {@linkplain AbstractPipelineAggregationBuilder} for compatibility + * with {@link SearchSourceBuilder#aggregation(PipelineAggregationBuilder)} but it + * doesn't support any "server" side things like {@linkplain #doWriteTo(StreamOutput)} + * or {@linkplain #createInternal(Map)} + */ +public class InferencePipelineAggregationBuilder extends AbstractPipelineAggregationBuilder { + + public static String NAME = "inference"; + + public static final ParseField MODEL_ID = new ParseField("model_id"); + private static final ParseField INFERENCE_CONFIG = new ParseField("inference_config"); + + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + NAME, false, + (args, name) -> new InferencePipelineAggregationBuilder(name, (String)args[0], (Map) args[1]) + ); + + static { + PARSER.declareString(constructorArg(), MODEL_ID); + PARSER.declareObject(constructorArg(), (p, c) -> p.mapStrings(), BUCKETS_PATH_FIELD); + PARSER.declareNamedObject(InferencePipelineAggregationBuilder::setInferenceConfig, + (p, c, n) -> p.namedObject(InferenceConfig.class, n, c), INFERENCE_CONFIG); + } + + private final Map bucketPathMap; + private final String modelId; + private InferenceConfig inferenceConfig; + + public static InferencePipelineAggregationBuilder parse(String pipelineAggregatorName, + XContentParser parser) { + return PARSER.apply(parser, pipelineAggregatorName); + } + + public InferencePipelineAggregationBuilder(String name, String modelId, Map bucketsPath) { + super(name, NAME, new TreeMap<>(bucketsPath).values().toArray(new String[] {})); + this.modelId = modelId; + this.bucketPathMap = bucketsPath; + } + + public void setInferenceConfig(InferenceConfig inferenceConfig) { + this.inferenceConfig = inferenceConfig; + } + + @Override + protected void validate(ValidationContext context) { + // validation occurs on the server + } + + @Override + protected void doWriteTo(StreamOutput out) { + throw new UnsupportedOperationException(); + } + + @Override + protected PipelineAggregator createInternal(Map metaData) { + throw new UnsupportedOperationException(); + } + + @Override + protected boolean overrideBucketsPath() { + return true; + } + + @Override + protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(MODEL_ID.getPreferredName(), modelId); + builder.field(BUCKETS_PATH_FIELD.getPreferredName(), bucketPathMap); + if (inferenceConfig != null) { + builder.startObject(INFERENCE_CONFIG.getPreferredName()); + builder.field(inferenceConfig.getName(), inferenceConfig); + builder.endObject(); + } + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), bucketPathMap, modelId, inferenceConfig); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (super.equals(obj) == false) return false; + + InferencePipelineAggregationBuilder other = (InferencePipelineAggregationBuilder) obj; + return Objects.equals(bucketPathMap, other.bucketPathMap) + && Objects.equals(modelId, other.modelId) + && Objects.equals(inferenceConfig, other.inferenceConfig); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedInference.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedInference.java new file mode 100644 index 0000000000000..4fe03fb4c5b5a --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/analytics/ParsedInference.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.analytics; + +import org.elasticsearch.client.ml.inference.results.FeatureImportance; +import org.elasticsearch.client.ml.inference.results.TopClassEntry; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.ParsedAggregation; + +import java.io.IOException; +import java.util.List; + +/** + * This class parses the superset of all possible fields that may be written by + * InferenceResults. The warning field is mutually exclusive with all the other fields. + * + * In the case of classification results {@link #getValue()} may return a String, + * Boolean or a Double. For regression results {@link #getValue()} is always + * a Double. + */ +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class ParsedInference extends ParsedAggregation { + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(ParsedInference.class.getSimpleName(), true, + args -> new ParsedInference(args[0], (List) args[1], + (List) args[2], (String) args[3])); + + public static final ParseField FEATURE_IMPORTANCE = new ParseField("feature_importance"); + public static final ParseField WARNING = new ParseField("warning"); + public static final ParseField TOP_CLASSES = new ParseField("top_classes"); + + static { + PARSER.declareField(optionalConstructorArg(), (p, n) -> { + Object o; + XContentParser.Token token = p.currentToken(); + if (token == XContentParser.Token.VALUE_STRING) { + o = p.text(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + o = p.booleanValue(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + o = p.doubleValue(); + } else { + throw new XContentParseException(p.getTokenLocation(), + "[" + ParsedInference.class.getSimpleName() + "] failed to parse field [" + CommonFields.VALUE + "] " + + "value [" + token + "] is not a string, boolean or number"); + } + return o; + }, CommonFields.VALUE, ObjectParser.ValueType.VALUE); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> FeatureImportance.fromXContent(p), FEATURE_IMPORTANCE); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> TopClassEntry.fromXContent(p), TOP_CLASSES); + PARSER.declareString(optionalConstructorArg(), WARNING); + declareAggregationFields(PARSER); + } + + public static ParsedInference fromXContent(XContentParser parser, final String name) { + ParsedInference parsed = PARSER.apply(parser, null); + parsed.setName(name); + return parsed; + } + + private final Object value; + private final List featureImportance; + private final List topClasses; + private final String warning; + + ParsedInference(Object value, + List featureImportance, + List topClasses, + String warning) { + this.value = value; + this.warning = warning; + this.featureImportance = featureImportance; + this.topClasses = topClasses; + } + + public Object getValue() { + return value; + } + + public List getFeatureImportance() { + return featureImportance; + } + + public List getTopClasses() { + return topClasses; + } + + public String getWarning() { + return warning; + } + + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + if (warning != null) { + builder.field(WARNING.getPreferredName(), warning); + } else { + builder.field(CommonFields.VALUE.getPreferredName(), value); + if (topClasses != null && topClasses.size() > 0) { + builder.field(TOP_CLASSES.getPreferredName(), topClasses); + } + if (featureImportance != null && featureImportance.size() > 0) { + builder.field(FEATURE_IMPORTANCE.getPreferredName(), featureImportance); + } + } + return builder; + } + + @Override + public String getType() { + return InferencePipelineAggregationBuilder.NAME; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index 7416803b8345a..10cda684ae2a7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.searchafter.SearchAfterBuilder; import java.io.IOException; import java.util.Arrays; @@ -40,12 +39,10 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private QueryBuilder filter = null; private String timestampField = "@timestamp"; private String eventCategoryField = "event.category"; - private String implicitJoinKeyField = "agent.id"; private boolean isCaseSensitive = true; private int size = 10; private int fetchSize = 1000; - private SearchAfterBuilder searchAfterBuilder; private String query; private String tiebreakerField; @@ -58,11 +55,9 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field"; static final String KEY_EVENT_CATEGORY_FIELD = "event_category_field"; - static final String KEY_IMPLICIT_JOIN_KEY_FIELD = "implicit_join_key_field"; static final String KEY_CASE_SENSITIVE = "case_sensitive"; static final String KEY_SIZE = "size"; static final String KEY_FETCH_SIZE = "fetch_size"; - static final String KEY_SEARCH_AFTER = "search_after"; static final String KEY_QUERY = "query"; static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; static final String KEY_KEEP_ALIVE = "keep_alive"; @@ -84,16 +79,8 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(KEY_TIEBREAKER_FIELD, tiebreakerField()); } builder.field(KEY_EVENT_CATEGORY_FIELD, eventCategoryField()); - if (implicitJoinKeyField != null) { - builder.field(KEY_IMPLICIT_JOIN_KEY_FIELD, implicitJoinKeyField()); - } builder.field(KEY_SIZE, size()); builder.field(KEY_FETCH_SIZE, fetchSize()); - - if (searchAfterBuilder != null) { - builder.array(KEY_SEARCH_AFTER, searchAfterBuilder.getSortValues()); - } - builder.field(KEY_CASE_SENSITIVE, isCaseSensitive()); builder.field(KEY_QUERY, query); @@ -156,10 +143,6 @@ public EqlSearchRequest eventCategoryField(String eventCategoryField) { return this; } - public String implicitJoinKeyField() { - return this.implicitJoinKeyField; - } - public boolean isCaseSensitive() { return this.isCaseSensitive; } @@ -169,12 +152,6 @@ public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) { return this; } - public EqlSearchRequest implicitJoinKeyField(String implicitJoinKeyField) { - Objects.requireNonNull(implicitJoinKeyField, "implicit join key must not be null"); - this.implicitJoinKeyField = implicitJoinKeyField; - return this; - } - public int size() { return this.size; } @@ -199,23 +176,6 @@ public EqlSearchRequest fetchSize(int fetchSize) { return this; } - public Object[] searchAfter() { - if (searchAfterBuilder == null) { - return null; - } - return searchAfterBuilder.getSortValues(); - } - - public EqlSearchRequest searchAfter(Object[] values) { - this.searchAfterBuilder = new SearchAfterBuilder().setSortValues(values); - return this; - } - - private EqlSearchRequest setSearchAfter(SearchAfterBuilder builder) { - this.searchAfterBuilder = builder; - return this; - } - public String query() { return this.query; } @@ -269,8 +229,6 @@ public boolean equals(Object o) { Objects.equals(timestampField, that.timestampField) && Objects.equals(tiebreakerField, that.tiebreakerField) && Objects.equals(eventCategoryField, that.eventCategoryField) && - Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) && - Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && Objects.equals(query, that.query) && Objects.equals(isCaseSensitive, that.isCaseSensitive) && Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && @@ -289,8 +247,6 @@ public int hashCode() { timestampField, tiebreakerField, eventCategoryField, - implicitJoinKeyField, - searchAfterBuilder, query, isCaseSensitive, waitForCompletionTimeout, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java index 7cc2000a44e67..0f79048261ac8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfig.java @@ -57,6 +57,7 @@ public static Builder builder() { static final ParseField CREATE_TIME = new ParseField("create_time"); static final ParseField VERSION = new ParseField("version"); static final ParseField ALLOW_LAZY_START = new ParseField("allow_lazy_start"); + static final ParseField MAX_NUM_THREADS = new ParseField("max_num_threads"); private static final ObjectParser PARSER = new ObjectParser<>("data_frame_analytics_config", true, Builder::new); @@ -80,6 +81,7 @@ public static Builder builder() { ValueType.VALUE); PARSER.declareString(Builder::setVersion, Version::fromString, VERSION); PARSER.declareBoolean(Builder::setAllowLazyStart, ALLOW_LAZY_START); + PARSER.declareInt(Builder::setMaxNumThreads, MAX_NUM_THREADS); } private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOException { @@ -100,11 +102,13 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser) throws IOE private final Instant createTime; private final Version version; private final Boolean allowLazyStart; + private final Integer maxNumThreads; private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String description, @Nullable DataFrameAnalyticsSource source, @Nullable DataFrameAnalyticsDest dest, @Nullable DataFrameAnalysis analysis, @Nullable FetchSourceContext analyzedFields, @Nullable ByteSizeValue modelMemoryLimit, - @Nullable Instant createTime, @Nullable Version version, @Nullable Boolean allowLazyStart) { + @Nullable Instant createTime, @Nullable Version version, @Nullable Boolean allowLazyStart, + @Nullable Integer maxNumThreads) { this.id = id; this.description = description; this.source = source; @@ -115,6 +119,7 @@ private DataFrameAnalyticsConfig(@Nullable String id, @Nullable String descripti this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli());; this.version = version; this.allowLazyStart = allowLazyStart; + this.maxNumThreads = maxNumThreads; } public String getId() { @@ -157,6 +162,10 @@ public Boolean getAllowLazyStart() { return allowLazyStart; } + public Integer getMaxNumThreads() { + return maxNumThreads; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -193,6 +202,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (allowLazyStart != null) { builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); } + if (maxNumThreads != null) { + builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); + } builder.endObject(); return builder; } @@ -212,12 +224,14 @@ public boolean equals(Object o) { && Objects.equals(modelMemoryLimit, other.modelMemoryLimit) && Objects.equals(createTime, other.createTime) && Objects.equals(version, other.version) - && Objects.equals(allowLazyStart, other.allowLazyStart); + && Objects.equals(allowLazyStart, other.allowLazyStart) + && Objects.equals(maxNumThreads, other.maxNumThreads); } @Override public int hashCode() { - return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version, allowLazyStart); + return Objects.hash(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, version, allowLazyStart, + maxNumThreads); } @Override @@ -237,6 +251,7 @@ public static class Builder { private Instant createTime; private Version version; private Boolean allowLazyStart; + private Integer maxNumThreads; private Builder() {} @@ -290,9 +305,14 @@ public Builder setAllowLazyStart(Boolean allowLazyStart) { return this; } + public Builder setMaxNumThreads(Integer maxNumThreads) { + this.maxNumThreads = maxNumThreads; + return this; + } + public DataFrameAnalyticsConfig build() { return new DataFrameAnalyticsConfig(id, description, source, dest, analysis, analyzedFields, modelMemoryLimit, createTime, - version, allowLazyStart); + version, allowLazyStart, maxNumThreads); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdate.java index 1d5ecb6657762..f6bda01bcf3b3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdate.java @@ -51,22 +51,25 @@ public static Builder builder() { DataFrameAnalyticsConfig.MODEL_MEMORY_LIMIT, VALUE); PARSER.declareBoolean(Builder::setAllowLazyStart, DataFrameAnalyticsConfig.ALLOW_LAZY_START); - + PARSER.declareInt(Builder::setMaxNumThreads, DataFrameAnalyticsConfig.MAX_NUM_THREADS); } private final String id; private final String description; private final ByteSizeValue modelMemoryLimit; private final Boolean allowLazyStart; + private final Integer maxNumThreads; private DataFrameAnalyticsConfigUpdate(String id, @Nullable String description, @Nullable ByteSizeValue modelMemoryLimit, - @Nullable Boolean allowLazyStart) { + @Nullable Boolean allowLazyStart, + @Nullable Integer maxNumThreads) { this.id = id; this.description = description; this.modelMemoryLimit = modelMemoryLimit; this.allowLazyStart = allowLazyStart; + this.maxNumThreads = maxNumThreads; } public String getId() { @@ -85,6 +88,10 @@ public Boolean isAllowLazyStart() { return allowLazyStart; } + public Integer getMaxNumThreads() { + return maxNumThreads; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -98,6 +105,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (allowLazyStart != null) { builder.field(DataFrameAnalyticsConfig.ALLOW_LAZY_START.getPreferredName(), allowLazyStart); } + if (maxNumThreads != null) { + builder.field(DataFrameAnalyticsConfig.MAX_NUM_THREADS.getPreferredName(), maxNumThreads); + } builder.endObject(); return builder; } @@ -117,12 +127,13 @@ public boolean equals(Object other) { return Objects.equals(this.id, that.id) && Objects.equals(this.description, that.description) && Objects.equals(this.modelMemoryLimit, that.modelMemoryLimit) - && Objects.equals(this.allowLazyStart, that.allowLazyStart); + && Objects.equals(this.allowLazyStart, that.allowLazyStart) + && Objects.equals(this.maxNumThreads, that.maxNumThreads); } @Override public int hashCode() { - return Objects.hash(id, description, modelMemoryLimit, allowLazyStart); + return Objects.hash(id, description, modelMemoryLimit, allowLazyStart, maxNumThreads); } public static class Builder { @@ -131,6 +142,7 @@ public static class Builder { private String description; private ByteSizeValue modelMemoryLimit; private Boolean allowLazyStart; + private Integer maxNumThreads; private Builder() {} @@ -158,8 +170,13 @@ public Builder setAllowLazyStart(Boolean allowLazyStart) { return this; } + public Builder setMaxNumThreads(Integer maxNumThreads) { + this.maxNumThreads = maxNumThreads; + return this; + } + public DataFrameAnalyticsConfigUpdate build() { - return new DataFrameAnalyticsConfigUpdate(id, description, modelMemoryLimit, allowLazyStart); + return new DataFrameAnalyticsConfigUpdate(id, description, modelMemoryLimit, allowLazyStart, maxNumThreads); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/FeatureImportance.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/FeatureImportance.java new file mode 100644 index 0000000000000..d6d0bd4b04f41 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/FeatureImportance.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml.inference.results; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class FeatureImportance implements ToXContentObject { + + public static final String IMPORTANCE = "importance"; + public static final String FEATURE_NAME = "feature_name"; + public static final String CLASS_IMPORTANCE = "class_importance"; + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("feature_importance", true, + a -> new FeatureImportance((String) a[0], (Double) a[1], (Map) a[2]) + ); + + static { + PARSER.declareString(constructorArg(), new ParseField(FeatureImportance.FEATURE_NAME)); + PARSER.declareDouble(constructorArg(), new ParseField(FeatureImportance.IMPORTANCE)); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(HashMap::new, XContentParser::doubleValue), + new ParseField(FeatureImportance.CLASS_IMPORTANCE)); + } + + public static FeatureImportance fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + private final Map classImportance; + private final double importance; + private final String featureName; + + public FeatureImportance(String featureName, double importance, Map classImportance) { + this.featureName = Objects.requireNonNull(featureName); + this.importance = importance; + this.classImportance = classImportance == null ? null : Collections.unmodifiableMap(classImportance); + } + + public Map getClassImportance() { + return classImportance; + } + + public double getImportance() { + return importance; + } + + public String getFeatureName() { + return featureName; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FEATURE_NAME, featureName); + builder.field(IMPORTANCE, importance); + if (classImportance != null && classImportance.isEmpty() == false) { + builder.startObject(CLASS_IMPORTANCE); + for (Map.Entry entry : classImportance.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object object) { + if (object == this) { return true; } + if (object == null || getClass() != object.getClass()) { return false; } + FeatureImportance that = (FeatureImportance) object; + return Objects.equals(featureName, that.featureName) + && Objects.equals(importance, that.importance) + && Objects.equals(classImportance, that.classImportance); + } + + @Override + public int hashCode() { + return Objects.hash(featureName, importance, classImportance); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/TopClassEntry.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/TopClassEntry.java new file mode 100644 index 0000000000000..9afd663f6812d --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/results/TopClassEntry.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml.inference.results; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class TopClassEntry implements ToXContentObject { + + public static final ParseField CLASS_NAME = new ParseField("class_name"); + public static final ParseField CLASS_PROBABILITY = new ParseField("class_probability"); + public static final ParseField CLASS_SCORE = new ParseField("class_score"); + + public static final String NAME = "top_class"; + + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(NAME, true, a -> new TopClassEntry(a[0], (Double) a[1], (Double) a[2])); + + static { + PARSER.declareField(constructorArg(), (p, n) -> { + Object o; + XContentParser.Token token = p.currentToken(); + if (token == XContentParser.Token.VALUE_STRING) { + o = p.text(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + o = p.booleanValue(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + o = p.doubleValue(); + } else { + throw new XContentParseException(p.getTokenLocation(), + "[" + NAME + "] failed to parse field [" + CLASS_NAME + "] value [" + token + + "] is not a string, boolean or number"); + } + return o; + }, CLASS_NAME, ObjectParser.ValueType.VALUE); + PARSER.declareDouble(constructorArg(), CLASS_PROBABILITY); + PARSER.declareDouble(constructorArg(), CLASS_SCORE); + } + + public static TopClassEntry fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + private final Object classification; + private final double probability; + private final double score; + + public TopClassEntry(Object classification, double probability, double score) { + this.classification = Objects.requireNonNull(classification); + this.probability = probability; + this.score = score; + } + + public Object getClassification() { + return classification; + } + + public double getProbability() { + return probability; + } + + public double getScore() { + return score; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field(CLASS_NAME.getPreferredName(), classification); + builder.field(CLASS_PROBABILITY.getPreferredName(), probability); + builder.field(CLASS_SCORE.getPreferredName(), score); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object object) { + if (object == this) { return true; } + if (object == null || getClass() != object.getClass()) { return false; } + TopClassEntry that = (TopClassEntry) object; + return Objects.equals(classification, that.classification) && probability == that.probability && score == that.score; + } + + @Override + public int hashCode() { + return Objects.hash(classification, probability, score); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java index 9627a98ab6cf3..bd1b6a50cad6e 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java @@ -352,8 +352,10 @@ public static class IndexPrivilegeName { public static final String MANAGE_ILM = "manage_ilm"; public static final String CREATE_DOC = "create_doc"; public static final String MAINTENANCE = "maintenance"; + public static final String AUTO_CONFIGURE = "auto_configure"; public static final String[] ALL_ARRAY = new String[] { NONE, ALL, READ, READ_CROSS, CREATE, INDEX, DELETE, WRITE, MONITOR, MANAGE, - DELETE_INDEX, CREATE_INDEX, VIEW_INDEX_METADATA, MANAGE_FOLLOW_INDEX, MANAGE_ILM, CREATE_DOC, MAINTENANCE }; + DELETE_INDEX, CREATE_INDEX, VIEW_INDEX_METADATA, MANAGE_FOLLOW_INDEX, MANAGE_ILM, CREATE_DOC, MAINTENANCE, + AUTO_CONFIGURE}; } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 9abea8ef3a001..3e0a006598d75 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -1308,6 +1308,7 @@ public void testPutDataFrameAnalyticsConfig_GivenOutlierDetectionAnalysis() thro assertThat(createdConfig.getAnalyzedFields(), equalTo(config.getAnalyzedFields())); assertThat(createdConfig.getModelMemoryLimit(), equalTo(ByteSizeValue.parseBytesSizeValue("1gb", ""))); // default value assertThat(createdConfig.getDescription(), equalTo("some description")); + assertThat(createdConfig.getMaxNumThreads(), equalTo(1)); } public void testPutDataFrameAnalyticsConfig_GivenRegression() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 14ce4dd489abe..dd0b4bf6540f2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -688,6 +688,7 @@ public void testDefaultNamedXContents() { // Explicitly check for metrics from the analytics module because they aren't in InternalAggregationTestCase assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("string_stats"))); assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("top_metrics"))); + assertTrue(namedXContents.removeIf(e -> e.name.getPreferredName().equals("inference"))); assertEquals(expectedInternalAggregations + expectedSuggestions, namedXContents.size()); Map, Integer> categories = new HashMap<>(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/InferenceAggIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/InferenceAggIT.java new file mode 100644 index 0000000000000..fd530a23ec54e --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/analytics/InferenceAggIT.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.analytics; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.ESRestHighLevelClientTestCase; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.ml.PutTrainedModelRequest; +import org.elasticsearch.client.ml.inference.TrainedModelConfig; +import org.elasticsearch.client.ml.inference.TrainedModelDefinition; +import org.elasticsearch.client.ml.inference.TrainedModelInput; +import org.elasticsearch.client.ml.inference.trainedmodel.RegressionConfig; +import org.elasticsearch.client.ml.inference.trainedmodel.tree.Tree; +import org.elasticsearch.client.ml.inference.trainedmodel.tree.TreeNode; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class InferenceAggIT extends ESRestHighLevelClientTestCase { + + public void testInferenceAgg() throws IOException { + + // create a very simple decision tree with a root node and 2 leaves + List featureNames = Collections.singletonList("cost"); + Tree.Builder builder = Tree.builder(); + builder.setFeatureNames(featureNames); + TreeNode.Builder root = builder.addJunction(0, 0, true, 1.0); + int leftChild = root.getLeftChild(); + int rightChild = root.getRightChild(); + builder.addLeaf(leftChild, 10.0); + builder.addLeaf(rightChild, 20.0); + + final String modelId = "simple_regression"; + putTrainedModel(modelId, featureNames, builder.build()); + + final String index = "inference-test-data"; + indexData(index); + + TermsAggregationBuilder termsAgg = new TermsAggregationBuilder("fruit_type").field("fruit"); + AvgAggregationBuilder avgAgg = new AvgAggregationBuilder("avg_cost").field("cost"); + termsAgg.subAggregation(avgAgg); + + Map bucketPaths = new HashMap<>(); + bucketPaths.put("cost", "avg_cost"); + InferencePipelineAggregationBuilder inferenceAgg = new InferencePipelineAggregationBuilder("infer", modelId, bucketPaths); + termsAgg.subAggregation(inferenceAgg); + + SearchRequest search = new SearchRequest(index); + search.source().aggregation(termsAgg); + SearchResponse response = highLevelClient().search(search, RequestOptions.DEFAULT); + ParsedTerms terms = response.getAggregations().get("fruit_type"); + List buckets = terms.getBuckets(); + { + assertThat(buckets.get(0).getKey(), equalTo("apple")); + ParsedInference inference = buckets.get(0).getAggregations().get("infer"); + assertThat((Double) inference.getValue(), closeTo(20.0, 0.01)); + assertNull(inference.getWarning()); + assertNull(inference.getFeatureImportance()); + assertNull(inference.getTopClasses()); + } + { + assertThat(buckets.get(1).getKey(), equalTo("banana")); + ParsedInference inference = buckets.get(1).getAggregations().get("infer"); + assertThat((Double) inference.getValue(), closeTo(10.0, 0.01)); + assertNull(inference.getWarning()); + assertNull(inference.getFeatureImportance()); + assertNull(inference.getTopClasses()); + } + } + + private void putTrainedModel(String modelId, List inputFields, Tree tree) throws IOException { + TrainedModelDefinition definition = new TrainedModelDefinition.Builder().setTrainedModel(tree).build(); + TrainedModelConfig trainedModelConfig = TrainedModelConfig.builder() + .setDefinition(definition) + .setModelId(modelId) + .setInferenceConfig(new RegressionConfig()) + .setInput(new TrainedModelInput(inputFields)) + .setDescription("test model") + .build(); + highLevelClient().machineLearning().putTrainedModel(new PutTrainedModelRequest(trainedModelConfig), RequestOptions.DEFAULT); + } + + private void indexData(String index) throws IOException { + CreateIndexRequest create = new CreateIndexRequest(index); + create.mapping("{\"properties\": {\"fruit\": {\"type\": \"keyword\"}," + + "\"cost\": {\"type\": \"double\"}}}", XContentType.JSON); + highLevelClient().indices().create(create, RequestOptions.DEFAULT); + BulkRequest bulk = new BulkRequest(index).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + bulk.add(new IndexRequest().source(XContentType.JSON, "fruit", "apple", "cost", "1.2")); + bulk.add(new IndexRequest().source(XContentType.JSON, "fruit", "banana", "cost", "0.8")); + bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + highLevelClient().bulk(bulk, RequestOptions.DEFAULT); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index b1bcf4ccefe6d..7ccee107985bb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -3040,6 +3040,7 @@ public void testPutDataFrameAnalytics() throws Exception { .setAnalyzedFields(analyzedFields) // <5> .setModelMemoryLimit(new ByteSizeValue(5, ByteSizeUnit.MB)) // <6> .setDescription("this is an example description") // <7> + .setMaxNumThreads(1) // <8> .build(); // end::put-data-frame-analytics-config @@ -3096,6 +3097,7 @@ public void testUpdateDataFrameAnalytics() throws Exception { .setId("my-analytics-config") // <1> .setDescription("new description") // <2> .setModelMemoryLimit(new ByteSizeValue(128, ByteSizeUnit.MB)) // <3> + .setMaxNumThreads(4) // <4> .build(); // end::update-data-frame-analytics-config-update diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java index 48e14ee898b6c..23edf06418c27 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchRequestTests.java @@ -39,9 +39,6 @@ protected EqlSearchRequest createClientTestInstance() { if (randomBoolean()) { EqlSearchRequest.fetchSize(randomIntBetween(1, Integer.MAX_VALUE)); } - if (randomBoolean()) { - EqlSearchRequest.implicitJoinKeyField(randomAlphaOfLength(10)); - } if (randomBoolean()) { EqlSearchRequest.eventCategoryField(randomAlphaOfLength(10)); } @@ -54,9 +51,6 @@ protected EqlSearchRequest createClientTestInstance() { if (randomBoolean()) { EqlSearchRequest.tiebreakerField(randomAlphaOfLength(10)); } - if (randomBoolean()) { - EqlSearchRequest.searchAfter(randomArray(1, 4, Object[]::new, () -> randomAlphaOfLength(3))); - } if (randomBoolean()) { if (randomBoolean()) { EqlSearchRequest.filter(QueryBuilders.matchAllQuery()); @@ -76,12 +70,10 @@ protected org.elasticsearch.xpack.eql.action.EqlSearchRequest doParseToServerIns protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchRequest serverInstance, EqlSearchRequest clientTestInstance) { assertThat(serverInstance.eventCategoryField(), equalTo(clientTestInstance.eventCategoryField())); - assertThat(serverInstance.implicitJoinKeyField(), equalTo(clientTestInstance.implicitJoinKeyField())); assertThat(serverInstance.timestampField(), equalTo(clientTestInstance.timestampField())); assertThat(serverInstance.tiebreakerField(), equalTo(clientTestInstance.tiebreakerField())); assertThat(serverInstance.filter(), equalTo(clientTestInstance.filter())); assertThat(serverInstance.query(), equalTo(clientTestInstance.query())); - assertThat(serverInstance.searchAfter(), equalTo(clientTestInstance.searchAfter())); assertThat(serverInstance.indicesOptions(), equalTo(clientTestInstance.indicesOptions())); assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices())); assertThat(serverInstance.fetchSize(), equalTo(clientTestInstance.fetchSize())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java index 24688a6070915..623e7a98cc888 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -69,6 +69,9 @@ public static DataFrameAnalyticsConfig randomDataFrameAnalyticsConfig() { if (randomBoolean()) { builder.setAllowLazyStart(randomBoolean()); } + if (randomBoolean()) { + builder.setMaxNumThreads(randomIntBetween(1, 20)); + } return builder.build(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java index 086629a323117..0b1bf767a20aa 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java @@ -46,6 +46,9 @@ public static DataFrameAnalyticsConfigUpdate randomDataFrameAnalyticsConfigUpdat if (randomBoolean()) { builder.setAllowLazyStart(randomBoolean()); } + if (randomBoolean()) { + builder.setMaxNumThreads(randomIntBetween(1, 20)); + } return builder.build(); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelInputTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelInputTests.java index 30b6c46402df4..ca93a456c37e5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelInputTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelInputTests.java @@ -54,5 +54,4 @@ public static TrainedModelInput createRandomInput() { protected TrainedModelInput createTestInstance() { return createRandomInput(); } - } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/FeatureImportanceTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/FeatureImportanceTests.java new file mode 100644 index 0000000000000..9e6c8492e7453 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/FeatureImportanceTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml.inference.results; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class FeatureImportanceTests extends AbstractXContentTestCase { + + @Override + protected FeatureImportance createTestInstance() { + return new FeatureImportance( + randomAlphaOfLength(10), + randomDoubleBetween(-10.0, 10.0, false), + randomBoolean() ? null : + Stream.generate(() -> randomAlphaOfLength(10)) + .limit(randomLongBetween(2, 10)) + .collect(Collectors.toMap(Function.identity(), (k) -> randomDoubleBetween(-10, 10, false)))); + + } + + @Override + protected FeatureImportance doParseInstance(XContentParser parser) throws IOException { + return FeatureImportance.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.equals(FeatureImportance.CLASS_IMPORTANCE); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/TopClassEntryTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/TopClassEntryTests.java new file mode 100644 index 0000000000000..672d8a80df010 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/results/TopClassEntryTests.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml.inference.results; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class TopClassEntryTests extends AbstractXContentTestCase { + @Override + protected TopClassEntry createTestInstance() { + Object classification; + if (randomBoolean()) { + classification = randomAlphaOfLength(10); + } else if (randomBoolean()) { + classification = randomBoolean(); + } else { + classification = randomDouble(); + } + return new TopClassEntry(classification, randomDouble(), randomDouble()); + } + + @Override + protected TopClassEntry doParseInstance(XContentParser parser) throws IOException { + return TopClassEntry.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index e687a761a1325..53291ad067e3d 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -25,7 +25,7 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar - +import groovy.io.FileType import java.nio.file.Files import java.nio.file.Path @@ -243,6 +243,13 @@ subprojects { project.delete(archiveExtractionDir) archiveExtractionDir.mkdirs() } + // common sanity checks on extracted archive directly as part of checkExtraction + doLast { + // check no plain class files are packaged + archiveExtractionDir.eachFileRecurse (FileType.FILES) { file -> + assert file.name.endsWith(".class") == false + } + } } tasks.named('check').configure { dependsOn checkExtraction } if (project.name.contains('tar')) { diff --git a/distribution/build.gradle b/distribution/build.gradle index 74c2ba53c2dfb..a4a2e8993b8d8 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -259,6 +259,7 @@ copyModule(processSystemdOutputs, project(':modules:systemd')) configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { apply plugin: 'elasticsearch.jdk-download' + apply plugin: 'elasticsearch.repositories' // Setup all required JDKs project.jdks { @@ -278,6 +279,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { /***************************************************************************** * Properties to expand when copying packaging files * *****************************************************************************/ + configurations { + ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli'].each { + create(it) { + canBeConsumed = false + canBeResolved = true + attributes { + attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) + attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) + } + } + } + } + + dependencies { + libs project(':server') + libs project(':libs:elasticsearch-plugin-classloader') + libs project(':distribution:tools:java-version-checker') + libs project(':distribution:tools:launchers') + + libsPluginCli project(':distribution:tools:plugin-cli') + libsKeystoreCli project(path: ':distribution:tools:keystore-cli') + libsSecurityCli project(':x-pack:plugin:security:cli') + } + project.ext { /***************************************************************************** @@ -286,22 +312,16 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { libFiles = { oss -> copySpec { // delay by using closures, since they have not yet been configured, so no jar task exists yet - from { project(':server').jar } - from { project(':server').configurations.runtimeClasspath } - from { project(':libs:elasticsearch-plugin-classloader').jar } - from { project(':distribution:tools:java-version-checker').jar } - from { project(':distribution:tools:launchers').jar } + from(configurations.libs) into('tools/plugin-cli') { - from { project(':distribution:tools:plugin-cli').jar } - from { project(':distribution:tools:plugin-cli').configurations.runtimeClasspath } + from(configurations.libsPluginCli) } into('tools/keystore-cli') { - from { project(':distribution:tools:keystore-cli').jar } + from(configurations.libsKeystoreCli) } if (oss == false) { into('tools/security-cli') { - from { project(':x-pack:plugin:security:cli').jar } - from { project(':x-pack:plugin:security:cli').configurations.runtimeClasspath } + from(configurations.libsSecurityCli) } } } @@ -612,14 +632,14 @@ subprojects { } ['archives:windows-zip', 'archives:oss-windows-zip', - 'archives:darwin-tar', 'archives:oss-darwin-tar', - 'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar', - 'archives:linux-tar', 'archives:oss-linux-tar', - 'archives:integ-test-zip', - 'packages:rpm', 'packages:deb', - 'packages:aarch64-rpm', 'packages:aarch64-deb', - 'packages:oss-rpm', 'packages:oss-deb', - 'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb' + 'archives:darwin-tar', 'archives:oss-darwin-tar', + 'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar', + 'archives:linux-tar', 'archives:oss-linux-tar', + 'archives:integ-test-zip', + 'packages:rpm', 'packages:deb', + 'packages:aarch64-rpm', 'packages:aarch64-deb', + 'packages:oss-rpm', 'packages:oss-deb', + 'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb' ].forEach { subName -> Project subproject = project("${project.path}:${subName}") Configuration configuration = configurations.create(subproject.name) diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 08d7e54283a9b..25ca5283917c4 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -384,7 +384,7 @@ Closure commonRpmConfig(boolean oss, boolean jdk, String architecture) { prefix '/usr' packager 'Elasticsearch' - version = project.version.replace('-', '_') + archiveVersion = project.version.replace('-', '_') release = '1' os 'LINUX' distribution 'Elasticsearch' diff --git a/distribution/tools/launchers/build.gradle b/distribution/tools/launchers/build.gradle index 0263205521081..65d43c45b43e5 100644 --- a/distribution/tools/launchers/build.gradle +++ b/distribution/tools/launchers/build.gradle @@ -21,7 +21,7 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis apply plugin: 'elasticsearch.build' dependencies { - api parent.project('java-version-checker') + compileOnly project(':distribution:tools:java-version-checker') testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" testImplementation "junit:junit:${versions.junit}" testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/docs/build.gradle b/docs/build.gradle index 3f6527469a812..81b698a2a1b14 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -54,7 +54,6 @@ testClusters.integTest { setting 'indices.lifecycle.history_index_enabled', 'false' if (BuildParams.isSnapshotBuild() == false) { systemProperty 'es.autoscaling_feature_flag_registered', 'true' - systemProperty 'es.eql_feature_flag_registered', 'true' systemProperty 'es.searchable_snapshots_feature_enabled', 'true' } setting 'xpack.autoscaling.enabled', 'true' diff --git a/docs/java-rest/high-level/aggs-builders.asciidoc b/docs/java-rest/high-level/aggs-builders.asciidoc index 4ac24b7f00d97..718ac5056298d 100644 --- a/docs/java-rest/high-level/aggs-builders.asciidoc +++ b/docs/java-rest/high-level/aggs-builders.asciidoc @@ -12,21 +12,21 @@ This page lists all the available aggregations with their corresponding `Aggrega [options="header"] |====== | Aggregation | AggregationBuilder Class | Method in AggregationBuilders -| {ref}/search-aggregations-metrics-avg-aggregation.html[Avg] | {agg-ref}/metrics/avg/AvgAggregationBuilder.html[AvgAggregationBuilder] | {agg-ref}/AggregationBuilders.html#avg-java.lang.String-[AggregationBuilders.avg()] -| {ref}/search-aggregations-metrics-cardinality-aggregation.html[Cardinality] | {agg-ref}/metrics/cardinality/CardinalityAggregationBuilder.html[CardinalityAggregationBuilder] | {agg-ref}/AggregationBuilders.html#cardinality-java.lang.String-[AggregationBuilders.cardinality()] -| {ref}/search-aggregations-metrics-extendedstats-aggregation.html[Extended Stats] | {agg-ref}/metrics/stats/extended/ExtendedStatsAggregationBuilder.html[ExtendedStatsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#extendedStats-java.lang.String-[AggregationBuilders.extendedStats()] -| {ref}/search-aggregations-metrics-geobounds-aggregation.html[Geo Bounds] | {agg-ref}/metrics/geobounds/GeoBoundsAggregationBuilder.html[GeoBoundsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#geoBounds-java.lang.String-[AggregationBuilders.geoBounds()] -| {ref}/search-aggregations-metrics-geocentroid-aggregation.html[Geo Centroid] | {agg-ref}/metrics/geocentroid/GeoCentroidAggregationBuilder.html[GeoCentroidAggregationBuilder] | {agg-ref}/AggregationBuilders.html#geoCentroid-java.lang.String-[AggregationBuilders.geoCentroid()] -| {ref}/search-aggregations-metrics-max-aggregation.html[Max] | {agg-ref}/metrics/max/MaxAggregationBuilder.html[MaxAggregationBuilder] | {agg-ref}/AggregationBuilders.html#max-java.lang.String-[AggregationBuilders.max()] -| {ref}/search-aggregations-metrics-min-aggregation.html[Min] | {agg-ref}/metrics/min/MinAggregationBuilder.html[MinxAggregationBuilder] | {agg-ref}/AggregationBuilders.html#min-java.lang.String-[AggregationBuilders.min()] -| {ref}/search-aggregations-metrics-percentile-aggregation.html[Percentiles] | {agg-ref}/metrics/percentiles/PercentilesAggregationBuilder.html[PercentilesAggregationBuilder] | {agg-ref}/AggregationBuilders.html#percentiles-java.lang.String-[AggregationBuilders.percentiles()] -| {ref}/search-aggregations-metrics-percentile-rank-aggregation.html[Percentile Ranks] | {agg-ref}/metrics/percentiles/PercentileRanksAggregationBuilder.html[PercentileRanksAggregationBuilder] | {agg-ref}/AggregationBuilders.html#percentileRanks-java.lang.String-[AggregationBuilders.percentileRanks()] -| {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Scripted Metric] | {agg-ref}/metrics/scripted/ScriptedMetricAggregationBuilder.html[ScriptedMetricAggregationBuilder] | {agg-ref}/AggregationBuilders.html#scriptedMetric-java.lang.String-[AggregationBuilders.scriptedMetric()] -| {ref}/search-aggregations-metrics-stats-aggregation.html[Stats] | {agg-ref}/metrics/stats/StatsAggregationBuilder.html[StatsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#stats-java.lang.String-[AggregationBuilders.stats()] -| {ref}/search-aggregations-metrics-sum-aggregation.html[Sum] | {agg-ref}/metrics/sum/SumAggregationBuilder.html[SumAggregationBuilder] | {agg-ref}/AggregationBuilders.html#sum-java.lang.String-[AggregationBuilders.sum()] -| {ref}/search-aggregations-metrics-top-hits-aggregation.html[Top hits] | {agg-ref}/metrics/tophits/TopHitsAggregationBuilder.html[TopHitsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#topHits-java.lang.String-[AggregationBuilders.topHits()] +| {ref}/search-aggregations-metrics-avg-aggregation.html[Avg] | {agg-ref}/metrics/AvgAggregationBuilder.html[AvgAggregationBuilder] | {agg-ref}/AggregationBuilders.html#avg-java.lang.String-[AggregationBuilders.avg()] +| {ref}/search-aggregations-metrics-cardinality-aggregation.html[Cardinality] | {agg-ref}/metrics/CardinalityAggregationBuilder.html[CardinalityAggregationBuilder] | {agg-ref}/AggregationBuilders.html#cardinality-java.lang.String-[AggregationBuilders.cardinality()] +| {ref}/search-aggregations-metrics-extendedstats-aggregation.html[Extended Stats] | {agg-ref}/metrics/ExtendedStatsAggregationBuilder.html[ExtendedStatsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#extendedStats-java.lang.String-[AggregationBuilders.extendedStats()] +| {ref}/search-aggregations-metrics-geobounds-aggregation.html[Geo Bounds] | {agg-ref}/metrics/GeoBoundsAggregationBuilder.html[GeoBoundsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#geoBounds-java.lang.String-[AggregationBuilders.geoBounds()] +| {ref}/search-aggregations-metrics-geocentroid-aggregation.html[Geo Centroid] | {agg-ref}/metrics/GeoCentroidAggregationBuilder.html[GeoCentroidAggregationBuilder] | {agg-ref}/AggregationBuilders.html#geoCentroid-java.lang.String-[AggregationBuilders.geoCentroid()] +| {ref}/search-aggregations-metrics-max-aggregation.html[Max] | {agg-ref}/metrics/MaxAggregationBuilder.html[MaxAggregationBuilder] | {agg-ref}/AggregationBuilders.html#max-java.lang.String-[AggregationBuilders.max()] +| {ref}/search-aggregations-metrics-min-aggregation.html[Min] | {agg-ref}/metrics/MinAggregationBuilder.html[MinAggregationBuilder] | {agg-ref}/AggregationBuilders.html#min-java.lang.String-[AggregationBuilders.min()] +| {ref}/search-aggregations-metrics-percentile-aggregation.html[Percentiles] | {agg-ref}/metrics/PercentilesAggregationBuilder.html[PercentilesAggregationBuilder] | {agg-ref}/AggregationBuilders.html#percentiles-java.lang.String-[AggregationBuilders.percentiles()] +| {ref}/search-aggregations-metrics-percentile-rank-aggregation.html[Percentile Ranks] | {agg-ref}/metrics/PercentileRanksAggregationBuilder.html[PercentileRanksAggregationBuilder] | {agg-ref}/AggregationBuilders.html#percentileRanks-java.lang.String-[AggregationBuilders.percentileRanks()] +| {ref}/search-aggregations-metrics-scripted-metric-aggregation.html[Scripted Metric] | {agg-ref}/metrics/ScriptedMetricAggregationBuilder.html[ScriptedMetricAggregationBuilder] | {agg-ref}/AggregationBuilders.html#scriptedMetric-java.lang.String-[AggregationBuilders.scriptedMetric()] +| {ref}/search-aggregations-metrics-stats-aggregation.html[Stats] | {agg-ref}/metrics/StatsAggregationBuilder.html[StatsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#stats-java.lang.String-[AggregationBuilders.stats()] +| {ref}/search-aggregations-metrics-sum-aggregation.html[Sum] | {agg-ref}/metrics/SumAggregationBuilder.html[SumAggregationBuilder] | {agg-ref}/AggregationBuilders.html#sum-java.lang.String-[AggregationBuilders.sum()] +| {ref}/search-aggregations-metrics-top-hits-aggregation.html[Top hits] | {agg-ref}/metrics/TopHitsAggregationBuilder.html[TopHitsAggregationBuilder] | {agg-ref}/AggregationBuilders.html#topHits-java.lang.String-[AggregationBuilders.topHits()] | {ref}/search-aggregations-metrics-top-metrics.html[Top Metrics] | {javadoc-client}/analytics/TopMetricsAggregationBuilder.html[TopMetricsAggregationBuilder] | None -| {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count] | {agg-ref}/metrics/valuecount/ValueCountAggregationBuilder.html[ValueCountAggregationBuilder] | {agg-ref}/AggregationBuilders.html#count-java.lang.String-[AggregationBuilders.count()] +| {ref}/search-aggregations-metrics-valuecount-aggregation.html[Value Count] | {agg-ref}/metrics/ValueCountAggregationBuilder.html[ValueCountAggregationBuilder] | {agg-ref}/AggregationBuilders.html#count-java.lang.String-[AggregationBuilders.count()] | {ref}/search-aggregations-metrics-string-stats-aggregation.html[String Stats] | {javadoc-client}/analytics/StringStatsAggregationBuilder.html[StringStatsAggregationBuilder] | None |====== @@ -59,20 +59,21 @@ This page lists all the available aggregations with their corresponding `Aggrega ==== Pipeline Aggregations [options="header"] |====== -| Pipeline on | PipelineAggregationBuilder Class | Method in PipelineAggregatorBuilders -| {ref}/search-aggregations-pipeline-avg-bucket-aggregation.html[Avg Bucket] | {agg-ref}/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregationBuilder.html[AvgBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#avgBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.avgBucket()] -| {ref}/search-aggregations-pipeline-derivative-aggregation.html[Derivative] | {agg-ref}/pipeline/derivative/DerivativePipelineAggregationBuilder.html[DerivativePipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#derivative-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.derivative()] -| {ref}/search-aggregations-pipeline-max-bucket-aggregation.html[Max Bucket] | {agg-ref}/pipeline/bucketmetrics/max/MaxBucketPipelineAggregationBuilder.html[MaxBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#maxBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.maxBucket()] -| {ref}/search-aggregations-pipeline-min-bucket-aggregation.html[Min Bucket] | {agg-ref}/pipeline/bucketmetrics/min/MinBucketPipelineAggregationBuilder.html[MinBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#minBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.minBucket()] -| {ref}/search-aggregations-pipeline-sum-bucket-aggregation.html[Sum Bucket] | {agg-ref}/pipeline/bucketmetrics/sum/SumBucketPipelineAggregationBuilder.html[SumBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#sumBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.sumBucket()] -| {ref}/search-aggregations-pipeline-stats-bucket-aggregation.html[Stats Bucket] | {agg-ref}/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.html[StatsBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#statsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.statsBucket()] -| {ref}/search-aggregations-pipeline-extended-stats-bucket-aggregation.html[Extended Stats Bucket] | {agg-ref}/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.html[ExtendedStatsBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#extendedStatsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.extendedStatsBucket()] -| {ref}/search-aggregations-pipeline-percentiles-bucket-aggregation.html[Percentiles Bucket] | {agg-ref}/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.html[PercentilesBucketPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#percentilesBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.percentilesBucket()] -| {ref}/search-aggregations-pipeline-movavg-aggregation.html[Moving Average] | {agg-ref}/pipeline/movavg/MovAvgPipelineAggregationBuilder.html[MovAvgPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#movingAvg-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.movingAvg()] -| {ref}/search-aggregations-pipeline-cumulative-sum-aggregation.html[Cumulative Sum] | {agg-ref}/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.html[CumulativeSumPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#cumulativeSum-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.cumulativeSum()] -| {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Bucket Script] | {agg-ref}/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.html[BucketScriptPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#bucketScript-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketScript()] -| {ref}/search-aggregations-pipeline-bucket-selector-aggregation.html[Bucket Selector] | {agg-ref}/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.html[BucketSelectorPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#bucketSelector-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketSelector()] -| {ref}/search-aggregations-pipeline-serialdiff-aggregation.html[Serial Differencing] | {agg-ref}/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.html[SerialDiffPipelineAggregationBuilder] | {agg-ref}/pipeline/PipelineAggregatorBuilders.html#diff-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.diff()] +| Pipeline on | PipelineAggregationBuilder Class | Method in PipelineAggregatorBuilders +| {ref}/search-aggregations-pipeline-avg-bucket-aggregation.html[Avg Bucket] | {agg-ref}/pipeline/AvgBucketPipelineAggregationBuilder.html[AvgBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#avgBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.avgBucket()] +| {ref}/search-aggregations-pipeline-derivative-aggregation.html[Derivative] | {agg-ref}/pipeline/DerivativePipelineAggregationBuilder.html[DerivativePipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#derivative-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.derivative()] +| {ref}/search-aggregations-pipeline-inference-bucket-aggregation.html[Inference] | {javadoc-client}/analytics/InferencePipelineAggregationBuilder.html[InferencePipelineAggregationBuilder] | None +| {ref}/search-aggregations-pipeline-max-bucket-aggregation.html[Max Bucket] | {agg-ref}/pipeline/MaxBucketPipelineAggregationBuilder.html[MaxBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#maxBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.maxBucket()] +| {ref}/search-aggregations-pipeline-min-bucket-aggregation.html[Min Bucket] | {agg-ref}/pipeline/MinBucketPipelineAggregationBuilder.html[MinBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#minBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.minBucket()] +| {ref}/search-aggregations-pipeline-sum-bucket-aggregation.html[Sum Bucket] | {agg-ref}/pipeline/SumBucketPipelineAggregationBuilder.html[SumBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#sumBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.sumBucket()] +| {ref}/search-aggregations-pipeline-stats-bucket-aggregation.html[Stats Bucket] | {agg-ref}/pipeline/StatsBucketPipelineAggregationBuilder.html[StatsBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#statsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.statsBucket()] +| {ref}/search-aggregations-pipeline-extended-stats-bucket-aggregation.html[Extended Stats Bucket] | {agg-ref}/pipeline/ExtendedStatsBucketPipelineAggregationBuilder.html[ExtendedStatsBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#extendedStatsBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.extendedStatsBucket()] +| {ref}/search-aggregations-pipeline-percentiles-bucket-aggregation.html[Percentiles Bucket] | {agg-ref}/pipeline/PercentilesBucketPipelineAggregationBuilder.html[PercentilesBucketPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#percentilesBucket-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.percentilesBucket()] +| {ref}/search-aggregations-pipeline-movfn-aggregation.html[Moving Function] | {agg-ref}/pipeline/MovFnPipelineAggregationBuilder.html[MovFnPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#movingFunction-java.lang.String-org.elasticsearch.script.Script-java.lang.String-int-[PipelineAggregatorBuilders.movingFunction()] +| {ref}/search-aggregations-pipeline-cumulative-sum-aggregation.html[Cumulative Sum] | {agg-ref}/pipeline/CumulativeSumPipelineAggregationBuilder.html[CumulativeSumPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#cumulativeSum-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.cumulativeSum()] +| {ref}/search-aggregations-pipeline-bucket-script-aggregation.html[Bucket Script] | {agg-ref}/pipeline/BucketScriptPipelineAggregationBuilder.html[BucketScriptPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#bucketScript-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketScript()] +| {ref}/search-aggregations-pipeline-bucket-selector-aggregation.html[Bucket Selector] | {agg-ref}/pipeline/BucketSelectorPipelineAggregationBuilder.html[BucketSelectorPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#bucketSelector-java.lang.String-java.util.Map-org.elasticsearch.script.Script-[PipelineAggregatorBuilders.bucketSelector()] +| {ref}/search-aggregations-pipeline-serialdiff-aggregation.html[Serial Differencing] | {agg-ref}/pipeline/SerialDiffPipelineAggregationBuilder.html[SerialDiffPipelineAggregationBuilder] | {agg-ref}/PipelineAggregatorBuilders.html#diff-java.lang.String-java.lang.String-[PipelineAggregatorBuilders.diff()] |====== ==== Matrix Aggregations diff --git a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc index dab05b533cf50..8221dff43bbd5 100644 --- a/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc @@ -38,6 +38,7 @@ include-tagged::{doc-tests-file}[{api}-config] <5> The fields to be included in / excluded from the analysis <6> The memory limit for the model created as part of the analysis process <7> Optionally, a human-readable description +<8> The maximum number of threads to be used by the analysis. Defaults to 1. [id="{upid}-{api}-query-config"] diff --git a/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc b/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc index b6df7d25d0453..a110baa49e60e 100644 --- a/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc +++ b/docs/java-rest/high-level/ml/update-data-frame-analytics.asciidoc @@ -34,6 +34,7 @@ include-tagged::{doc-tests-file}[{api}-config-update] <1> The {dfanalytics-job} ID <2> The human-readable description <3> The memory limit for the model created as part of the analysis process +<4> The maximum number of threads to be used by the analysis [id="{upid}-{api}-query-config"] diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc index d0b95ae7f497b..b85cf8103858f 100644 --- a/docs/plugins/analysis-icu.asciidoc +++ b/docs/plugins/analysis-icu.asciidoc @@ -30,7 +30,7 @@ include::install_remove.asciidoc[] ==== ICU Analyzer The `icu_analyzer` analyzer performs basic normalization, tokenization and character folding, using the -`icu_normalizer` char filter, `icu_tokenizer` and `icu_normalizer` token filter +`icu_normalizer` char filter, `icu_tokenizer` and `icu_folding` token filter The following parameters are accepted: diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc index 393ac4d634372..0abcf7c19ebff 100644 --- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc @@ -535,10 +535,10 @@ first (ascending order, `asc`) or last (descending order, `desc`). ==== Size The `size` parameter can be set to define how many composite buckets should be returned. -Each composite bucket is considered as a single bucket so setting a size of 10 will return the +Each composite bucket is considered as a single bucket, so setting a size of 10 will return the first 10 composite buckets created from the values source. The response contains the values for each composite bucket in an array containing the values extracted -from each value source. +from each value source. Defaults to `10`. ==== Pagination diff --git a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc index 0c18dd19459ee..850ea8576613d 100644 --- a/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/histogram-aggregation.asciidoc @@ -286,3 +286,93 @@ POST /sales/_search?size=0 // TEST[setup:sales] <1> Documents without a value in the `quantity` field will fall into the same bucket as documents that have the value `0`. + +[[search-aggregations-bucket-histogram-aggregation-histogram-fields]] +==== Histogram fields + +Running a histogram aggregation over histogram fields computes the total number of counts for each interval. + +For example, executing a histogram aggregation against the following index that stores pre-aggregated histograms +with latency metrics (in milliseconds) for different networks: + +[source,console] +-------------------------------------------------- +PUT metrics_index/_doc/1 +{ + "network.name" : "net-1", + "latency_histo" : { + "values" : [1, 3, 8, 12, 15], + "counts" : [3, 7, 23, 12, 6] + } +} + +PUT metrics_index/_doc/2 +{ + "network.name" : "net-2", + "latency_histo" : { + "values" : [1, 6, 8, 12, 14], + "counts" : [8, 17, 8, 7, 6] + } +} + +POST /metrics_index/_search?size=0 +{ + "aggs" : { + "latency_buckets" : { + "histogram" : { + "field" : "latency_histo", + "interval" : 5 + } + } + } +} +-------------------------------------------------- + + +The `histogram` aggregation will sum the counts of each interval computed based on the `values` and +return the following output: + +[source,console-result] +-------------------------------------------------- +{ + ... + "aggregations": { + "prices" : { + "buckets": [ + { + "key": 0.0, + "doc_count": 18 + }, + { + "key": 5.0, + "doc_count": 48 + }, + { + "key": 10.0, + "doc_count": 25 + }, + { + "key": 15.0, + "doc_count": 6 + } + ] + } + } +} +-------------------------------------------------- +// TESTRESPONSE[skip:test not setup] + +[IMPORTANT] +======== +Histogram aggregation is a bucket aggregation, which partitions documents into buckets rather than calculating metrics over fields like +metrics aggregations do. Each bucket represents a collection of documents which sub-aggregations can run on. +On the other hand, a histogram field is a pre-aggregated field representing multiple values inside a single field: +buckets of numerical data and a count of items/documents for each bucket. This mismatch between the histogram aggregations expected input +(expecting raw documents) and the histogram field (that provides summary information) limits the outcome of the aggregation +to only the doc counts for each bucket. + + +**Consequently, when executing a histogram aggregation over a histogram field, no sub-aggregations are allowed.** +======== + +Also, when running histogram aggregation over histogram field the `missing` parameter is not supported. diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index 4973df5625b51..258d9e6a8c4d7 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -288,3 +288,4 @@ include::pipeline/normalize-aggregation.asciidoc[] include::pipeline/serial-diff-aggregation.asciidoc[] include::pipeline/stats-bucket-aggregation.asciidoc[] include::pipeline/extended-stats-bucket-aggregation.asciidoc[] +include::pipeline/inference-bucket-aggregation.asciidoc[] diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index 12b039e7b00d5..35bea691c7e51 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[data-streams-change-mappings-and-settings]] == Change mappings and settings for a data stream @@ -579,10 +580,7 @@ contains information about the stream's oldest backing index, `.ds-logs-000001`. { "name": "logs", "timestamp_field": { - "name": "@timestamp", - "mapping": { - "type": "date" - } + "name": "@timestamp" }, "indices": [ { diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index a19a4bc78773c..e5cb6c6953878 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[data-stream-apis]] == Data stream APIs diff --git a/docs/reference/data-streams/data-streams-overview.asciidoc b/docs/reference/data-streams/data-streams-overview.asciidoc index 4a66946b9ab4d..0ec7a984b7317 100644 --- a/docs/reference/data-streams/data-streams-overview.asciidoc +++ b/docs/reference/data-streams/data-streams-overview.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[data-streams-overview]] == Data streams overview ++++ @@ -16,9 +17,9 @@ the stream's backing indices. It contains: * A name or wildcard (`*`) pattern for the data stream. -* The data stream's _timestamp field_. This field must be mapped as a - <> or <> field data type and must be - included in every document indexed to the data stream. +* A mapping for the data stream's `@timestamp` field. This field must be mapped +as a <> or <> field data type and must be +included in every document indexed to the data stream. * The mappings and settings applied to each backing index when it's created. diff --git a/docs/reference/data-streams/data-streams.asciidoc b/docs/reference/data-streams/data-streams.asciidoc index 612c0425ba5cf..d0ed6fd6e924b 100644 --- a/docs/reference/data-streams/data-streams.asciidoc +++ b/docs/reference/data-streams/data-streams.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[data-streams]] = Data streams ++++ diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index 3b8fb9802e32b..d07b59d54ceb1 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[set-up-a-data-stream]] == Set up a data stream @@ -8,6 +9,7 @@ To set up a data stream, follow these steps: . <>. . <>. . <> to verify it exists. +. <>. After you set up a data stream, you can <> for indexing, searches, and other supported operations. @@ -20,11 +22,8 @@ and its backing indices. === Prerequisites * {es} data streams are intended for time-series data only. Each document -indexed to a data stream must contain a shared timestamp field. -+ -TIP: Data streams work well with most common log formats. While no schema is -required to use data streams, we recommend the {ecs-ref}[Elastic Common Schema -(ECS)]. +indexed to a data stream must contain the `@timestamp` field. This field must be +mapped as a <> or <> field data type. * Data streams are best suited for time-based, <> use cases. If you frequently need to @@ -132,17 +131,17 @@ this pattern. ---- ==== -* A `data_stream` definition containing the `timestamp_field` property. - This timestamp field must be included in every document indexed to the data - stream. +* A `data_stream` definition containing `@timestamp` in the `timestamp_field` +property. The `@timestamp` field must be included in every document indexed to +the data stream. * A <> or <> field mapping for the -timestamp field specified in the `timestamp_field` property. +`@timestamp` field. + -IMPORTANT: Carefully consider the timestamp field's mapping, including +IMPORTANT: Carefully consider the `@timestamp` field's mapping, including <> such as <>. -Once the stream is created, you can only update the timestamp field's mapping by -reindexing the data stream. See +Once the stream is created, you can only update the `@timestamp` field's mapping +by reindexing the data stream. See <>. * If you intend to use {ilm-init}, you must specify the @@ -283,7 +282,7 @@ PUT /_data_stream/logs_alt You can use the <> to get information about one or more data streams, including: -* The timestamp field and its mapping +* The timestamp field * The current backing indices, which is returned as an array. The last item in the array contains information about the stream's current write index. * The current generation @@ -325,10 +324,7 @@ contains information about the stream's write index, `.ds-logs-000002`. { "name": "logs", "timestamp_field": { - "name": "@timestamp", - "mapping": { - "type": "date" - } + "name": "@timestamp" }, "indices": [ { @@ -356,6 +352,13 @@ contains information about the stream's write index, `.ds-logs-000002`. contains information about the stream's current write index, `.ds-logs-000002`. ==== +[discrete] +[[secure-a-data-stream]] +=== Secure a data stream + +You can use {es} {security-features} to control access to a data stream and its +data. See <>. + [discrete] [[delete-a-data-stream]] === Delete a data stream diff --git a/docs/reference/data-streams/use-a-data-stream.asciidoc b/docs/reference/data-streams/use-a-data-stream.asciidoc index 79e716863934f..15f18ad07b164 100644 --- a/docs/reference/data-streams/use-a-data-stream.asciidoc +++ b/docs/reference/data-streams/use-a-data-stream.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[use-a-data-stream]] == Use a data stream @@ -122,6 +123,64 @@ PUT /logs/_bulk?refresh ==== -- +You can use an <> with these requests to pre-process +data before it's indexed. + +.*Example: Ingest pipeline* +[%collapsible] +==== +The following <> request creates the +`lowercase_message_field` ingest pipeline. The pipeline uses the +<> to change the `message` +field value to lowercase before indexing. + +[source,console] +---- +PUT /_ingest/pipeline/lowercase_message_field +{ + "description" : "Lowercases the message field value", + "processors" : [ + { + "lowercase" : { + "field" : "message" + } + } + ] +} +---- +// TEST[continued] + +The following index API request adds a new document to the `logs` data stream. + +The request includes a `?pipeline=lowercase_message_field` query parameter. +This parameter indicates {es} should use the `lowercase_message_field` pipeline +to pre-process the document before indexing it. + +During pre-processing, the pipeline changes the letter case of the document's +`message` field value from `LOGIN Successful` to `login successful`. + +[source,console] +---- +POST /logs/_doc?pipeline=lowercase_message_field +{ + "@timestamp": "2020-12-08T11:12:01.000Z", + "user": { + "id": "I1YBEOxJ" + }, + "message": "LOGIN Successful" +} +---- +// TEST[continued] + +//// +[source,console] +---- +DELETE /_ingest/pipeline/lowercase_message_field +---- +// TEST[continued] +//// +==== + [discrete] [[search-a-data-stream]] === Search a data stream diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 352019afe83cc..c50afa8531faf 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -163,6 +163,9 @@ Each bulk item can include the routing value using the `routing` field. It automatically follows the behavior of the index / delete operation based on the `_routing` mapping. +NOTE: Data streams do not support custom routing. Instead, target the +appropriate backing index for the stream. + [float] [[bulk-wait-for-active-shards]] ===== Wait For Active Shards diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index cb263fabd3248..7bcd77a27c4d1 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -302,6 +302,9 @@ additional document parsing pass. If the `_routing` mapping is defined and set to be `required`, the index operation will fail if no routing value is provided or extracted. +NOTE: Data streams do not support custom routing. Instead, target the +appropriate backing index for the stream. + [float] [[index-distributed]] ===== Distributed diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc index 9b585c28c5515..32bc8207a8ed0 100644 --- a/docs/reference/eql/delete-async-eql-search-api.asciidoc +++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc @@ -7,7 +7,7 @@ Delete async EQL search ++++ -dev::[] +experimental::[] Deletes an <> or a <>. The API also diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index b9da2d9af0d09..473fbbca2600e 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -7,11 +7,12 @@ EQL search ++++ -dev::[] +experimental::[] Returns search results for an <> query. -In {es}, EQL assumes each document in an index corresponds to an event. +In {es}, EQL assumes each document in a data stream or index corresponds to an +event. //// [source,console] @@ -44,9 +45,9 @@ GET /my_index/_eql/search [[eql-search-api-request]] ==== {api-request-title} -`GET //_eql/search` +`GET //_eql/search` -`POST //_eql/search` +`POST //_eql/search` [[eql-search-api-prereqs]] ==== {api-prereq-title} @@ -61,12 +62,13 @@ See <>. [[eql-search-api-path-params]] ==== {api-path-parms-title} -``:: +``:: (Required, string) -Comma-separated list of index names or <> used to -limit the request. Accepts wildcard expressions. +Comma-separated list of data streams, indices, or <> used to limit the request. Accepts wildcard (`*`) expressions. + -To search all indices, use `_all` or `*`. +To search all data streams and indices in a cluster, use +`_all` or `*`. [[eql-search-api-query-params]] ==== {api-query-parms-title} @@ -157,18 +159,30 @@ Field containing the event classification, such as `process`, `file`, or `network`. + Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic -Common Schema (ECS)]. If an index does not contain the `event.category` field, -this value is required. +Common Schema (ECS)]. If a data stream or index does not contain the +`event.category` field, this value is required. + +`fetch_size`:: +(Optional, integer) +Maximum number of events to search at a time for sequence queries. Defaults to +`1000`. ++ +This value must be greater than `2` but cannot exceed the value of the +<> setting, which defaults to +`10000`. ++ +Internally, a sequence query fetches and paginates sets of events to search for +matches. This parameter controls the size of those sets. This parameter does not +limit the total number of events searched or the number of matching events +returned. ++ +A greater `fetch_size` value often increases search speed but uses more memory. `filter`:: (Optional, <>) Query, written in query DSL, used to filter the events on which the EQL query runs. -`implicit_join_key_field`:: -(Optional, string) -Reserved for future use. - `keep_alive`:: + -- @@ -219,10 +233,6 @@ If both parameters are specified, only the query parameter is used. IMPORTANT: This parameter supports a subset of EQL syntax. See <>. -`search_after`:: -(Optional, string) -Reserved for future use. - `size`:: (Optional, integer or float) For <>, the maximum number of matching events to @@ -231,7 +241,10 @@ return. For <>, the maximum number of matching sequences to return. + -Defaults to `50`. This value must be greater than `0`. +Defaults to `10`. This value must be greater than `0`. ++ +NOTE: You cannot use <>, such as `head` or `tail`, to exceed +this value. [[eql-search-api-tiebreaker-field]] `tiebreaker_field`:: @@ -254,8 +267,8 @@ field is used to sort the events in ascending, lexicographic order. Field containing event timestamp. Defaults to `@timestamp`, as defined in the -{ecs-ref}/ecs-event.html[Elastic Common Schema (ECS)]. If an index does not -contain the `@timestamp` field, this value is required. +{ecs-ref}/ecs-event.html[Elastic Common Schema (ECS)]. If a data stream or index +does not contain the `@timestamp` field, this value is required. Events in the API response are sorted by this field's value, converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in diff --git a/docs/reference/eql/functions.asciidoc b/docs/reference/eql/functions.asciidoc index 32845a3876abb..969ace1cd37d1 100644 --- a/docs/reference/eql/functions.asciidoc +++ b/docs/reference/eql/functions.asciidoc @@ -6,7 +6,7 @@ Function reference ++++ -dev::[] +experimental::[] {es} supports the following EQL functions: diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc index 721a8788bb7f4..db2f0bf5ee126 100644 --- a/docs/reference/eql/get-async-eql-search-api.asciidoc +++ b/docs/reference/eql/get-async-eql-search-api.asciidoc @@ -7,7 +7,7 @@ Get async EQL search ++++ -dev::[] +experimental::[] Returns the current status and available results for an <> or a <EQL ++++ -dev::[] +experimental::[] {eql-ref}/index.html[Event Query Language (EQL)] is a query language used for logs and other event-based data. diff --git a/docs/reference/eql/limitations.asciidoc b/docs/reference/eql/limitations.asciidoc index dfd0d1ee65b09..872d9cce05bed 100644 --- a/docs/reference/eql/limitations.asciidoc +++ b/docs/reference/eql/limitations.asciidoc @@ -6,15 +6,15 @@ Limitations ++++ -dev::[] +experimental::[] [discrete] [[eql-nested-fields]] === EQL search on nested fields is not supported You cannot use EQL to search the values of a <> field or the -sub-fields of a `nested` field. However, indices containing `nested` field -mappings are otherwise supported. +sub-fields of a `nested` field. However, data streams and indices containing +`nested` field mappings are otherwise supported. [discrete] [[eql-unsupported-syntax]] @@ -41,5 +41,3 @@ queries that contain: ** {eql-ref}/pipes.html#sort[`sort`] ** {eql-ref}/pipes.html#unique[`unique`] ** {eql-ref}/pipes.html#unique-count[`unique_count`] - -* The `until` {eql-ref}/sequences.html[sequence keyword] \ No newline at end of file diff --git a/docs/reference/eql/pipes.asciidoc b/docs/reference/eql/pipes.asciidoc index a61ffd3a20a77..9593f0930cf14 100644 --- a/docs/reference/eql/pipes.asciidoc +++ b/docs/reference/eql/pipes.asciidoc @@ -6,7 +6,7 @@ Pipe reference ++++ -dev::[] +experimental::[] {es} supports the following EQL pipes: @@ -17,21 +17,21 @@ dev::[] [[eql-pipe-head]] === `head` -Returns up to a specified number of events, starting with the earliest matching -events. Works similarly to the +Returns up to a specified number of events or sequences, starting with the +earliest matches. Works similarly to the https://en.wikipedia.org/wiki/Head_(Unix)[Unix head command]. [%collapsible] ==== *Example* -The following EQL query returns up to fifty of the earliest powershell +The following EQL query returns up to three of the earliest powershell commands. [source,eql] ---- process where process.name == "powershell.exe" -| head 50 +| head 3 ---- *Syntax* @@ -44,28 +44,28 @@ head ``:: (Required, integer) -Maximum number of matching events to return. +Maximum number of matching events or sequences to return. ==== [discrete] [[eql-pipe-tail]] === `tail` -Returns up to a specified number of events, starting with the most recent -matching events. Works similarly to the +Returns up to a specified number of events or sequences, starting with the most +recent matches. Works similarly to the https://en.wikipedia.org/wiki/Tail_(Unix)[Unix tail command]. [%collapsible] ==== *Example* -The following EQL query returns up to thirty of the most recent `svchost.exe` +The following EQL query returns up to five of the most recent `svchost.exe` processes. [source,eql] ---- process where process.name == "svchost.exe" -| tail 30 +| tail 5 ---- *Syntax* @@ -78,5 +78,5 @@ tail ``:: (Required, integer) -Maximum number of matching events to return. -==== \ No newline at end of file +Maximum number of matching events or sequences to return. +==== diff --git a/docs/reference/eql/requirements.asciidoc b/docs/reference/eql/requirements.asciidoc index 3f3e581315806..81ec9cd9fa594 100644 --- a/docs/reference/eql/requirements.asciidoc +++ b/docs/reference/eql/requirements.asciidoc @@ -6,7 +6,7 @@ Requirements ++++ -dev::[] +experimental::[] EQL is schema-less and works well with most common log formats. @@ -21,10 +21,11 @@ with core ECS fields by default. [[eql-required-fields]] === Required fields -In {es}, EQL assumes each document in an index corresponds to an event. +In {es}, EQL assumes each document in a data stream or index corresponds to an +event. -To search an index using EQL, each document in the index must contain the -following field archetypes: +To search a data stream or index using EQL, each document in the data stream or +index must contain the following field archetypes: Event category:: A field containing the event classification, such as `process`, `file`, or diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc index 8fdf1a8ff5a30..21773c4d76262 100644 --- a/docs/reference/eql/search.asciidoc +++ b/docs/reference/eql/search.asciidoc @@ -3,16 +3,17 @@ [[eql-search]] == Run an EQL search -dev::[] +experimental::[] To start using EQL in {es}, first ensure your event data meets <>. You can then use the <> to search event data stored in one or more {es} indices. +search API>> to search event data stored in one or more {es} data streams or +indices. .*Example* [%collapsible] ==== -To get started, ingest or add the data to an {es} index. +To get started, ingest or add the data to an {es} data stream or index. The following <> request adds some example log data to the `sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema @@ -31,6 +32,8 @@ PUT /sec_logs/_bulk?refresh { "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } {"index":{"_index" : "sec_logs", "_id" : "5"}} { "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{"_index" : "sec_logs", "_id" : "6"}} +{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } ---- // TESTSETUP @@ -99,7 +102,7 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } - }, + }, "sort": [ 1607252645000 ] @@ -389,6 +392,27 @@ contains the shared `agent.id` value for each matching event. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] + +You can use the <> to specify an expiration +event for sequences. Matching sequences must end before this event. + +The following request adds +`until [ process where event.type == "termination" ]` to the previous EQL query. +This ensures matching sequences end before a process termination event. + +[source,console] +---- +GET /sec_logs/_eql/search +{ + "query": """ + sequence by agent.id with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + until [ process where event.type == "termination" ] + """ +} +---- +// TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/] ==== [discrete] @@ -547,7 +571,7 @@ tiebreaker for events with the same timestamp. } ---- // TESTRESPONSE[s/"took": 34/"took": $body.took/] -<1> The event's <>, converted to +<1> The event's <>, converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch] <2> The event's `event.id` value. diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index b826318062788..1b20f5a3c97a4 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -6,7 +6,7 @@ Syntax reference ++++ -dev::[] +experimental::[] [IMPORTANT] ==== @@ -485,7 +485,7 @@ sequence by user.name ---- ==== -You can combine the `sequence by` and `with maxspan` keywords to constrain a +You can combine the `sequence by` and `with maxspan` keywords to constrain a sequence by both field values and a timespan. [source,eql] @@ -513,6 +513,89 @@ sequence by user.name with maxspan=15m ---- ==== +[discrete] +[[eql-until-keyword]] +==== `until` keyword + +You can use the `until` keyword to specify an expiration event for sequences. +Matching sequences must end before this event, which is not included the +results. If this event occurs within a sequence, the sequence is not considered +a match. + +[source,eql] +---- +sequence + [ event_category_1 where condition_1 ] + [ event_category_2 where condition_2 ] + ... +until [ event_category_2 where condition_2 ] +---- + +.*Example* +[%collapsible] +==== +The following EQL sequence query uses the `until` keyword to end sequences +before a process termination event. Process termination events have an event +category of `process` and `event.type` value of `termination`. + +[source,eql] +---- +sequence + [ file where file.extension == "exe" ] + [ process where true ] +until [ process where event.type == "termination" ] +---- +==== + +[TIP] +==== +The `until` keyword can be helpful when searching for process sequences in +Windows event logs, such as those ingested using +{winlogbeat-ref}/index.html[Winlogbeat]. + +In Windows, a process ID (PID) is unique only while a process is running. After +a process terminates, its PID can be reused. + +You can search for a sequence of events with the same PID value using the `by` +and `sequence by` keywords. + +.*Example* +[%collapsible] +===== +The following EQL query uses the `sequence by` keyword to match a sequence of +events that share the same `process.pid` value. + +[source,eql] +---- +sequence by process.pid + [ process where process.name == "cmd.exe" ] + [ process where process.name == "whoami.exe" ] +---- +===== + +However, due to PID reuse, this can result in a matching sequence that +contains events across unrelated processes. To prevent false positives, you can +use the `until` keyword to end matching sequences before a process termination +event. + +.*Example* +[%collapsible] +===== +The following EQL query uses the `until` keyword to end sequences before +`process` events with an `event.type` of `termination`. These events indicate a +process has been terminated. + +[source,eql] +---- +sequence by process.pid + [ process where process.name == "cmd.exe" ] + [ process where process.name == "whoami.exe" ] +until [ process where event.type == "termination" ] +---- +===== + +==== + [discrete] [[eql-functions]] === Functions diff --git a/docs/reference/glossary.asciidoc b/docs/reference/glossary.asciidoc index 8c172c99202ea..fc78872840046 100644 --- a/docs/reference/glossary.asciidoc +++ b/docs/reference/glossary.asciidoc @@ -480,8 +480,9 @@ See the {ref}/indices-shrink-index.html[shrink index API]. [[glossary-snapshot]] snapshot :: // tag::snapshot-def[] -A backup taken from a running {es} cluster. -You can take snapshots of individual indices or of the entire cluster. +A backup taken from a running {es} cluster. +A snapshot can include backups of an entire cluster or only data streams and +indices you specify. // end::snapshot-def[] [[glossary-snapshot-lifecycle-policy]] snapshot lifecycle policy :: diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc index 97cd88d06e080..8fac808eb76f7 100644 --- a/docs/reference/how-to/indexing-speed.asciidoc +++ b/docs/reference/how-to/indexing-speed.asciidoc @@ -62,7 +62,7 @@ gets indexed and when it becomes visible, increasing the If you have a large amount of data that you want to load all at once into Elasticsearch, it may be beneficial to set `index.number_of_replicas` to `0` in -order to speep up indexing. Having no replicas means that losing a single node +order to speed up indexing. Having no replicas means that losing a single node may incur data loss, so it is important that the data lives elsewhere so that this initial load can be retried in case of an issue. Once the initial load is finished, you can set `index.number_of_replicas` back to its original value. diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index 1c9e0b112425b..49b9e5c238da9 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -6,12 +6,14 @@ Explain lifecycle ++++ -Shows an index's current lifecycle status. +Retrieves the current lifecycle status for one or more indices. For data +streams, the API retrieves the current lifecycle status for the stream's backing +indices. [[ilm-explain-lifecycle-request]] ==== {api-request-title} -`GET /_ilm/explain` +`GET /_ilm/explain` [[ilm-explain-lifecycle-prereqs]] ==== {api-prereq-title} @@ -31,8 +33,12 @@ about any failures. [[ilm-explain-lifecycle-path-params]] ==== {api-path-parms-title} -``:: - (Required, string) Identifier for the index. +``:: +(Required, string) +Comma-separated list of data streams, indices, and index aliases to target. +Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, use `_all` or `*`. [[ilm-explain-lifecycle-query-params]] ==== {api-query-parms-title} diff --git a/docs/reference/ilm/ilm-and-snapshots.asciidoc b/docs/reference/ilm/ilm-and-snapshots.asciidoc index ce19388c519d0..8682391df99f5 100644 --- a/docs/reference/ilm/ilm-and-snapshots.asciidoc +++ b/docs/reference/ilm/ilm-and-snapshots.asciidoc @@ -1,9 +1,9 @@ [role="xpack"] [testenv="basic"] [[index-lifecycle-and-snapshots]] -== Restore a managed index +== Restore a managed data stream or index -When you restore a snapshot that contains managed indices, +When you restore a managed index or a data stream with managed backing indices, {ilm-init} automatically resumes executing the restored indices' policies. A restored index's `min_age` is relative to when it was originally created or rolled over, not its restoration time. @@ -12,8 +12,8 @@ an index has been restored from a snapshot. If you restore an index that was accidentally deleted half way through its month long lifecycle, it proceeds normally through the last two weeks of its lifecycle. -In some cases, you might want to restore a managed index and -prevent {ilm-init} from immediately executing its policy. +In some cases, you might want to prevent {ilm-init} from immediately executing +its policy on a restored index. For example, if you are restoring an older snapshot you might want to prevent it from rapidly progressing through all of its lifecycle phases. You might want to add or update documents before it's marked read-only or shrunk, diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index c71c5ecc4b365..3ee7f13435ceb 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -25,9 +25,7 @@ include::search/index.asciidoc[] include::query-dsl.asciidoc[] -ifdef::permanently-unreleased-branch[] include::eql/index.asciidoc[] -endif::[] include::sql/index.asciidoc[] diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc index 231b5712b0e79..75dbd41a45de1 100644 --- a/docs/reference/indices/add-alias.asciidoc +++ b/docs/reference/indices/add-alias.asciidoc @@ -37,6 +37,8 @@ to add to the alias. + To add all indices in the cluster to the alias, use a value of `_all`. ++ +NOTE: You cannot add <> to an index alias. ``:: (Required, string) diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 5d46b249736b5..41d6cf2d01913 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -75,6 +75,8 @@ used to perform the action. + If the `indices` parameter is not specified, this parameter is required. ++ +NOTE: You cannot add <> to an index alias. `indices`:: (Array) @@ -83,6 +85,8 @@ used to perform the action. + If the `index` parameter is not specified, this parameter is required. ++ +NOTE: You cannot add <> to an index alias. `alias`:: (String) diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc index b9b5bc79732b3..38195f72bc50d 100644 --- a/docs/reference/indices/apis/reload-analyzers.asciidoc +++ b/docs/reference/indices/apis/reload-analyzers.asciidoc @@ -7,6 +7,8 @@ ++++ Reloads an index's <> and their resources. +For data streams, the API reloads search analyzers and resources for the +stream's backing indices. [source,console] -------------------------------------------------- @@ -18,9 +20,9 @@ POST /twitter/_reload_search_analyzers [[indices-reload-analyzers-api-request]] === {api-request-title} -`POST //_reload_search_analyzers` +`POST //_reload_search_analyzers` -`GET //_reload_search_analyzers` +`GET //_reload_search_analyzers` [discrete] @@ -63,10 +65,12 @@ in the future. [[indices-reload-analyzers-api-path-params]] === {api-path-parms-title} -``:: +``:: (Required, string) -Comma-separated list or wildcard expression of index names -used to limit the request. +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, use `_all` or `*`. [discrete] diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc index ba83b8baf341b..af3696e38a0f2 100644 --- a/docs/reference/indices/clearcache.asciidoc +++ b/docs/reference/indices/clearcache.asciidoc @@ -4,7 +4,8 @@ Clear cache ++++ -Clears caches for one or more indices. +Clears the caches of one or more indices. For data streams, the API clears the +caches of the stream's backing indices. [source,console] ---- @@ -16,7 +17,7 @@ POST /twitter/_cache/clear [[clear-cache-api-request]] ==== {api-request-title} -`POST //_cache/clear` +`POST //_cache/clear` `POST /_cache/clear` @@ -24,7 +25,13 @@ POST /twitter/_cache/clear [[clear-cache-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[clear-cache-api-query-params]] @@ -127,7 +134,7 @@ POST /twitter/_cache/clear?fields=foo,bar <1> [[clear-cache-api-multi-ex]] -===== Clear caches for several indices +===== Clear caches for several data streams and indices [source,console] ---- @@ -137,7 +144,7 @@ POST /kimchy,elasticsearch/_cache/clear [[clear-cache-api-all-ex]] -===== Clear caches for all indices +===== Clear caches for all data streams and indices [source,console] ---- diff --git a/docs/reference/indices/create-data-stream.asciidoc b/docs/reference/indices/create-data-stream.asciidoc index 4651c2bc2aef0..d195a514b7b85 100644 --- a/docs/reference/indices/create-data-stream.asciidoc +++ b/docs/reference/indices/create-data-stream.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[indices-create-data-stream]] === Create data stream API ++++ diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index ce9cdb7a57746..62a432cfabac3 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[indices-delete-data-stream]] === Delete data stream API ++++ diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index 585ca772c103a..0f0e9ed2faa0c 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -1,3 +1,4 @@ +[role="xpack"] [[indices-get-data-stream]] === Get data stream API ++++ @@ -84,9 +85,10 @@ GET /_data_stream/my-data-stream ==== {api-path-parms-title} ``:: -(Required, string) -Name of the data stream to retrieve. -Wildcard (`*`) expressions are supported. +(Optional, string) +Comma-separated list of data stream names used to limit the request. Wildcard +(`*`) expressions are supported. If omitted, all data streams will be +returned. [role="child_attributes"] [[get-data-stream-api-response-body]] @@ -105,19 +107,16 @@ Name of the data stream. `timestamp_field`:: (object) -Contains information about the data stream's timestamp field. +Contains information about the data stream's `@timestamp` field. + .Properties of `timestamp_field` [%collapsible%open] ===== `name`:: (string) -Name of the data stream's timestamp field. This field must be included in every -document indexed to the data stream. - -`mapping`:: -(<>) -Field mapping for the data stream's timestamp field. +Name of the data stream's timestamp field, which must be `@timestamp`. The +`@timestamp` field must be included in every document indexed to the data +stream. ===== `indices`:: @@ -205,10 +204,7 @@ The API returns the following response: { "name": "my-data-stream", "timestamp_field": { - "name": "@timestamp", - "mapping": { - "type": "date" - } + "name": "@timestamp" }, "indices": [ { @@ -228,10 +224,7 @@ The API returns the following response: { "name": "my-data-stream_two", "timestamp_field": { - "name": "@timestamp", - "mapping": { - "type": "date" - } + "name": "@timestamp" }, "indices": [ { diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc index 129fa89c842b6..40e99b5c29993 100644 --- a/docs/reference/indices/get-index.asciidoc +++ b/docs/reference/indices/get-index.asciidoc @@ -4,7 +4,8 @@ Get index ++++ -Returns information about one or more indexes. +Returns information about one or more indices. For data streams, the API +returns information about the stream's backing indices. [source,console] -------------------------------------------------- @@ -15,20 +16,19 @@ GET /twitter [[get-index-api-request]] ==== {api-request-title} -`GET /` +`GET /` [[get-index-api-path-params]] ==== {api-path-parms-title} -``:: +``:: +(Required, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. + --- -(Required, string) Comma-separated list or wildcard expression of index names -used to limit the request. - -Use a value of `_all` to retrieve information for all indices in the cluster. --- +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[get-index-api-query-params]] diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc index b602b9fb7691f..a86b1182b9fad 100644 --- a/docs/reference/indices/get-mapping.asciidoc +++ b/docs/reference/indices/get-mapping.asciidoc @@ -4,7 +4,8 @@ Get mapping ++++ -Retrieves <> for indices in a cluster. +Retrieves <> for one or more indices. For data +streams, the API retrieves mappings for the stream's backing indices. [source,console] -------------------------------------------------- @@ -17,13 +18,19 @@ GET /twitter/_mapping `GET /_mapping` -`GET //_mapping` +`GET //_mapping` [[get-mapping-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[get-mapping-api-query-params]] @@ -48,12 +55,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] ==== {api-examples-title} [[get-mapping-api-multi-ex]] -===== Multiple indices +===== Multiple data streams and indices -The get mapping API can be used to get more than one index with a +The get mapping API can be used to get more than one data stream or index with a single call. General usage of the API follows the following syntax: -`host:port/{index}/_mapping` where `{index}` can accept a comma-separated -list of names. To get mappings for all indices you can use `_all` for `{index}`. +`host:port//_mapping` where `` can accept a comma-separated +list of names. To get mappings for all data streams and indices in a cluster, use `_all` or `*` for `` +or omit the `` parameter. The following are some examples: [source,console] @@ -63,11 +71,13 @@ GET /twitter,kimchy/_mapping // TEST[setup:twitter] // TEST[s/^/PUT kimchy\nPUT book\n/] -If you want to get mappings of all indices and types then the following -two examples are equivalent: +If you want to get mappings of all indices in a cluster, the following +examples are equivalent: [source,console] -------------------------------------------------- +GET /*/_mapping + GET /_all/_mapping GET /_mapping diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc index 19defec6743b6..58dab6e9f01d3 100644 --- a/docs/reference/indices/get-settings.asciidoc +++ b/docs/reference/indices/get-settings.asciidoc @@ -4,7 +4,8 @@ Get index settings ++++ -Returns setting information for an index. +Returns setting information for one or more indices. For data streams, the API +returns setting information for the stream's backing indices. [source,console] -------------------------------------------------- @@ -16,17 +17,21 @@ GET /twitter/_settings [[get-index-settings-api-request]] ==== {api-request-title} -`GET //_settings` +`GET //_settings` -`GET //_settings/` +`GET //_settings/` [[get-index-settings-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. + -Use a value of `_all` to retrieve information for all indices in the cluster. +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. ``:: (Optional, string) Comma-separated list or wildcard expression of setting names @@ -58,10 +63,10 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-index-settings-api-example]] ==== {api-examples-title} -===== Multiple indices +===== Multiple data streams and indices -The get settings API can be used to get settings for more than one index with a -single call. To get settings for all indices you can use `_all` for ``. +The get settings API can be used to get settings for more than one data stream or index with a +single call. To get settings for all indices in a cluster, you can use `_all` or `*` for ``. Wildcard expressions are also supported. The following are some examples: [source,console] diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc index a16cf6e641919..8c97259e64109 100644 --- a/docs/reference/indices/recovery.asciidoc +++ b/docs/reference/indices/recovery.asciidoc @@ -5,7 +5,9 @@ ++++ -Returns information about ongoing and completed shard recoveries. +Returns information about ongoing and completed shard recoveries for one or more +indices. For data streams, the API returns information for the stream's backing +indices. [source,console] ---- @@ -17,7 +19,7 @@ GET /twitter/_recovery [[index-recovery-api-request]] ==== {api-request-title} -`GET //_recovery` +`GET //_recovery` `GET /_recovery` @@ -41,9 +43,13 @@ include::{es-repo-dir}/glossary.asciidoc[tag=recovery-triggers] [[index-recovery-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. + -Use a value of `_all` to retrieve information for all indices in the cluster. +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[index-recovery-api-query-params]] @@ -165,7 +171,7 @@ Statistics about time to open and start the index. [[index-recovery-api-multi-ex]] -===== Get recovery information for several indices +===== Get recovery information for several data streams and indices [source,console] -------------------------------------------------- @@ -175,7 +181,7 @@ GET index1,index2/_recovery?human [[index-recovery-api-all-ex]] -===== Get segment information for all indices +===== Get segment information for all data streams and indices in a cluster ////////////////////////// Here we create a repository and snapshot index1 in diff --git a/docs/reference/indices/refresh.asciidoc b/docs/reference/indices/refresh.asciidoc index 8d0c5655f4441..042f49d2ac0ca 100644 --- a/docs/reference/indices/refresh.asciidoc +++ b/docs/reference/indices/refresh.asciidoc @@ -4,7 +4,8 @@ Refresh ++++ -Refreshes one or more indices. +Refreshes one or more indices. For data streams, the API refreshes the stream's +backing indices. [source,console] ---- @@ -16,9 +17,9 @@ POST /twitter/_refresh [[refresh-api-request]] ==== {api-request-title} -`POST /_refresh` +`POST /_refresh` -`GET /_refresh` +`GET /_refresh` `POST /_refresh` @@ -29,6 +30,7 @@ POST /twitter/_refresh ==== {api-description-title} Use the refresh API to explicitly refresh one or more indices. +If the request targets a data stream, it refreshes the stream's backing indices. A _refresh_ makes all operations performed on an index since the last refresh available for search. @@ -87,7 +89,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab [[refresh-api-multiple-ex]] -===== Refresh several indices +===== Refresh several data streams and indices [source,console] ---- @@ -97,7 +99,7 @@ POST /kimchy,elasticsearch/_refresh [[refresh-api-all-ex]] -===== Refresh all indices +===== Refresh all data streams and indices in a cluster [source,console] ---- diff --git a/docs/reference/indices/segments.asciidoc b/docs/reference/indices/segments.asciidoc index f8d084ae34f42..d19d683ade050 100644 --- a/docs/reference/indices/segments.asciidoc +++ b/docs/reference/indices/segments.asciidoc @@ -5,7 +5,8 @@ ++++ Returns low-level information about the https://lucene.apache.org/core/[Lucene] -segments in index shards. +segments in index shards. For data streams, the API returns information about +the stream's backing indices. [source,console] ---- @@ -17,7 +18,7 @@ GET /twitter/_segments [[index-segments-api-request]] ==== {api-request-title} -`GET //_segments` +`GET //_segments` `GET /_segments` @@ -25,7 +26,13 @@ GET /twitter/_segments [[index-segments-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[index-segments-api-query-params]] @@ -102,7 +109,7 @@ Contains information about whether high compression was enabled. ==== {api-examples-title} -===== Get segment information for a specific index +===== Get segment information for a specific data stream or index [source,console] -------------------------------------------------- @@ -111,7 +118,7 @@ GET /test/_segments // TEST[s/^/PUT test\n{"settings":{"number_of_shards":1, "number_of_replicas": 0}}\nPOST test\/_doc\?refresh\n{"test": "test"}\n/] -===== Get segment information for several indices +===== Get segment information for several data streams and indices [source,console] -------------------------------------------------- @@ -120,7 +127,7 @@ GET /test1,test2/_segments // TEST[s/^/PUT test1\nPUT test2\n/] -===== Get segment information for all indices +===== Get segment information for all data streams and indices in a cluster [source,console] -------------------------------------------------- diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 88adc86e46e69..ce94c6d3cbba7 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -4,7 +4,8 @@ Index stats ++++ -Returns statistics for an index. +Returns statistics for one or more indices. For data streams, the API retrieves +statistics for the stream's backing indices. [source,console] ---- @@ -16,9 +17,9 @@ GET /twitter/_stats [[index-stats-api-request]] ==== {api-request-title} -`GET //_stats/` +`GET //_stats/` -`GET //_stats` +`GET //_stats` `GET /_stats` @@ -26,7 +27,8 @@ GET /twitter/_stats [[index-stats-api-desc]] ==== {api-description-title} -Use the index stats API to get high-level aggregation and statistics for an index. +Use the index stats API to get high-level aggregation and statistics for one or +more data streams and indices. By default, the returned statistics are index-level @@ -51,10 +53,13 @@ to which the shard contributed. [[index-stats-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. + -To retrieve statistics for all indices, -use a value of `_all` or `*` or omit this parameter. +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-metric] @@ -91,7 +96,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segme [[index-stats-api-multiple-ex]] -===== Get statistics for multiple indices +===== Get statistics for multiple data streams and indices [source,console] -------------------------------------------------- @@ -101,7 +106,7 @@ GET /index1,index2/_stats [[index-stats-api-all-ex]] -===== Get statistics for all indices +===== Get statistics for all data streams and indices in a cluster [source,console] -------------------------------------------------- diff --git a/docs/reference/licensing/get-trial-status.asciidoc b/docs/reference/licensing/get-trial-status.asciidoc index b411f17b9e3c9..8517c313a92e3 100644 --- a/docs/reference/licensing/get-trial-status.asciidoc +++ b/docs/reference/licensing/get-trial-status.asciidoc @@ -6,7 +6,7 @@ Get trial status ++++ -This API enables you to check the status of your trial license. +Enables you to check the status of your trial. [float] ==== Request @@ -16,16 +16,14 @@ This API enables you to check the status of your trial license. [float] ==== Description -If you want to try the features that are included in a platinum license, you can -start a 30-day trial. +If you want to try all the subscription features, you can start a 30-day trial. -NOTE: You are allowed to initiate a trial license only if your cluster has not -already activated a trial license for the current major product version. For -example, if you have already activated a trial for v6.0, you cannot start a new -trial until v7.0. You can, however, contact `info@elastic.co` to request an -extended trial license. +NOTE: You are allowed to initiate a trial only if your cluster has not +already activated a trial for the current major product version. For example, if +you have already activated a trial for v6.0, you cannot start a new trial until +v7.0. You can, however, request an extended trial at {extendtrial}. -For more information about the different types of licenses, see +For more information about features and subscriptions, see https://www.elastic.co/subscriptions. ==== Authorization diff --git a/docs/reference/licensing/start-trial.asciidoc b/docs/reference/licensing/start-trial.asciidoc index 62123d2ab4253..4401feb3062b0 100644 --- a/docs/reference/licensing/start-trial.asciidoc +++ b/docs/reference/licensing/start-trial.asciidoc @@ -6,7 +6,7 @@ Start trial ++++ -This API starts a 30-day trial license. +Starts a 30-day trial. [float] ==== Request @@ -16,19 +16,17 @@ This API starts a 30-day trial license. [float] ==== Description -The `start trial` API enables you to upgrade from a basic license to a 30-day -trial license, which gives access to the platinum features. +The `start trial` API enables you to start a 30-day trial, which gives access to +all subscription features. -NOTE: You are allowed to initiate a trial license only if your cluster has not -already activated a trial license for the current major product version. For -example, if you have already activated a trial for v6.0, you cannot start a new -trial until v7.0. You can, however, contact `info@elastic.co` to request an -extended trial license. +NOTE: You are allowed to initiate a trial only if your cluster has not already +activated a trial for the current major product version. For example, if you +have already activated a trial for v6.0, you cannot start a new trial until v7.0. +You can, however, request an extended trial at {extendtrial}. -To check the status of your trial license, use the following API: -<>. +To check the status of your trial, use <>. -For more information about the different types of licenses, see +For more information about features and subscriptions, see https://www.elastic.co/subscriptions. ==== Authorization @@ -40,8 +38,8 @@ For more information, see [float] ==== Examples -The following example starts a 30-day trial license. The acknowledge -parameter is required as you are initiating a license that will expire. +The following example starts a 30-day trial. The acknowledge parameter is +required as you are initiating a license that will expire. [source,console] ------------------------------------------------------------ diff --git a/docs/reference/licensing/update-license.asciidoc b/docs/reference/licensing/update-license.asciidoc index ec5d2b930656e..bd20f09aa17a4 100644 --- a/docs/reference/licensing/update-license.asciidoc +++ b/docs/reference/licensing/update-license.asciidoc @@ -21,7 +21,7 @@ Updates the license for your {es} cluster. If {es} {security-features} are enabled, you need `manage` cluster privileges to install the license. -If {es} {security-features} are enabled and you are installing a gold or platinum +If {es} {security-features} are enabled and you are installing a gold or higher license, you must enable TLS on the transport networking layer before you install the license. See <>. diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc index 46a204ccddfde..c0a4c0b29a654 100644 --- a/docs/reference/mapping/fields/routing-field.asciidoc +++ b/docs/reference/mapping/fields/routing-field.asciidoc @@ -43,6 +43,9 @@ GET my_index/_search <1> Querying on the `_routing` field (also see the <>) +NOTE: Data streams do not support custom routing. Instead, target the +appropriate backing index for the stream. + ==== Searching with custom routing Custom routing can reduce the impact of searches. Instead of having to fan diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc index b1c98834d12ca..4e4b923d5a79a 100644 --- a/docs/reference/mapping/types/histogram.asciidoc +++ b/docs/reference/mapping/types/histogram.asciidoc @@ -41,6 +41,7 @@ following aggregations and queries: * <> aggregation * <> aggregation * <> aggregation +* <> aggregation * <> query [[mapping-types-histogram-building-histogram]] diff --git a/docs/reference/migration/migrate_8_0/security.asciidoc b/docs/reference/migration/migrate_8_0/security.asciidoc index 5d09c5b9568b8..2686793b0ec85 100644 --- a/docs/reference/migration/migrate_8_0/security.asciidoc +++ b/docs/reference/migration/migrate_8_0/security.asciidoc @@ -219,7 +219,7 @@ on startup. [[builtin-users-changes]] ==== Changes to built-in users -.The `kibana` user has been renamed `kibana_system`. +.The `kibana` user has been replaced by `kibana_system`. [%collapsible] ==== *Details* + @@ -243,6 +243,9 @@ then you should update to use the new `kibana_system` user instead: -------------------------------------------------- elasticsearch.username: kibana_system -------------------------------------------------- + +IMPORTANT: The new `kibana_system` user does not preserve the previous `kibana` +user password. You must explicitly set a password for the `kibana_system` user. ==== [discrete] diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index 1658af07d6d0b..c574260faa910 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -326,6 +326,14 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=description-dfa] `dest`:: (Required, object) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dest] + +`max_num_threads`:: +(Optional, integer) +The maximum number of threads to be used by the analysis. +The default value is `1`. Using more threads may decrease the time +necessary to complete the analysis at the cost of using more CPU. +Note that the process may use additional threads for operational +functionality other than the analysis itself. `model_memory_limit`:: (Optional, string) @@ -507,7 +515,8 @@ The API returns the following result: "model_memory_limit": "1gb", "create_time" : 1562265491319, "version" : "8.0.0", - "allow_lazy_start" : false + "allow_lazy_start" : false, + "max_num_threads": 1 } ---- // TESTRESPONSE[s/1562265491319/$body.$_path/] diff --git a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc index 0c3a36e95e416..3cb11d7b32b06 100644 --- a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc @@ -71,6 +71,14 @@ the `starting` state until sufficient {ml} node capacity is available. (Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=description-dfa] +`max_num_threads`:: +(Optional, integer) +The maximum number of threads to be used by the analysis. +The default value is `1`. Using more threads may decrease the time +necessary to complete the analysis at the cost of using more CPU. +Note that the process may use additional threads for operational +functionality other than the analysis itself. + `model_memory_limit`:: (Optional, string) The approximate maximum amount of memory resources that are permitted for diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index ea405262e7542..c3d7041f2355e 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -17,6 +17,7 @@ requests to the appropriate node. By default, a node is all of the following types: master-eligible, data, ingest, and (if available) machine learning. All data nodes are also transform nodes. // end::modules-node-description-tag[] + TIP: As the cluster grows and in particular if you have large {ml} jobs or {ctransforms}, consider separating dedicated master-eligible nodes from dedicated data nodes, {ml} nodes, and {transform} nodes. diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc index ecd8d3dbca0c1..cde3e82e90fde 100644 --- a/docs/reference/query-dsl/match-query.asciidoc +++ b/docs/reference/query-dsl/match-query.asciidoc @@ -78,7 +78,7 @@ expand. Defaults to `50`. (Optional, integer) Number of beginning characters left unchanged for fuzzy matching. Defaults to `0`. -`transpositions`:: +`fuzzy_transpositions`:: (Optional, boolean) If `true`, edits for fuzzy matching include transpositions of two adjacent characters (ab → ba). Defaults to `true`. diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 61abcdb7e3c9a..5365352f0d16e 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -142,6 +142,10 @@ Example response: "watcher" : { "available" : true, "enabled" : true + }, + "data_streams" : { + "available" : true, + "enabled" : true, } }, "tagline" : "You know, for X" diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 0c5677861984d..6bd467273265f 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -278,6 +278,12 @@ GET /_xpack/usage "string_stats_usage" : 0, "moving_percentiles_usage" : 0 } + }, + "data_streams" : { + "available" : true, + "enabled" : true, + "data_streams" : 0, + "indices_count" : 0 } } ------------------------------------------------------------ diff --git a/docs/reference/rollup/apis/rollup-index-caps.asciidoc b/docs/reference/rollup/apis/rollup-index-caps.asciidoc index 9fb770d33c657..a79048f7d3616 100644 --- a/docs/reference/rollup/apis/rollup-index-caps.asciidoc +++ b/docs/reference/rollup/apis/rollup-index-caps.asciidoc @@ -14,7 +14,7 @@ experimental[] [[rollup-get-rollup-index-caps-request]] ==== {api-request-title} -`GET /_rollup/data` +`GET /_rollup/data` [[rollup-get-rollup-index-caps-prereqs]] ==== {api-prereq-title} @@ -38,9 +38,9 @@ and what aggregations can be performed on each job? [[rollup-get-rollup-index-caps-path-params]] ==== {api-path-parms-title} -``:: - (Required, string) Index or index-pattern of concrete rollup indices to check - for capabilities. +``:: +(Required, string) Data stream or index to check for rollup capabilities. +Wildcard (`*`) expressions are supported. [[rollup-get-rollup-index-caps-example]] ==== {api-examples-title} diff --git a/docs/reference/rollup/apis/rollup-search.asciidoc b/docs/reference/rollup/apis/rollup-search.asciidoc index 3bb4d1ac2faa9..ab496a8f1116f 100644 --- a/docs/reference/rollup/apis/rollup-search.asciidoc +++ b/docs/reference/rollup/apis/rollup-search.asciidoc @@ -6,14 +6,14 @@ Rollup search ++++ -Enables searching rolled-up data using the standard query DSL. +Enables searching rolled-up data using the standard query DSL. experimental[] [[rollup-search-request]] ==== {api-request-title} -`GET /_rollup_search` +`GET /_rollup_search` [[rollup-search-desc]] ==== {api-description-title} @@ -27,20 +27,28 @@ expect given the original query. [[rollup-search-path-params]] ==== {api-path-parms-title} -``:: - (Required, string) Index, indices or index-pattern to execute a rollup search - against. This can include both rollup and non-rollup indices. +``:: ++ +-- +(Required, string) +Comma-separated list of data streams and indices used to limit +the request. Wildcard expressions (`*`) are supported. -Rules for the `index` parameter: +This target can include both rollup and non-rollup indices. -- At least one index/index-pattern must be specified. This can be either a -rollup or non-rollup index. Omitting the index parameter, or using `_all`, is -not permitted. -- Multiple non-rollup indices may be specified +Rules for the `` parameter: + +- At least one data stream, index, or wildcard expression must be specified. +This target can include a rollup or non-rollup index. For data streams, the +stream's backing indices can only serve as non-rollup indices. Omitting the +`` parameter or using `_all` is not permitted. +- Multiple non-rollup indices may be specified. - Only one rollup index may be specified. If more than one are supplied, an exception occurs. -- Index patterns may be used, but if they match more than one rollup index an -exception occurs. +- Wildcard expressions may be used, but, if they match more than one rollup index, an +exception occurs. However, you can use an expression to match multiple non-rollup +indices or data streams. +-- [[rollup-search-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/search.asciidoc b/docs/reference/search.asciidoc index 4ece1b36d0362..944ec64bfab86 100644 --- a/docs/reference/search.asciidoc +++ b/docs/reference/search.asciidoc @@ -168,16 +168,12 @@ include::search/suggesters.asciidoc[] include::search/multi-search.asciidoc[] -ifdef::permanently-unreleased-branch[] - include::eql/eql-search-api.asciidoc[] include::eql/get-async-eql-search-api.asciidoc[] include::eql/delete-async-eql-search-api.asciidoc[] -endif::[] - include::search/count.asciidoc[] include::search/validate.asciidoc[] diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 7248ec50fe828..1c42a454bd0a1 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -2,6 +2,8 @@ === Field Capabilities API Allows you to retrieve the capabilities of fields among multiple indices. +For data streams, the API returns field capabilities among the stream's backing +indices. [source,console] -------------------------------------------------- @@ -16,9 +18,9 @@ GET /_field_caps?fields=rating `POST /_field_caps` -`GET //_field_caps` +`GET //_field_caps` -`POST //_field_caps` +`POST //_field_caps` [[search-field-caps-api-desc]] @@ -32,7 +34,13 @@ fields among multiple indices. [[search-field-caps-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases used to limit +the request. Wildcard expressions (`*`) are supported. ++ +To target all data streams and indices in a cluster, omit this parameter or use +`_all` or `*`. [[search-field-caps-api-query-params]] @@ -104,7 +112,7 @@ field types are all described as the `keyword` family type. ==== {api-examples-title} -The request can be restricted to specific indices: +The request can be restricted to specific data streams and indices: [source,console] -------------------------------------------------- diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 3e61f43b9b5fe..9231e1e854a17 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -13,7 +13,7 @@ GET twitter/_validate/query?q=user:foo [[search-validate-api-request]] ==== {api-request-title} -`GET //_validate/` +`GET //_validate/` [[search-validate-api-desc]] @@ -27,7 +27,13 @@ request body. [[search-validate-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +``:: +(Optional, string) +Comma-separated list of data streams, indices, and index aliases to search. +Wildcard (`*`) expressions are supported. ++ +To search all data streams or indices in a cluster, omit this parameter or use +`_all` or `*`. include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] diff --git a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc index 5ca9cc5a4e085..9081eac4c5523 100644 --- a/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc +++ b/docs/reference/searchable-snapshots/apis/clear-cache.asciidoc @@ -15,7 +15,7 @@ Clear the cache of searchable snapshots. `POST /_searchable_snapshots/cache/clear` -`POST //_searchable_snapshots/cache/clear` +`POST //_searchable_snapshots/cache/clear` [[searchable-snapshots-api-clear-cache-prereqs]] ==== {api-prereq-title} @@ -32,9 +32,9 @@ For more information, see <>. [[searchable-snapshots-api-clear-cache-path-params]] ==== {api-path-parms-title} -``:: +``:: (Optional, string) -A comma-separated list of index names for which the +A comma-separated list of data streams and indices for which the searchable snapshots cache must be cleared. diff --git a/docs/reference/searchable-snapshots/apis/get-stats.asciidoc b/docs/reference/searchable-snapshots/apis/get-stats.asciidoc index 4a8914553a55f..c3b17a5ca1e48 100644 --- a/docs/reference/searchable-snapshots/apis/get-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/get-stats.asciidoc @@ -15,7 +15,7 @@ Retrieve various statistics about searchable snapshots. `GET /_searchable_snapshots/stats` -`GET //_searchable_snapshots/stats` +`GET //_searchable_snapshots/stats` [[searchable-snapshots-api-stats-prereqs]] ==== {api-prereq-title} @@ -32,9 +32,9 @@ For more information, see <>. [[searchable-snapshots-api-stats-path-params]] ==== {api-path-parms-title} -``:: +``:: (Optional, string) -A comma-separated list of index names for which the +A comma-separated list of data streams and indices for which the statistics must be retrieved. diff --git a/docs/reference/setup/restart-cluster.asciidoc b/docs/reference/setup/restart-cluster.asciidoc index c5dad99e6411c..87734a86fd462 100644 --- a/docs/reference/setup/restart-cluster.asciidoc +++ b/docs/reference/setup/restart-cluster.asciidoc @@ -36,8 +36,7 @@ POST /_flush . *Temporarily stop the tasks associated with active {ml} jobs and {dfeeds}.* (Optional) + -- -{ml-cap} features require a platinum license or higher. For more information about Elastic -license levels, see https://www.elastic.co/subscriptions[the subscription page]. +{ml-cap} features require specific {subscriptions}[subscriptions]. You have two options to handle {ml} jobs and {dfeeds} when you shut down a cluster: diff --git a/docs/reference/slm/apis/index.asciidoc b/docs/reference/slm/apis/index.asciidoc index 22920205edf1f..f3359ffaad19a 100644 --- a/docs/reference/slm/apis/index.asciidoc +++ b/docs/reference/slm/apis/index.asciidoc @@ -4,12 +4,12 @@ == Manage the snapshot lifecycle You can set up snapshot lifecycle policies to automate the timing, frequency, and retention of snapshots. -Snapshot policies can apply to multiple indices. +Snapshot policies can apply to multiple data streams and indices. The snapshot lifecycle management (SLM) <> provide the building blocks for the snapshot policy features that are part of the Management application in {kib}. -The Snapshot and Restore UI makes it easy to set up policies, register snapshot repositories, -view and manage snapshots, and restore indices. +The Snapshot and Restore UI makes it easy to set up policies, register snapshot repositories, +view and manage snapshots, and restore data streams or indices. You can stop and restart SLM to temporarily pause automatic backups while performing upgrades or other maintenance. diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index b993a022eb38f..6c531c7da1e64 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -56,7 +56,7 @@ Configuration for each snapshot created by the policy. ==== `ignore_unavailable`:: (Optional, boolean) -If `true`, missing indices do *not* cause snapshot creation to fail and return +If `true`, missing data streams or indices do *not* cause snapshot creation to fail and return an error. Defaults to `false`. `include_global_state`:: @@ -65,14 +65,15 @@ If `true`, cluster states are included in snapshots. Defaults to `false`. `indices`:: (Optional, array of strings) -Array of index names or wildcard pattern of index names included in snapshots. It -supports <> expressions. +Array of data streams and indices to include in snapshots. +<> and wildcard (`*`) expressions are +supported. ==== `name`:: (Required, string) -Name automatically assigned to each snapshot created by the policy. This value -supports the same <> supported in index names. +Name automatically assigned to each snapshot created by the policy. +<> is supported. To prevent conflicting snapshot names, a UUID is automatically appended to each snapshot name. @@ -141,7 +142,7 @@ PUT /_slm/policy/daily-snapshots <2> The name each snapshot should be given <3> Which repository to take the snapshot in <4> Any extra snapshot configuration -<5> Which indices the snapshot should contain +<5> Data streams and indices the snapshot should contain <6> Optional retention configuration <7> Keep snapshots for 30 days <8> Always keep at least 5 successful snapshots, even if they're more than 30 days old diff --git a/docs/reference/slm/apis/slm-stop.asciidoc b/docs/reference/slm/apis/slm-stop.asciidoc index a311d5359288e..c902f3b8123be 100644 --- a/docs/reference/slm/apis/slm-stop.asciidoc +++ b/docs/reference/slm/apis/slm-stop.asciidoc @@ -27,7 +27,7 @@ cluster privilege to use this API. For more information, see Halts all {slm} ({slm-init}) operations and stops the {slm-init} plugin. This is useful when you are performing maintenance on a cluster and need to -prevent {slm-init} from performing any actions on your indices. +prevent {slm-init} from performing any actions on your data streams or indices. Stopping {slm-init} does not stop any snapshots that are in progress. You can manually trigger snapshots with the <> even if {slm-init} is stopped. diff --git a/docs/reference/slm/getting-started-slm.asciidoc b/docs/reference/slm/getting-started-slm.asciidoc index 8f70946e39f2c..3c8e8cd6d2496 100644 --- a/docs/reference/slm/getting-started-slm.asciidoc +++ b/docs/reference/slm/getting-started-slm.asciidoc @@ -3,8 +3,8 @@ [[getting-started-snapshot-lifecycle-management]] === Tutorial: Automate backups with {slm-init} -This tutorial demonstrates how to automate daily backups of {es} indices using an {slm-init} policy. -The policy takes <> of all indices in the cluster +This tutorial demonstrates how to automate daily backups of {es} data streams and indices using an {slm-init} policy. +The policy takes <> of all data streams and indices in the cluster and stores them in a local repository. It also defines a retention policy and automatically deletes snapshots when they are no longer needed. @@ -47,7 +47,7 @@ PUT /_snapshot/my_repository Once you have a repository in place, you can define an {slm-init} policy to take snapshots automatically. -The policy defines when to take snapshots, which indices should be included, +The policy defines when to take snapshots, which data streams or indices should be included, and what to name the snapshots. A policy can also specify a <> and automatically delete snapshots when they are no longer needed. @@ -58,7 +58,7 @@ Snapshots are incremental and make efficient use of storage. You can define and manage policies through {kib} Management or with the put policy API. For example, you could define a `nightly-snapshots` policy -to back up all of your indices daily at 2:30AM UTC. +to back up all of your data streams and indices daily at 2:30AM UTC. A put policy request defines the policy configuration in JSON: @@ -86,13 +86,13 @@ PUT /_slm/policy/nightly-snapshots <> to include the current date in the snapshot name <3> Where to store the snapshot <4> The configuration to be used for the snapshot requests (see below) -<5> Which indices to include in the snapshot: all indices +<5> Which data streams or indices to include in the snapshot: all data streams and indices <6> Optional retention policy: keep snapshots for 30 days, retaining at least 5 and no more than 50 snapshots regardless of age You can specify additional snapshot configuration options to customize how snapshots are taken. For example, you could configure the policy to fail the snapshot -if one of the specified indices is missing. +if one of the specified data streams or indices is missing. For more information about snapshot options, see <>. [discrete] diff --git a/docs/reference/slm/index.asciidoc b/docs/reference/slm/index.asciidoc index 34594910d99b7..a90489ff7589d 100644 --- a/docs/reference/slm/index.asciidoc +++ b/docs/reference/slm/index.asciidoc @@ -4,12 +4,12 @@ == {slm-init}: Manage the snapshot lifecycle You can set up snapshot lifecycle policies to automate the timing, frequency, and retention of snapshots. -Snapshot policies can apply to multiple indices. +Snapshot policies can apply to multiple data streams and indices. The {slm} ({slm-init}) <> provide the building blocks for the snapshot policy features that are part of {kib} Management. {kibana-ref}/snapshot-repositories.html[Snapshot and Restore] makes it easy to -set up policies, register snapshot repositories, view and manage snapshots, and restore indices. +set up policies, register snapshot repositories, view and manage snapshots, and restore data streams or indices. You can stop and restart {slm-init} to temporarily pause automatic backups while performing upgrades or other maintenance. diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index 4bc544c6c2ac0..e6e05f83bf3be 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -45,11 +45,11 @@ cluster, as well as the cluster state. You can change this behavior by specifying a list of data streams and indices to back up in the body of the snapshot request. -NOTE: You must register a snapshot before performing snapshot and restore operations. Use the <> to register new repositories and update existing ones. +NOTE: You must register a snapshot repository before performing snapshot and restore operations. Use the <> to register new repositories and update existing ones. The snapshot process is incremental. When creating a snapshot, {es} analyzes the list of files that are already stored in the repository and copies only files that were created or changed since the last snapshot. This process allows multiple snapshots to be preserved in the repository in a compact form. -The snapshot process is executed in non-blocking fashion, so all indexing and searching operations can run concurrently against the data stream or index that {es} is snapshotting. Only one snapshot process can run in the cluster at any time. +The snapshot process is executed in non-blocking fashion, so all indexing and searching operations can run concurrently against the data stream or index that {es} is snapshotting. A snapshot represents a point-in-time view of the moment when the snapshot was created. No records that were added to a data stream or index after the snapshot process started will be present in the snapshot. @@ -124,9 +124,6 @@ If `true`, allows taking a partial snapshot of indices with unavailable shards. If `true`, the request returns a response when the snapshot is complete. If `false`, the request returns a response when the snapshot initializes. Defaults to `false`. -+ -NOTE: During snapshot initialization, information about all -previous snapshots is loaded into memory. In large repositories, this load time can cause requests to take several seconds (or even minutes) to return a response, even if the `wait_for_completion` parameter is `false`. [[create-snapshot-api-example]] ==== {api-examples-title} diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc new file mode 100644 index 0000000000000..5ff01f6a201fe --- /dev/null +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -0,0 +1,240 @@ +[[get-snapshot-api]] +=== Get snapshot API +++++ +Get snapshot +++++ + +Retrieves information about one or more snapshots. + +//// +[source,console] +---- +PUT /_snapshot/my_repository +{ + "type": "fs", + "settings": { + "location": "my_backup_location" + } +} + +PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true + +PUT /_snapshot/my_repository/snapshot_2?wait_for_completion=true +---- +// TESTSETUP +//// + +[source,console] +---- +GET /_snapshot/my_repository/my_snapshot +---- + +[[get-snapshot-api-request]] +==== {api-request-title} + +`GET /_snapshot//` + +[[get-snapshot-api-desc]] +==== {api-description-title} + +Use the get snapshot API to return information about one or more snapshots, including: + +* Start and end time values +* Version of {es} that created the snapshot +* List of included indices +* Current state of the snapshot +* List of failures that occurred during the snapshot + +[[get-snapshot-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Comma-separated list of snapshot repository names used to limit the request. +Wildcard (`*`) expressions are supported. ++ +To get information about all snapshot repositories registered in the +cluster, omit this parameter or use `*` or `_all`. + +``:: +(Required, string) +Comma-separated list of snapshot names to retrieve. Also accepts wildcards (`*`). ++ +* To get information about all snapshots in a registered repository, use a wildcard (`*`) or `_all`. +* To get information about any snapshots that are currently running, use `_current`. ++ +NOTE: Using `_all` in a request fails if any snapshots are unavailable. +Set <> to `true` to return only available snapshots. + +[role="child_attributes"] +[[get-snapshot-api-request-body]] +==== {api-request-body-title} + +[[get-snapshot-api-ignore-unavailable]] +`ignore_unavailable`:: +(Optional, boolean) +If `false`, the request returns an error for any snapshots that are unavailable. Defaults to `false`. ++ +If `true`, the request ignores snapshots that are unavailable, such as those that are corrupted or temporarily cannot be returned. + +`verbose`:: +(Optional, boolean) +If `true`, returns all available information about a snapshot. Defaults to `true`. ++ +If `false`, omits additional information about the snapshot, such as version information, start and end times, and the number of snapshotted shards. + +[role="child_attributes"] +[[get-snapshot-api-response-body]] +==== {api-response-body-title} + +`snapshot`:: +(string) +Name of the snapshot. + +`uuid`:: +(string) +Universally unique identifier (UUID) of the snapshot. + +`version_id`:: +(int) +Build ID of the {es} version used to create the snapshot. + +`version`:: +(float) +{es} version used to create the snapshot. + +`indices`:: +(array) +List of indices included in the snapshot. + +`data_streams`:: +(array) +List of <> included in the snapshot. + +`include_global_state`:: +(boolean) +Indicates whether the current cluster state is included in the snapshot. + +`start_time`:: +(string) +Date timestamp of when the snapshot creation process started. + +`start_time_in_millis`:: +(long) +The time, in milliseconds, when the snapshot creation process started. + +`end_time`:: +(string) +Date timestamp of when the snapshot creation process ended. + +`end_time_in_millis`:: +(long) +The time, in milliseconds, when the snapshot creation process ended. + +`duration_in_millis`:: +(long) +How long, in milliseconds, it took to create the snapshot. + +[[get-snapshot-api-response-failures]] +`failures`:: +(array) +Lists any failures that occurred when creating the snapshot. + +`shards`:: +(object) +Contains a count of shards in the snapshot. ++ +.Properties of `shards` +[%collapsible%open] +==== +`total`:: +(integer) +Total number of shards included in the snapshot. + +`successful`:: +(integer) +Number of shards that were successfully included in the snapshot. + +`failed`:: +(integer) +Number of shards that failed to be included in the snapshot. +==== + +`state`:: ++ +-- +(string) +The snapshot `state` can be one of the following values: + +.Values for `state` +[%collapsible%open] +==== +`IN_PROGRESS`:: + The snapshot is currently running. + +`SUCCESS`:: + The snapshot finished and all shards were stored successfully. + +`FAILED`:: + The snapshot finished with an error and failed to store any data. + +`PARTIAL`:: + The global cluster state was stored, but data of at least one shard was not stored successfully. + The <> section of the response contains more detailed information about shards + that were not processed correctly. +==== +-- + +[[get-snapshot-api-example]] +==== {api-examples-title} + +The following request returns information for `snapshot_2` in the `my_repository` repository. + +[source,console] +---- +GET /_snapshot/my_repository/snapshot_2 +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "responses": [ + { + "repository": "my_repository", + "snapshots": [ + { + "snapshot": "snapshot_2", + "uuid": "vdRctLCxSketdKb54xw67g", + "version_id": , + "version": , + "indices": [], + "data_streams": [], + "include_global_state": true, + "state": "SUCCESS", + "start_time": "2020-07-06T21:55:18.129Z", + "start_time_in_millis": 1593093628850, + "end_time": "2020-07-06T21:55:18.129Z", + "end_time_in_millis": 1593094752018, + "duration_in_millis": 0, + "failures": [], + "shards": { + "total": 0, + "failed": 0, + "successful": 0 + } + } + ] + } + ] +} +---- +// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.responses.0.snapshots.0.uuid/] +// TESTRESPONSE[s/"version_id": /"version_id": $body.responses.0.snapshots.0.version_id/] +// TESTRESPONSE[s/"version": /"version": $body.responses.0.snapshots.0.version/] +// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.responses.0.snapshots.0.start_time/] +// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.responses.0.snapshots.0.start_time_in_millis/] +// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.responses.0.snapshots.0.end_time/] +// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.responses.0.snapshots.0.end_time_in_millis/] +// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.responses.0.snapshots.0.duration_in_millis/] diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index 6310cb3917797..f30f04d754310 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -25,6 +25,7 @@ content may not be included yet. [[snapshot-management-apis]] === Snapshot management APIs * <> +* <> * <> include::put-repo-api.asciidoc[] @@ -33,4 +34,5 @@ include::get-repo-api.asciidoc[] include::delete-repo-api.asciidoc[] include::clean-up-repo-api.asciidoc[] include::create-snapshot-api.asciidoc[] +include::get-snapshot-api.asciidoc[] include::delete-snapshot-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/index.asciidoc b/docs/reference/snapshot-restore/index.asciidoc index ae0547db64095..16397a7d3f8b2 100644 --- a/docs/reference/snapshot-restore/index.asciidoc +++ b/docs/reference/snapshot-restore/index.asciidoc @@ -5,25 +5,28 @@ -- // tag::snapshot-intro[] -A _snapshot_ is a backup taken from a running {es} cluster. -You can take snapshots of individual indices or of the entire cluster. -Snapshots can be stored in either local or remote repositories. -Remote repositories can reside on S3, HDFS, Azure, Google Cloud Storage, +A _snapshot_ is a backup taken from a running {es} cluster. +You can take snapshots of an entire cluster, including all its data streams and +indices. You can also take snapshots of only specific data streams or indices in +the cluster. + +Snapshots can be stored in either local or remote repositories. +Remote repositories can reside on S3, HDFS, Azure, Google Cloud Storage, and other platforms supported by a repository plugin. -Snapshots are incremental: each snapshot of an index only stores data that -is not part of an earlier snapshot. +Snapshots are incremental: each snapshot only stores data that +is not part of an earlier snapshot. This enables you to take frequent snapshots with minimal overhead. -// end::snapshot-intro[] +// end::snapshot-intro[] // tag::restore-intro[] -You can restore snapshots to a running cluster with the <>. -By default, all indices in the snapshot are restored. -Alternatively, you can restore specific indices or restore the cluster state from a snapshot. -When restoring indices, you can modify the index name and selected index settings. +You can restore snapshots to a running cluster with the <>. +By default, all data streams and indices in the snapshot are restored. +However, you can choose to restore only the cluster state or specific data +streams or indices from a snapshot. // end::restore-intro[] -You must <> +You must <> before you can <>. You can use <> @@ -50,7 +53,7 @@ compatibility. Follow the <> when migrating between versions. A snapshot contains a copy of the on-disk data structures that make up an -index. This means that snapshots can only be restored to versions of +index or a data stream's backing indices. This means that snapshots can only be restored to versions of {es} that can read the indices: * A snapshot of an index created in 6.x can be restored to 7.x. @@ -67,20 +70,21 @@ We do not recommend restoring snapshots from later {es} versions in earlier versions. In some cases, the snapshots cannot be restored. For example, a snapshot taken in 7.6.0 cannot be restored to 7.5.0. -Each snapshot can contain indices created in various versions of {es}, -and when restoring a snapshot it must be possible to restore all of the indices -into the target cluster. If any indices in a snapshot were created in an -incompatible version, you will not be able restore the snapshot. +Each snapshot can contain indices created in various versions of {es}. This +includes backing indices created for data streams. When restoring a snapshot, it +must be possible to restore all of these indices into the target cluster. If any +indices in a snapshot were created in an incompatible version, you will not be +able restore the snapshot. IMPORTANT: When backing up your data prior to an upgrade, keep in mind that you won't be able to restore snapshots after you upgrade if they contain indices created in a version that's incompatible with the upgrade version. -If you end up in a situation where you need to restore a snapshot of an index +If you end up in a situation where you need to restore a snapshot of a data stream or index that is incompatible with the version of the cluster you are currently running, you can restore it on the latest compatible version and use -<> to rebuild the index on the current -version. Reindexing from remote is only possible if the original index has +<> to rebuild the data stream or index on the current +version. Reindexing from remote is only possible if the original data stream or index has source enabled. Retrieving and reindexing the data can take significantly longer than simply restoring a snapshot. If you have a large amount of data, we recommend testing the reindex from remote process with a subset of your data to diff --git a/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc b/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc index 0f2c8619b06ac..77cf59b1afc41 100644 --- a/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc +++ b/docs/reference/snapshot-restore/monitor-snapshot-restore.asciidoc @@ -76,15 +76,15 @@ DELETE /_snapshot/my_backup/snapshot_1 // TEST[continued] The restore operation uses the standard shard recovery mechanism. Therefore, any currently running restore operation can -be canceled by deleting indices that are being restored. Please note that data for all deleted indices will be removed +be canceled by deleting data streams and indices that are being restored. Please note that data for all deleted data streams and indices will be removed from the cluster as a result of this operation. [float] === Effect of cluster blocks on snapshot and restore Many snapshot and restore operations are affected by cluster and index blocks. For example, registering and unregistering -repositories require write global metadata access. The snapshot operation requires that all indices and their metadata as -well as the global metadata were readable. The restore operation requires the global metadata to be writable, however +repositories require write global metadata access. The snapshot operation requires that all indices, backing indices, and their metadata as +well as the global metadata be readable. The restore operation requires the global metadata to be writable, however the index level blocks are ignored during restore because indices are essentially recreated during restore. Please note that a repository content is not part of the cluster and therefore cluster blocks don't affect internal repository operations such as listing or deleting snapshots from an already registered repository. diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 1b0ec4585c73d..3ef4beee96116 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -1,9 +1,5 @@ [[snapshots-restore-snapshot]] -== Restore indices from a snapshot - -++++ -Restore a snapshot -++++ +== Restore a snapshot //// [source,console] @@ -29,15 +25,48 @@ A snapshot can be restored using the following command: POST /_snapshot/my_backup/snapshot_1/_restore ----------------------------------- -By default, all indices in the snapshot are restored, and the cluster state is -*not* restored. It's possible to select indices that should be restored as well +By default, all data streams and indices in the snapshot are restored, but the cluster state is +*not* restored. It's possible to select specific data streams or indices that should be restored as well as to allow the global cluster state from being restored by using `indices` and -`include_global_state` options in the restore request body. The list of indices -supports <>. The `rename_pattern` -and `rename_replacement` options can be also used to rename indices on restore +`include_global_state` options in the restore request body. The list +supports <>. + +[WARNING] +==== +Each data stream requires a matching +<>. The stream uses this +template to create new backing indices. + +When restoring a data stream, ensure a matching template exists for the stream. +You can do this using one of the following methods: + +* Check for existing templates that match the stream. If no matching template + exists, <>. + +* Restore a global cluster state that includes a matching template for the + stream. + +If no index template matches a data stream, the stream cannot +<> or create new backing indices. +==== + +The `rename_pattern` +and `rename_replacement` options can be also used to rename data streams and indices on restore using regular expression that supports referencing the original text as explained http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]. + +If you rename a restored data stream, its backing indices are also +renamed. For example, if you rename the `logs` data stream to `restored-logs`, +the backing index `.ds-logs-000005` is renamed to `.ds-restored-logs-000005`. + +[WARNING] +==== +If you rename a restored stream, ensure an index template matches the new stream +name. If no index template matches the stream, it cannot +<> or create new backing indices. +==== + Set `include_aliases` to `false` to prevent aliases from being restored together with associated indices. @@ -45,7 +74,7 @@ with associated indices. ----------------------------------- POST /_snapshot/my_backup/snapshot_1/_restore { - "indices": "index_1,index_2", + "indices": "data_stream_1,index_1,index_2", "ignore_unavailable": true, "include_global_state": false, <1> "rename_pattern": "index_(.+)", @@ -69,10 +98,22 @@ has the same number of shards as the index in the snapshot. The restore operation automatically opens restored indices if they were closed and creates new indices if they didn't exist in the cluster. +If a data stream is restored, its backing indices are also restored. The restore +operation automatically opens restored backing indices if they were closed. + +NOTE: You cannot restore a data stream if a stream with the same name already +exists. + +In addition to entire data streams, you can restore only specific backing +indices from a snapshot. However, restored backing indices are not automatically +added to any existing data streams. For example, if only the `.ds-logs-000003` +backing index is restored from a snapshot, it is not automatically added to the +existing `logs` data stream. + [float] === Partial restore -By default, the entire restore operation will fail if one or more indices participating in the operation don't have +By default, the entire restore operation will fail if one or more indices or backing indices participating in the operation don't have snapshots of all shards available. It can occur if some shards failed to snapshot for example. It is still possible to restore such indices by setting `partial` to `true`. Please note, that only successfully snapshotted shards will be restored in this case and all missing shards will be recreated empty. @@ -102,6 +143,21 @@ POST /_snapshot/my_backup/snapshot_1/_restore Please note, that some settings such as `index.number_of_shards` cannot be changed during restore operation. +For data streams, these index settings are applied to the restored backing +indices. + +[IMPORTANT] +==== +The `index_settings` and `ignore_index_settings` parameters affect +restored backing indices only. New backing indices created for a stream use the index +settings specified in the stream's matching +<>. + +If you change index settings during a restore, we recommend you make similar +changes in the stream's matching index template. This ensures new backing +indices created for the stream use the same index settings. +==== + [float] === Restoring to a different cluster @@ -111,11 +167,11 @@ containing the snapshot in the new cluster and starting the restore process. The same size or topology. However, the version of the new cluster should be the same or newer (only 1 major version newer) than the cluster that was used to create the snapshot. For example, you can restore a 1.x snapshot to a 2.x cluster, but not a 1.x snapshot to a 5.x cluster. If the new cluster has a smaller size additional considerations should be made. First of all it's necessary to make sure -that new cluster have enough capacity to store all indices in the snapshot. It's possible to change indices settings +that new cluster have enough capacity to store all data streams and indices in the snapshot. It's possible to change index settings during restore to reduce the number of replicas, which can help with restoring snapshots into smaller cluster. It's also -possible to select only subset of the indices using the `indices` parameter. +possible to select only subset of the data streams or indices using the `indices` parameter. -If indices in the original cluster were assigned to particular nodes using +If indices or backing indices in the original cluster were assigned to particular nodes using <>, the same rules will be enforced in the new cluster. Therefore if the new cluster doesn't contain nodes with appropriate attributes that a restored index can be allocated on, such index will not be successfully restored unless these index allocation settings are changed during restore operation. diff --git a/docs/reference/snapshot-restore/take-snapshot.asciidoc b/docs/reference/snapshot-restore/take-snapshot.asciidoc index 495b107b60099..1d52928db538c 100644 --- a/docs/reference/snapshot-restore/take-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/take-snapshot.asciidoc @@ -1,9 +1,5 @@ [[snapshots-take-snapshot]] -== Take a snapshot of one or more indices - -++++ -Take a snapshot -++++ +== Take a snapshot A repository can contain multiple snapshots of the same cluster. Snapshots are identified by unique names within the cluster. A snapshot with the name `snapshot_1` in the repository `my_backup` can be created by executing the following @@ -33,14 +29,14 @@ initialization (default) or wait for snapshot completion. During snapshot initia previous snapshots is loaded into the memory, which means that in large repositories it may take several seconds (or even minutes) for this command to return even if the `wait_for_completion` parameter is set to `false`. -By default a snapshot of all open and started indices in the cluster is created. This behavior can be changed by -specifying the list of indices in the body of the snapshot request. +By default a snapshot backs up all data streams and open indices in the cluster. This behavior can be changed by +specifying the list of data streams and indices in the body of the snapshot request. [source,console] ----------------------------------- PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true { - "indices": "index_1,index_2", + "indices": "data_stream_1,index_1,index_2", "ignore_unavailable": true, "include_global_state": false, "metadata": { @@ -51,13 +47,31 @@ PUT /_snapshot/my_backup/snapshot_2?wait_for_completion=true ----------------------------------- // TEST[skip:cannot complete subsequent snapshot] -The list of indices that should be included into the snapshot can be specified using the `indices` parameter that +The list of data streams and indices that should be included into the snapshot can be specified using the `indices` parameter that supports <>, although the options which control the behavior of multi index syntax -must be supplied in the body of the request, rather than as request parameters. The snapshot request also supports the -`ignore_unavailable` option. Setting it to `true` will cause indices that do not exist to be ignored during snapshot -creation. By default, when `ignore_unavailable` option is not set and an index is missing the snapshot request will fail. +must be supplied in the body of the request, rather than as request parameters. + +Data stream backups include the stream's backing indices and metadata, such as +the current <> and timestamp field. + +You can also choose to include only specific backing indices in a snapshot. +However, these backups do not include the associated data stream's +metadata or its other backing indices. + +The snapshot request also supports the +`ignore_unavailable` option. Setting it to `true` will cause data streams and indices that do not exist to be ignored during snapshot +creation. By default, when the `ignore_unavailable` option is not set and a data stream or index is missing, the snapshot request will fail. + By setting `include_global_state` to false it's possible to prevent the cluster global state to be stored as part of -the snapshot. By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have +the snapshot. + +IMPORTANT: The global cluster state includes the cluster's index +templates, such as those <>. If your snapshot includes data streams, we recommend storing the +cluster state as part of the snapshot. This lets you later restored any +templates required for a data stream. + +By default, the entire snapshot will fail if one or more indices participating in the snapshot don't have all primary shards available. This behaviour can be changed by setting `partial` to `true`. The `expand_wildcards` option can be used to control whether hidden and closed indices will be included in the snapshot, and defaults to `all`. @@ -65,7 +79,7 @@ The `metadata` field can be used to attach arbitrary metadata to the snapshot. T why it was taken, or any other data that might be useful. Snapshot names can be automatically derived using <>, similarly as when creating -new indices. Note that special characters need to be URI encoded. +new data streams or indices. Note that special characters need to be URI encoded. For example, creating a snapshot with the current day in the name, like `snapshot-2018.05.11`, can be achieved with the following command: @@ -78,18 +92,18 @@ PUT /_snapshot/my_backup/%3Csnapshot-%7Bnow%2Fd%7D%3E // TEST[continued] -The index snapshot process is incremental. In the process of making the index snapshot Elasticsearch analyses -the list of the index files that are already stored in the repository and copies only files that were created or +The snapshot process is incremental. In the process of making the snapshot, {es} analyses +the list of the data stream and index files that are already stored in the repository and copies only files that were created or changed since the last snapshot. That allows multiple snapshots to be preserved in the repository in a compact form. Snapshotting process is executed in non-blocking fashion. All indexing and searching operation can continue to be -executed against the index that is being snapshotted. However, a snapshot represents the point-in-time view of the index -at the moment when snapshot was created, so no records that were added to the index after the snapshot process was started +executed against the data stream or index that is being snapshotted. However, a snapshot represents a point-in-time view +at the moment when snapshot was created, so no records that were added to the data stream or index after the snapshot process was started will be present in the snapshot. The snapshot process starts immediately for the primary shards that has been started and are not relocating at the moment. Before version 1.2.0, the snapshot operation fails if the cluster has any relocating or initializing primaries of indices participating in the snapshot. Starting with version 1.2.0, Elasticsearch waits for relocation or initialization of shards to complete before snapshotting them. -Besides creating a copy of each index the snapshot process can also store global cluster metadata, which includes persistent +Besides creating a copy of each data stream and index, the snapshot process can also store global cluster metadata, which includes persistent cluster settings and templates. The transient settings and registered snapshot repositories are not stored as part of the snapshot. @@ -107,35 +121,29 @@ GET /_snapshot/my_backup/snapshot_1 // TEST[continued] This command returns basic information about the snapshot including start and end time, version of -Elasticsearch that created the snapshot, the list of included indices, the current state of the +Elasticsearch that created the snapshot, the list of included data streams and indices, the current state of the snapshot and the list of failures that occurred during the snapshot. The snapshot `state` can be [horizontal] `IN_PROGRESS`:: - The snapshot is currently running. `SUCCESS`:: - The snapshot finished and all shards were stored successfully. `FAILED`:: - The snapshot finished with an error and failed to store any data. `PARTIAL`:: - - The global cluster state was stored, but data of at least one shard wasn't stored successfully. - The `failure` section in this case should contain more detailed information about shards + The global cluster state was stored, but data of at least one shard was not stored successfully. + The `failures` section of the response contains more detailed information about shards that were not processed correctly. `INCOMPATIBLE`:: - - The snapshot was created with an old version of Elasticsearch and therefore is incompatible with + The snapshot was created with an old version of {es} and is incompatible with the current version of the cluster. - -Similar as for repositories, information about multiple snapshots can be queried in one go, supporting wildcards as well: +Similar as for repositories, information about multiple snapshots can be queried in a single request, supporting wildcards as well: [source,console] ----------------------------------- @@ -156,7 +164,7 @@ return all snapshots that are currently available. Getting all snapshots in the repository can be costly on cloud-based repositories, both from a cost and performance perspective. If the only information required is -the snapshot names/uuids in the repository and the indices in each snapshot, then +the snapshot names/uuids in the repository and the data streams and indices in each snapshot, then the optional boolean parameter `verbose` can be set to `false` to execute a more performant and cost-effective retrieval of the snapshots in the repository. Note that setting `verbose` to `false` will omit all other information about the snapshot diff --git a/docs/reference/sql/functions/aggs.asciidoc b/docs/reference/sql/functions/aggs.asciidoc index 7cf08f1c5be10..a506eadd37192 100644 --- a/docs/reference/sql/functions/aggs.asciidoc +++ b/docs/reference/sql/functions/aggs.asciidoc @@ -599,6 +599,35 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[aggStddevPop] include-tagged::{sql-specs}/docs/docs.csv-spec[aggStddevPopScalars] -------------------------------------------------- +[[sql-functions-aggs-stddev-samp]] +==== `STDDEV_SAMP` + +.Synopsis: +[source, sql] +-------------------------------------------------- +STDDEV_SAMP(field_name) <1> +-------------------------------------------------- + +*Input*: + +<1> a numeric field + +*Output*: `double` numeric value + +*Description*: + +Returns the https://en.wikipedia.org/wiki/Standard_deviations[sample standard deviation] of input values in the field `field_name`. + +["source","sql",subs="attributes,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs/docs.csv-spec[aggStddevSamp] +-------------------------------------------------- + +["source","sql",subs="attributes,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs/docs.csv-spec[aggStddevSampScalars] +-------------------------------------------------- + [[sql-functions-aggs-sum-squares]] ==== `SUM_OF_SQUARES` @@ -657,3 +686,33 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[aggVarPop] -------------------------------------------------- include-tagged::{sql-specs}/docs/docs.csv-spec[aggVarPopScalars] -------------------------------------------------- + +[[sql-functions-aggs-var-samp]] +==== `VAR_SAMP` + +.Synopsis: +[source, sql] +-------------------------------------------------- +VAR_SAMP(field_name) <1> +-------------------------------------------------- + +*Input*: + +<1> a numeric field + +*Output*: `double` numeric value + +*Description*: + +Returns the https://en.wikipedia.org/wiki/Variance[sample variance] of input values in the field `field_name`. + +["source","sql",subs="attributes,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs/docs.csv-spec[aggVarSamp] +-------------------------------------------------- + + +["source","sql",subs="attributes,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs/docs.csv-spec[aggVarSampScalars] +-------------------------------------------------- diff --git a/docs/reference/sql/functions/index.asciidoc b/docs/reference/sql/functions/index.asciidoc index 393266fed8e95..c3c0cf27b2472 100644 --- a/docs/reference/sql/functions/index.asciidoc +++ b/docs/reference/sql/functions/index.asciidoc @@ -42,8 +42,10 @@ ** <> ** <> ** <> +** <> ** <> ** <> +** <> * <> ** <> * <> diff --git a/docs/reference/transform/painless-examples.asciidoc b/docs/reference/transform/painless-examples.asciidoc index 3e9758992d5f9..c2c24614ef1cd 100644 --- a/docs/reference/transform/painless-examples.asciidoc +++ b/docs/reference/transform/painless-examples.asciidoc @@ -106,7 +106,7 @@ You can retrieve the last value in a similar way: [discrete] [[painless-time-features]] -==== Getting time features as scripted fields +==== Getting time features by using aggregations This snippet shows how to extract time based features by using Painless in a {transform}. The snippet uses an index where `@timestamp` is defined as a `date` @@ -115,37 +115,39 @@ type field. [source,js] -------------------------------------------------- "aggregations": { - "script_fields": { - "hour_of_day": { <1> - "script": { - "lang": "painless", - "source": """ - ZonedDateTime date = doc['@timestamp'].value; <2> - return date.getHour(); <3> - """ - } - }, - "month_of_year": { <4> - "script": { - "lang": "painless", - "source": """ - ZonedDateTime date = doc['@timestamp'].value; <5> - return date.getMonthValue(); <6> - """ - } + "avg_hour_of_day": { <1> + "avg":{ + "script": { <2> + "source": """ + ZonedDateTime date = doc['@timestamp'].value; <3> + return date.getHour(); <4> + """ } - }, - ... + } + }, + "avg_month_of_year": { <5> + "avg":{ + "script": { <6> + "source": """ + ZonedDateTime date = doc['@timestamp'].value; <7> + return date.getMonthValue(); <8> + """ + } + } + }, + ... } -------------------------------------------------- // NOTCONSOLE -<1> Contains the Painless script that returns the hour of the day. -<2> Sets `date` based on the timestamp of the document. -<3> Returns the hour value from `date`. -<4> Contains the Painless script that returns the month of the year. -<5> Sets `date` based on the timestamp of the document. -<6> Returns the month value from `date`. +<1> Name of the aggregation. +<2> Contains the Painless script that returns the hour of the day. +<3> Sets `date` based on the timestamp of the document. +<4> Returns the hour value from `date`. +<5> Name of the aggregation. +<6> Contains the Painless script that returns the month of the year. +<7> Sets `date` based on the timestamp of the document. +<8> Returns the month value from `date`. [discrete] diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java index 988efecbc83c8..b9a5164e1942d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java @@ -210,6 +210,12 @@ public void declareLong(BiConsumer consumer, ParseField field) { declareField(consumer, p -> p.longValue(), field, ValueType.LONG); } + public void declareLongOrNull(BiConsumer consumer, long nullValue, ParseField field) { + // Using a method reference here angers some compilers + declareField(consumer, p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? nullValue : p.longValue(), + field, ValueType.LONG_OR_NULL); + } + public void declareInt(BiConsumer consumer, ParseField field) { // Using a method reference here angers some compilers declareField(consumer, p -> p.intValue(), field, ValueType.INT); diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle index 2a3bf9b22f0dd..aad781fb2cacd 100644 --- a/modules/aggs-matrix-stats/build.gradle +++ b/modules/aggs-matrix-stats/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.' @@ -28,3 +28,5 @@ restResources { includeCore '_common', 'indices', 'cluster', 'index', 'search', 'nodes' } } + +integTest.enabled = false diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/yamlRestTest/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java similarity index 100% rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java rename to modules/aggs-matrix-stats/src/yamlRestTest/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml similarity index 100% rename from modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yml rename to modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/10_basic.yml diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/20_empty_bucket.yml similarity index 100% rename from modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yml rename to modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/20_empty_bucket.yml diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/30_single_value_field.yml similarity index 100% rename from modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yml rename to modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/30_single_value_field.yml diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml b/modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/40_multi_value_field.yml similarity index 100% rename from modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yml rename to modules/aggs-matrix-stats/src/yamlRestTest/resources/rest-api-spec/test/stats/40_multi_value_field.yml diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index 4af7cf09b4ca7..f631aa725d017 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Adds "built in" analyzers to Elasticsearch.' diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java similarity index 98% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java rename to modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java index 215bf3e9f0f2c..48554fe3d26bb 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/QueryStringWithAnalyzersIT.java @@ -32,7 +32,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -public class QueryStringWithAnalyzersTests extends ESIntegTestCase { +public class QueryStringWithAnalyzersIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { return Arrays.asList(CommonAnalysisPlugin.class); diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java b/modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java similarity index 100% rename from modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java rename to modules/analysis-common/src/yamlRestTest/java/org/elasticsearch/analysis/common/CommonAnalysisClientYamlTestSuiteIT.java diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/10_basic.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/10_basic.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/10_basic.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/20_analyzers.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/20_analyzers.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/40_token_filters.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/50_char_filters.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/50_char_filters.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/cluster.stats/10_analysis_stats.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_analysis_stats.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/cluster.stats/10_analysis_stats.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_analysis_stats.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_analyze.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_analyze.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_analyze.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_synonyms.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_synonyms.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/indices.analyze/10_synonyms.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_synonyms.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices/validate_query/10_synonyms.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/10_match.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/10_match.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/10_match.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/20_ngram_search.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/20_ngram_search.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/20_ngram_search.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/30_ngram_highligthing.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/40_query_string.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/40_query_string.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/40_query_string.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/50_queries_with_synonyms.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/60_synonym_graph.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/60_synonym_graph.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/60_synonym_graph.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/70_intervals.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/70_intervals.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.query/70_intervals.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/70_intervals.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.suggest/20_phrase.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/20_phrase.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.suggest/20_phrase.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.suggest/30_synonyms.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/search.suggest/30_synonyms.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.suggest/30_synonyms.yml diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/termvectors/10_payloads.yml similarity index 100% rename from modules/analysis-common/src/test/resources/rest-api-spec/test/termvectors/10_payloads.yml rename to modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/termvectors/10_payloads.yml diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index e0f5147df8b90..c089b395df062 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -16,13 +16,21 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Placeholder plugin for geospatial features in ES. only registers geo_shape field mapper for now' classname 'org.elasticsearch.geo.GeoPlugin' } +restResources { + restApi { + includeCore '_common', 'indices', 'index', 'search' + } +} artifacts { - restTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) + restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } + +integTest.enabled = false +test.enabled = false diff --git a/modules/geo/src/test/java/org/elasticsearch/geo/GeoTests.java b/modules/geo/src/test/java/org/elasticsearch/geo/GeoTests.java deleted file mode 100644 index 056b485393082..0000000000000 --- a/modules/geo/src/test/java/org/elasticsearch/geo/GeoTests.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.geo; - -import org.elasticsearch.test.ESTestCase; - -public class GeoTests extends ESTestCase { - - public void testStub() { - // the build expects unit tests to exist in a module, so here one is. - } -} diff --git a/modules/geo/src/test/java/org/elasticsearch/geo/GeoClientYamlTestSuiteIT.java b/modules/geo/src/yamlRestTest/java/org/elasticsearch/geo/GeoClientYamlTestSuiteIT.java similarity index 100% rename from modules/geo/src/test/java/org/elasticsearch/geo/GeoClientYamlTestSuiteIT.java rename to modules/geo/src/yamlRestTest/java/org/elasticsearch/geo/GeoClientYamlTestSuiteIT.java diff --git a/modules/geo/src/test/resources/rest-api-spec/test/geo_shape/10_basic.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/10_basic.yml similarity index 100% rename from modules/geo/src/test/resources/rest-api-spec/test/geo_shape/10_basic.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/10_basic.yml diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 3026670bb33d4..2edf77c0c972c 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources' @@ -36,7 +36,7 @@ restResources { } } -testClusters.integTest { +testClusters.all { // Needed in order to test ingest pipeline templating: // (this is because the integTest node is not using default distribution, but only the minimal number of required modules) module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java similarity index 100% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java rename to modules/ingest-common/src/yamlRestTest/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/10_basic.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/110_sort.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/110_sort.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/120_grok.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/130_escape_dot.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/130_escape_dot.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/130_escape_dot.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/140_json.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/150_kv.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/150_kv.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/150_kv.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/160_urldecode.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/160_urldecode.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/160_urldecode.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/170_version.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/170_version.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/170_version.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/180_bytes_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/180_bytes_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/180_bytes_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/190_script_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/190_script_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_default_pipeline.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_default_pipeline.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_default_pipeline.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/200_dissect_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/200_dissect_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_crud.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_crud.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_conditional_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_conditional_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_conditional_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/220_drop_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/220_drop_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/220_drop_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/230_change_target_index.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/230_change_target_index.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/230_change_target_index.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/230_change_target_index.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/240_required_pipeline.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/240_required_pipeline.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/240_required_pipeline.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/250_csv.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/250_csv.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/250_csv.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/250_csv.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/260_seq_no.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/260_seq_no.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/260_seq_no.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/260_seq_no.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/270_set_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/30_date_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/40_mutate.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/40_mutate.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/50_on_failure.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/50_on_failure.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/60_fail.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/60_fail.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_foreach.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_foreach.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml similarity index 100% rename from modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yml rename to modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 3c86246484219..f0525a05d6f9d 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -19,7 +19,8 @@ import org.apache.tools.ant.taskdefs.condition.Os -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Ingest processor that uses lookup geo data based on IP adresses using the MaxMind geo database' @@ -42,6 +43,8 @@ restResources { } } +integTest.enabled = false + task copyDefaultGeoIp2DatabaseFiles(type: Copy) { from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) } into "${project.buildDir}/ingest-geoip" diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeTests.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java similarity index 99% rename from modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeTests.java rename to modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java index c79fdce16f346..d9e11c2090a16 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeTests.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpProcessorNonIngestNodeIT.java @@ -50,7 +50,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; -public class GeoIpProcessorNonIngestNodeTests extends ESIntegTestCase { +public class GeoIpProcessorNonIngestNodeIT extends ESIntegTestCase { public static class IngestGeoIpSettingsPlugin extends Plugin { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java similarity index 100% rename from modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java rename to modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java diff --git a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/10_basic.yml similarity index 100% rename from modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yml rename to modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/10_basic.yml diff --git a/modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml similarity index 100% rename from modules/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml rename to modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml diff --git a/modules/ingest-user-agent/build.gradle b/modules/ingest-user-agent/build.gradle index 9b12ab11df7c6..3d38ea9999574 100644 --- a/modules/ingest-user-agent/build.gradle +++ b/modules/ingest-user-agent/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Ingest processor that extracts information from a user agent' @@ -29,6 +29,8 @@ restResources { } } -testClusters.integTest { +testClusters.all { extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml') } + +integTest.enabled = false diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java similarity index 100% rename from modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java rename to modules/ingest-user-agent/src/yamlRestTest/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/10_basic.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yml rename to modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/10_basic.yml diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml rename to modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml similarity index 100% rename from modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml rename to modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml diff --git a/modules/lang-expression/build.gradle b/modules/lang-expression/build.gradle index 41b31a65fd424..3b8c42fea93e6 100644 --- a/modules/lang-expression/build.gradle +++ b/modules/lang-expression/build.gradle @@ -16,7 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' esplugin { description 'Lucene expressions integration for Elasticsearch' @@ -36,6 +37,8 @@ restResources { } } +integTest.enabled = false + tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' mapping from: /asm-.*/, to: 'asm' diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java similarity index 99% rename from modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java rename to modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java index 09802d5ec3f2c..294576c1c8230 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/MoreExpressionIT.java @@ -61,7 +61,7 @@ import static org.hamcrest.Matchers.notNullValue; // TODO: please convert to unit tests! -public class MoreExpressionTests extends ESIntegTestCase { +public class MoreExpressionIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java similarity index 98% rename from modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java rename to modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java index a9c80466a7a50..8857418a759b8 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/StoredExpressionTests.java +++ b/modules/lang-expression/src/internalClusterTest/java/org/elasticsearch/script/expression/StoredExpressionIT.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.containsString; //TODO: please convert to unit tests! -public class StoredExpressionTests extends ESIntegTestCase { +public class StoredExpressionIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal)); diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java similarity index 100% rename from modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java rename to modules/lang-expression/src/yamlRestTest/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yml b/modules/lang-expression/src/yamlRestTest/resources/rest-api-spec/test/lang_expression/10_basic.yml similarity index 100% rename from modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/10_basic.yml rename to modules/lang-expression/src/yamlRestTest/resources/rest-api-spec/test/lang_expression/10_basic.yml diff --git a/modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml b/modules/lang-expression/src/yamlRestTest/resources/rest-api-spec/test/lang_expression/20_search.yml similarity index 100% rename from modules/lang-expression/src/test/resources/rest-api-spec/test/lang_expression/20_search.yml rename to modules/lang-expression/src/yamlRestTest/resources/rest-api-spec/test/lang_expression/20_search.yml diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle index fb8e3e4e2db43..afa582bec966a 100644 --- a/modules/lang-mustache/build.gradle +++ b/modules/lang-mustache/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Mustache scripting integration for Elasticsearch' diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java similarity index 100% rename from modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java rename to modules/lang-mustache/src/yamlRestTest/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/10_basic.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/10_basic.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/10_basic.yml diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/25_custom_functions.yml diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/30_search_template.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/30_search_template.yml diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yml similarity index 100% rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yml rename to modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/60_typed_keys.yml diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 65c0be326df73..a8e8147108779 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -18,15 +18,15 @@ */ import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask; -apply plugin: 'elasticsearch.rest-resources' apply plugin: 'elasticsearch.validate-rest-spec' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'An easy, safe and fast scripting language for Elasticsearch' classname 'org.elasticsearch.painless.PainlessPlugin' } -testClusters.integTest { +testClusters.all { module project(':modules:mapper-extras').tasks.bundlePlugin.archiveFile systemProperty 'es.scripting.update.ctx_in_params', 'false' // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 @@ -54,6 +54,8 @@ restResources { } } +integTest.enabled = false + test { // in WhenThingsGoWrongTests we intentionally generate an out of memory error, this prevents the heap from being dumped to disk jvmArgs '-XX:-OmitStackTraceInFastThrow', '-XX:-HeapDumpOnOutOfMemoryError' diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java similarity index 100% rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java rename to modules/lang-painless/src/yamlRestTest/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/api/scripts_painless_context.json similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/api/scripts_painless_context.json rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/api/scripts_painless_context.json diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/100_terms_agg.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/100_terms_agg.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/100_terms_agg.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/10_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/10_basic.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/10_basic.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/10_basic.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/110_script_score_boost.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/110_script_score_boost.yml similarity index 98% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/110_script_score_boost.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/110_script_score_boost.yml index ba37d43ea95b3..37dbbe34a3ed1 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/110_script_score_boost.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/110_script_score_boost.yml @@ -1,7 +1,7 @@ # Integration tests for ScriptScoreQuery using Painless setup: - skip: - version: " - 7.9.99" + version: " - 7.99.99" reason: "boost was corrected in script_score query from 8.0" - do: indices.create: diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/15_update.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/15_update.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/16_update2.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/16_update2.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/16_update2.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/17_update_error.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/17_update_error.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/17_update_error.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/17_update_error.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/25_script_upsert.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/25_script_upsert.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/30_search.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/30_search.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/40_disabled.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/40_disabled.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/50_script_doc_values.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_execute_painless_scripts.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/70_mov_fn_agg.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/80_script_score.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/80_script_score.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/85_script_score_random_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/85_script_score_random_score.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/85_script_score_random_score.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/85_script_score_random_score.yml diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/90_interval_query_filter.yml similarity index 100% rename from modules/lang-painless/src/test/resources/rest-api-spec/test/painless/90_interval_query_filter.yml rename to modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/90_interval_query_filter.yml diff --git a/modules/mapper-extras/build.gradle b/modules/mapper-extras/build.gradle index 903e51100cae0..fc60820b33398 100644 --- a/modules/mapper-extras/build.gradle +++ b/modules/mapper-extras/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Adds advanced field mappers' diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index d6eeeb948a271..9d34562f35733 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -103,15 +103,6 @@ public RankFeatureFieldType(String name, Map meta, boolean posit setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected RankFeatureFieldType(RankFeatureFieldType ref) { - super(ref); - this.positiveScoreImpact = ref.positiveScoreImpact; - } - - public RankFeatureFieldType clone() { - return new RankFeatureFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java index 5e996a22d2feb..c7bee347f044c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureMetaFieldMapper.java @@ -85,15 +85,6 @@ private RankFeatureMetaFieldType() { super(NAME, false, false, TextSearchInfo.NONE, Collections.emptyMap()); } - protected RankFeatureMetaFieldType(RankFeatureMetaFieldType ref) { - super(ref); - } - - @Override - public RankFeatureMetaFieldType clone() { - return new RankFeatureMetaFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index e9d0b256e15d2..4968bf5ec6b0e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -80,14 +80,6 @@ public RankFeaturesFieldType(String name, Map meta) { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected RankFeaturesFieldType(RankFeaturesFieldType ref) { - super(ref); - } - - public RankFeaturesFieldType clone() { - return new RankFeaturesFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 88c55c9554084..cc11b3e736add 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -193,20 +193,10 @@ public ScaledFloatFieldType(String name, double scalingFactor) { this(name, true, true, Collections.emptyMap(), scalingFactor); } - ScaledFloatFieldType(ScaledFloatFieldType other) { - super(other); - this.scalingFactor = other.scalingFactor; - } - public double getScalingFactor() { return scalingFactor; } - @Override - public MappedFieldType clone() { - return new ScaledFloatFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -310,19 +300,6 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } } - @Override - public boolean equals(Object o) { - if (super.equals(o) == false) { - return false; - } - return scalingFactor == ((ScaledFloatFieldType) o).scalingFactor; - } - - @Override - public int hashCode() { - return 31 * super.hashCode() + Double.hashCode(scalingFactor); - } - /** * Parses input value and multiplies it with the scaling factor. * Uses the round-trip of creating a {@link BigDecimal} from the stringified {@code double} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index ffdbf198fd610..4ef205a921ecf 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -252,22 +252,6 @@ static class SearchAsYouTypeFieldType extends StringFieldType { new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta); } - SearchAsYouTypeFieldType(SearchAsYouTypeFieldType other) { - super(other); - - if (other.prefixField != null) { - this.prefixField = other.prefixField.clone(); - } - if (other.shingleFields != null) { - this.shingleFields = new ShingleFieldType[other.shingleFields.length]; - for (int i = 0; i < this.shingleFields.length; i++) { - if (other.shingleFields[i] != null) { - this.shingleFields[i] = other.shingleFields[i].clone(); - } - } - } - } - public void setPrefixField(PrefixFieldType prefixField) { this.prefixField = prefixField; } @@ -276,11 +260,6 @@ public void setShingleFields(ShingleFieldType[] shingleFields) { this.shingleFields = shingleFields; } - @Override - public MappedFieldType clone() { - return new SearchAsYouTypeFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -361,27 +340,6 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew return spanMulti; } } - - @Override - public boolean equals(Object otherObject) { - if (this == otherObject) { - return true; - } - if (otherObject == null || getClass() != otherObject.getClass()) { - return false; - } - if (!super.equals(otherObject)) { - return false; - } - final SearchAsYouTypeFieldType other = (SearchAsYouTypeFieldType) otherObject; - return Objects.equals(prefixField, other.prefixField) && - Arrays.equals(shingleFields, other.shingleFields); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), prefixField, Arrays.hashCode(shingleFields)); - } } /** @@ -401,13 +359,6 @@ static final class PrefixFieldType extends StringFieldType { this.parentField = parentField; } - PrefixFieldType(PrefixFieldType other) { - super(other); - this.minChars = other.minChars; - this.maxChars = other.maxChars; - this.parentField = other.parentField; - } - boolean termLengthWithinBounds(int length) { return length >= minChars - 1 && length <= maxChars; } @@ -431,11 +382,6 @@ public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, Quer .build(); } - @Override - public PrefixFieldType clone() { - return new PrefixFieldType(this); - } - @Override public String typeName() { return "prefix"; @@ -450,27 +396,6 @@ public String toString() { public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException(); } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - PrefixFieldType that = (PrefixFieldType) o; - return minChars == that.minChars && - maxChars == that.maxChars; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), minChars, maxChars); - } } static final class PrefixFieldMapper extends FieldMapper { @@ -552,23 +477,10 @@ static class ShingleFieldType extends StringFieldType { this.shingleSize = shingleSize; } - ShingleFieldType(ShingleFieldType other) { - super(other); - this.shingleSize = other.shingleSize; - if (other.prefixFieldType != null) { - this.prefixFieldType = other.prefixFieldType.clone(); - } - } - void setPrefixFieldType(PrefixFieldType prefixFieldType) { this.prefixFieldType = prefixFieldType; } - @Override - public ShingleFieldType clone() { - return new ShingleFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -629,27 +541,6 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew return spanMulti; } } - - @Override - public boolean equals(Object otherObject) { - if (this == otherObject) { - return true; - } - if (otherObject == null || getClass() != otherObject.getClass()) { - return false; - } - if (!super.equals(otherObject)) { - return false; - } - final ShingleFieldType other = (ShingleFieldType) otherObject; - return shingleSize == other.shingleSize - && Objects.equals(prefixFieldType, other.prefixFieldType); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), shingleSize, prefixFieldType); - } } private final int maxShingleSize; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java index 85ec2ec220984..a7566286f80bc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldTypeTests.java @@ -20,14 +20,8 @@ package org.elasticsearch.index.mapper; import java.util.Collections; -import java.util.Map; -public class RankFeatureFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new RankFeatureFieldMapper.RankFeatureFieldType(name, meta, true); - } +public class RankFeatureFieldTypeTests extends FieldTypeTestCase { public void testIsAggregatable() { MappedFieldType fieldType = new RankFeatureFieldMapper.RankFeatureFieldType("field", Collections.emptyMap(), true); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java index 5c02f860d8aca..c52f3a8c2412f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldTypeTests.java @@ -20,17 +20,11 @@ package org.elasticsearch.index.mapper; import java.util.Collections; -import java.util.Map; -public class RankFeaturesFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new RankFeaturesFieldMapper.RankFeaturesFieldType(name, meta); - } +public class RankFeaturesFieldTypeTests extends FieldTypeTestCase { public void testIsAggregatable() { - MappedFieldType fieldType = createDefaultFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new RankFeaturesFieldMapper.RankFeaturesFieldType("field", Collections.emptyMap()); assertFalse(fieldType.isAggregatable()); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java index ff517cf4e35a2..c0031d5b3789f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldTypeTests.java @@ -40,14 +40,9 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Map; -public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { +public class ScaledFloatFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new ScaledFloatFieldMapper.ScaledFloatFieldType(name, true, true, meta, 100); - } public void testTermQuery() { ScaledFloatFieldMapper.ScaledFloatFieldType ft diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index 579ae038b5d66..5d1a5679e557b 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -35,13 +35,12 @@ import org.elasticsearch.index.mapper.SearchAsYouTypeFieldMapper.ShingleFieldType; import java.util.Collections; -import java.util.Map; import static java.util.Arrays.asList; import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; import static org.hamcrest.Matchers.equalTo; -public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase { +public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase { private static final String NAME = "a_field"; private static final FieldType UNSEARCHABLE = new FieldType(); @@ -50,10 +49,9 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase meta) { - final SearchAsYouTypeFieldType fieldType - = new SearchAsYouTypeFieldType(name, Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, meta); + protected SearchAsYouTypeFieldType createFieldType() { + final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(NAME, Defaults.FIELD_TYPE, null, + Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap()); fieldType.setPrefixField(new PrefixFieldType(NAME, TextSearchInfo.SIMPLE_MATCH_ONLY, Defaults.MIN_GRAM, Defaults.MAX_GRAM)); fieldType.setShingleFields(new ShingleFieldType[] { new ShingleFieldType(fieldType.name(), 2, TextSearchInfo.SIMPLE_MATCH_ONLY) @@ -62,7 +60,7 @@ protected SearchAsYouTypeFieldType createDefaultFieldType(String name, Map meta) { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected JoinFieldType(JoinFieldType ref) { - super(ref); - } - - public JoinFieldType clone() { - return new JoinFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java b/modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java similarity index 100% rename from modules/parent-join/src/test/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java rename to modules/parent-join/src/yamlRestTest/java/org/elasticsearch/join/ParentChildClientYamlTestSuiteIT.java diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/11_parent_child.yml similarity index 100% rename from modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml rename to modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/11_parent_child.yml diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/20_parent_join.yml similarity index 100% rename from modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml rename to modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/20_parent_join.yml diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/30_inner_hits.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml similarity index 100% rename from modules/parent-join/src/test/resources/rest-api-spec/test/30_inner_hits.yml rename to modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml diff --git a/modules/percolator/build.gradle b/modules/percolator/build.gradle index 917e44fd21572..ce1e68025a044 100644 --- a/modules/percolator/build.gradle +++ b/modules/percolator/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'Percolator module adds capability to index queries and query these queries by specifying documents' @@ -45,3 +45,5 @@ restResources { includeCore '_common', 'indices', 'index', 'search', 'msearch' } } + +integTest.enabled = false diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 13836c993f464..0df62e260c1f1 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -155,10 +155,7 @@ static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderCon } static BinaryFieldMapper createQueryBuilderFieldBuilder(BuilderContext context) { - BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(QUERY_BUILDER_FIELD_NAME); - builder.docValues(true); - builder.indexOptions(IndexOptions.NONE); - builder.store(false); + BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(QUERY_BUILDER_FIELD_NAME, true); return builder.build(context); } @@ -202,21 +199,6 @@ static class PercolatorFieldType extends MappedFieldType { super(name, false, false, TextSearchInfo.NONE, meta); } - PercolatorFieldType(PercolatorFieldType ref) { - super(ref); - queryTermsField = ref.queryTermsField; - extractionResultField = ref.extractionResultField; - queryBuilderField = ref.queryBuilderField; - rangeField = ref.rangeField; - minimumShouldMatchField = ref.minimumShouldMatchField; - mapUnmappedFieldsAsText = ref.mapUnmappedFieldsAsText; - } - - @Override - public MappedFieldType clone() { - return new PercolatorFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java similarity index 100% rename from modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java rename to modules/percolator/src/yamlRestTest/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml b/modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml similarity index 100% rename from modules/percolator/src/test/resources/rest-api-spec/test/10_basic.yml rename to modules/percolator/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml diff --git a/modules/rank-eval/build.gradle b/modules/rank-eval/build.gradle index e66633b4d3b88..2bf54796abf2d 100644 --- a/modules/rank-eval/build.gradle +++ b/modules/rank-eval/build.gradle @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'The Rank Eval module adds APIs to evaluate ranking quality.' @@ -29,7 +29,7 @@ restResources { } } -testClusters.integTest { +testClusters.all { // Modules who's integration is explicitly tested in integration tests module project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/50_data_streams.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/50_data_streams.yml deleted file mode 100644 index 4470c34e605c7..0000000000000 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/50_data_streams.yml +++ /dev/null @@ -1,100 +0,0 @@ -"Verify rank eval with data streams": - - skip: - version: " - 7.99.99" - reason: "change to 7.8.99 after backport" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" - indices.put_index_template: - name: my-template - body: - index_patterns: [logs-*] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: logs-foobar - - is_true: acknowledged - - - do: - index: - index: logs-foobar - id: doc1 - op_type: create - body: { "text": "berlin", "@timestamp": "2020-01-01" } - - - do: - index: - index: logs-foobar - id: doc2 - op_type: create - body: { "text": "amsterdam", "@timestamp": "2020-01-01" } - - # rollover data stream to split documents across multiple backing indices - - do: - indices.rollover: - alias: "logs-foobar" - - - match: { old_index: .ds-logs-foobar-000001 } - - match: { new_index: .ds-logs-foobar-000002 } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - index: - index: logs-foobar - id: doc3 - op_type: create - body: { "text": "amsterdam", "@timestamp": "2020-01-01" } - - - do: - index: - index: logs-foobar - id: doc4 - op_type: create - body: { "text": "something about amsterdam and berlin", "@timestamp": "2020-01-01" } - - - do: - indices.refresh: - index: logs-foobar - - - do: - rank_eval: - index: logs-foobar - search_type: query_then_fetch - body: { - "requests" : [ - { - "id": "amsterdam_query", - "request": { "query": { "match" : {"text" : "amsterdam" }}}, - "ratings": [ - {"_index": ".ds-logs-foobar-000001", "_id": "doc1", "rating": 0}, - {"_index": ".ds-logs-foobar-000001", "_id": "doc2", "rating": 1}, - {"_index": ".ds-logs-foobar-000002", "_id": "doc3", "rating": 1}] - }, - { - "id" : "berlin_query", - "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"_index": ".ds-logs-foobar-000001", "_id": "doc1", "rating": 1}] - } - ], - "metric" : { "precision": { "ignore_unlabeled" : true }} - } - - - match: { metric_score: 1} - - match: { details.amsterdam_query.metric_score: 1.0} - - length: { details.amsterdam_query.hits: 3} - - match: { details.berlin_query.metric_score: 1.0} - - - do: - indices.delete_data_stream: - name: logs-foobar - - is_true: acknowledged diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java b/modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java similarity index 100% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java rename to modules/rank-eval/src/yamlRestTest/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/10_basic.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml rename to modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/10_basic.yml diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/20_dcg.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml rename to modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/20_dcg.yml diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml rename to modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/30_failures.yml diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml b/modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml similarity index 100% rename from modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml rename to modules/rank-eval/src/yamlRestTest/resources/rest-api-spec/test/rank_eval/40_rank_eval_templated.yml diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 7bc1c66798cb8..3d5d745255293 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -24,14 +24,14 @@ import org.elasticsearch.gradle.info.BuildParams apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.jdk-download' -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' esplugin { description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.' classname 'org.elasticsearch.index.reindex.ReindexPlugin' } -testClusters.integTest { +testClusters.all { // Modules who's integration is explicitly tested in integration tests module project(':modules:parent-join').tasks.bundlePlugin.archiveFile module project(':modules:lang-painless').tasks.bundlePlugin.archiveFile diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java similarity index 100% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java rename to modules/reindex/src/yamlRestTest/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/10_basic.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/20_validation.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/20_validation.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/40_versioning.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/40_versioning.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/50_wait_for_active_shards.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/70_throttle.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/80_slices.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/delete_by_query/80_slices.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/10_basic.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/10_basic.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/20_validation.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/20_validation.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/25_no_auto_create.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/25_no_auto_create.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/25_no_auto_create.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/30_search.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/30_search.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/30_search.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/35_search_failures.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/35_search_failures.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/40_versioning.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/40_versioning.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/40_versioning.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/50_routing.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/50_routing.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/50_routing.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/60_wait_for_active_shards.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/70_throttle.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/70_throttle.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_slices.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/80_slices.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/85_scripting.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/85_scripting.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/85_scripting.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/90_remote.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/90_remote.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/95_parent_join.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/95_parent_join.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/95_parent_join.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/10_basic.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/20_validation.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/20_validation.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/30_new_fields.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/30_new_fields.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/30_new_fields.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/35_search_failure.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/35_search_failure.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/40_versioning.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/40_versioning.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/50_consistency.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/50_consistency.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/50_consistency.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/60_throttle.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/60_throttle.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_slices.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/70_slices.yml diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/80_scripting.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/80_scripting.yml rename to modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/80_scripting.yml diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 9eaf68b7641a2..6f2dff252d9cc 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -21,7 +21,9 @@ import org.elasticsearch.gradle.PropertyNormalization import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.AntFixture -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' + esplugin { description 'Module for URL repository' @@ -34,6 +36,8 @@ restResources { } } +integTest.enabled = false + // This directory is shared between two URL repositories and one FS repository in YAML integration tests File repositoryDir = new File(project.buildDir, "shared-repository") @@ -47,11 +51,14 @@ task urlFixture(type: AntFixture) { executable = "${BuildParams.runtimeJavaHome}/bin/java" args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}" } +yamlRestTest { + dependsOn urlFixture +} -integTest { +internalClusterTest { dependsOn urlFixture } -testClusters.integTest { +testClusters.all { // repositoryDir is used by a FS repository to create snapshots setting 'path.repo', "${repositoryDir.absolutePath}", PropertyNormalization.IGNORE_VALUE // repositoryDir is used by two URL repositories to restore snapshots @@ -59,3 +66,4 @@ testClusters.integTest { "http://snapshot.test*,http://${urlFixture.addressAndPort}" }, PropertyNormalization.IGNORE_VALUE } + diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java b/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java similarity index 98% rename from modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java rename to modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java index 2cc1d0992fafc..2dde97a1884e4 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLSnapshotRestoreTests.java +++ b/modules/repository-url/src/internalClusterTest/java/org/elasticsearch/repositories/url/URLSnapshotRestoreIT.java @@ -42,7 +42,7 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) -public class URLSnapshotRestoreTests extends ESIntegTestCase { +public class URLSnapshotRestoreIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java similarity index 99% rename from modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java rename to modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index 65d9b87b07d4c..8080238ee7bd3 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/yamlRestTest/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -93,7 +93,7 @@ public void registerRepositories() throws IOException { // Create a URL repository using the file://{path.repo} URL Request createFileRepositoryRequest = new Request("PUT", "/_snapshot/repository-file"); - createFileRepositoryRequest.setEntity(buildRepositorySettings(URLRepository.TYPE, + createFileRepositoryRequest.setEntity(buildRepositorySettings("url", Settings.builder().put("url", pathRepoUri.toString()).build())); Response createFileRepositoryResponse = client().performRequest(createFileRepositoryRequest); assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); @@ -106,7 +106,7 @@ public void registerRepositories() throws IOException { InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost()); if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) { Request createUrlRepositoryRequest = new Request("PUT", "/_snapshot/repository-url"); - createUrlRepositoryRequest.setEntity(buildRepositorySettings(URLRepository.TYPE, + createUrlRepositoryRequest.setEntity(buildRepositorySettings("url", Settings.builder().put("url", allowedUrl).build())); Response createUrlRepositoryResponse = client().performRequest(createUrlRepositoryRequest); assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml similarity index 100% rename from modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/10_basic.yml rename to modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/10_basic.yml diff --git a/modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yml b/modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/20_repository.yml similarity index 100% rename from modules/repository-url/src/test/resources/rest-api-spec/test/repository_url/20_repository.yml rename to modules/repository-url/src/yamlRestTest/resources/rest-api-spec/test/repository_url/20_repository.yml diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/compat/RestCompatPlugin.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/compat/RestCompatPlugin.java index 5cc62a1484047..e846c925166d4 100644 --- a/modules/rest-compatibility/src/main/java/org/elasticsearch/compat/RestCompatPlugin.java +++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/compat/RestCompatPlugin.java @@ -7,7 +7,7 @@ * not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -46,6 +46,7 @@ import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.function.Supplier; public class RestCompatPlugin extends Plugin implements ActionPlugin { @@ -61,7 +62,8 @@ public List getRestHandlers( Supplier nodesInCluster ) { if (Version.CURRENT.major == 8) { - return validatedList( + return validateCompatibleHandlers( + 7, new RestDeleteByQueryActionV7(), new RestUpdateByQueryActionV7(), new RestCreateIndexActionV7(), @@ -82,11 +84,19 @@ public List getRestHandlers( return Collections.emptyList(); } - private List validatedList(RestHandler... handlers) { - List handlers1 = List.of(handlers); + // default scope for testing + List validateCompatibleHandlers(int expectedVersion, RestHandler... handlers) { for (RestHandler handler : handlers) { - assert handler.compatibleWithVersion().major == Version.CURRENT.major - 1 : "Handler is of incorrect version. " + handler; + if (handler.compatibleWithVersion().major != expectedVersion) { + String msg = String.format( + Locale.ROOT, + "Handler %s is of incorrect version %s.", + handler.getClass().getSimpleName(), + handler.compatibleWithVersion() + ); + throw new IllegalStateException(msg); + } } - return handlers1; + return List.of(handlers); } } diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionV7.java index 463866af9d894..26861e0b5cd1c 100644 --- a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionV7.java +++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionV7.java @@ -42,6 +42,7 @@ public class RestCreateIndexActionV7 extends RestCreateIndexAction { * Parameter that controls whether certain REST apis should include type names in their requests. */ public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; + public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; @Override public String getName() { diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingActionV7.java new file mode 100644 index 0000000000000..71817de9c5345 --- /dev/null +++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingActionV7.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.rest.RestRequest; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestGetFieldMappingActionV7 extends RestGetFieldMappingAction { + + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetFieldMappingActionV7.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + "field mapping requests is deprecated. The parameter will be removed in the next major version."; + + @Override + public List routes() { + return List.of( + new Route(GET, "/_mapping/field/{fields}"), + new Route(GET, "/{index}/_mapping/field/{fields}"), + + new Route(GET, "/_mapping/{type}/field/{fields}"), + new Route(GET, "/{index}/{type}/_mapping/field/{fields}"), + new Route(GET, "/{index}/_mapping/{type}/field/{fields}") + ); + } + + @Override + public String getName() { + return "get_field_mapping_action_v7"; + } + + @Override + public Version compatibleWithVersion() { + return Version.V_7_0_0; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); + + boolean includeTypeName = request.paramAsBoolean( + RestCreateIndexActionV7.INCLUDE_TYPE_NAME_PARAMETER, + RestCreateIndexActionV7.DEFAULT_INCLUDE_TYPE_NAME_POLICY + ); + if (includeTypeName == false && types.length > 0) { + throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + " is set to true."); + } + if (request.hasParam(RestCreateIndexActionV7.INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(RestCreateIndexActionV7.INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.deprecate("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + // todo compatible log about using INCLUDE_TYPE_NAME_PARAMETER + } + if (types.length > 0) { + // todo compatible log about using types in path + } + if (request.hasParam("local")) { + request.param("local"); + deprecationLogger.deprecate( + "get_field_mapping_local", + "Use [local] in get field mapping requests is deprecated. " + "The parameter will be removed in the next major version" + ); + } + return super.prepareRequest(request, client); + } +} diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingActionV7.java new file mode 100644 index 0000000000000..17a23a488dccf --- /dev/null +++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingActionV7.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.rest.RestRequest; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.HEAD; +import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexActionV7.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexActionV7.INCLUDE_TYPE_NAME_PARAMETER; + +public class RestGetMappingActionV7 extends RestGetMappingAction { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingActionV7.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get" + + " mapping requests is deprecated. The parameter will be removed in the next major version."; + + @Override + public List routes() { + return List.of( + new Route(GET, "/{index}/{type}/_mapping"), + new Route(GET, "/{index}/_mappings/{type}"), + new Route(GET, "/{index}/_mapping/{type}"), + new Route(HEAD, "/{index}/_mapping/{type}"), + new Route(GET, "/_mapping/{type}"), + new Route(GET, "/_mapping"), + new Route(GET, "/_mappings"), + new Route(GET, "/{index}/_mapping"), + new Route(GET, "/{index}/_mappings") + ); + } + + @Override + public String getName() { + return "get_mapping_action_v7"; + } + + @Override + public Version compatibleWithVersion() { + return Version.V_7_0_0; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + + if (request.method().equals(HEAD)) { + deprecationLogger.deprecate("get_mapping_with_types", "Type exists requests are deprecated, as types have been deprecated."); + } else if (includeTypeName == false && types.length > 0) { + throw new IllegalArgumentException( + "Types cannot be provided in get mapping requests, unless" + " include_type_name is set to true." + ); + } + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + deprecationLogger.deprecate("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + if (types.length > 0) { + // todo compatible log about using types in path + } + return super.prepareRequest(request, client); + } +} diff --git a/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingActionV7.java b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingActionV7.java new file mode 100644 index 0000000000000..ace1bbf3b548d --- /dev/null +++ b/modules/rest-compatibility/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingActionV7.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.RestRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; +import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexActionV7.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.action.admin.indices.RestCreateIndexActionV7.INCLUDE_TYPE_NAME_PARAMETER; + +public class RestPutMappingActionV7 extends RestPutMappingAction { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutMappingActionV7.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in put " + + "mapping requests is deprecated. The parameter will be removed in the next major version."; + + @Override + public List routes() { + return List.of( + new Route(PUT, "/{index}/{type}/_mapping"), + new Route(PUT, "/{index}/_mapping/{type}"), + new Route(PUT, "/_mapping/{type}"), + + new Route(POST, "/{index}/{type}/_mapping"), + new Route(POST, "/{index}/_mapping/{type}"), + new Route(POST, "/_mapping/{type}"), + + // register the same paths, but with plural form _mappings + new Route(PUT, "/{index}/{type}/_mappings"), + new Route(PUT, "/{index}/_mappings/{type}"), + new Route(PUT, "/_mappings/{type}"), + + new Route(POST, "/{index}/{type}/_mappings"), + new Route(POST, "/{index}/_mappings/{type}"), + new Route(POST, "/_mappings/{type}"), + + // no types in path, but type can be provided in body + new Route(POST, "/{index}/_mapping/"), + new Route(PUT, "/{index}/_mapping/"), + new Route(POST, "/{index}/_mappings/"), + new Route(PUT, "/{index}/_mappings/") + ); + } + + @Override + public String getName() { + return "put_mapping_action_v7"; + } + + @Override + public Version compatibleWithVersion() { + return Version.V_7_0_0; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + + String type = request.param("type"); + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + if (includeTypeName == false + && (type != null || MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap))) { + throw new IllegalArgumentException( + "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." + ); + } + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecate("put_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + if (includeTypeName) { + sourceAsMap = prepareMappingsV7(sourceAsMap, request); + } + return super.sendPutMappingRequest(request, client, sourceAsMap); + } + + private Map prepareMappingsV7(Map mappings, RestRequest request) { + String typeName = mappings.keySet().iterator().next(); + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + + // no matter what the type was, replace it with _doc, because the internal representation still uses single type `_doc`. + return Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings); + } +} diff --git a/modules/rest-compatibility/src/test/java/org/elasticsearch/compat/RestCompatPluginTests.java b/modules/rest-compatibility/src/test/java/org/elasticsearch/compat/RestCompatPluginTests.java new file mode 100644 index 0000000000000..212ab690ad0b7 --- /dev/null +++ b/modules/rest-compatibility/src/test/java/org/elasticsearch/compat/RestCompatPluginTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.compat; + +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.List; + +public class RestCompatPluginTests extends ESTestCase { + + public void testRestHandlersValidation() { + RestCompatPlugin restCompatPlugin = new RestCompatPlugin(); + Version prevVersion = Version.fromString("7.0.0"); + expectThrows( + IllegalStateException.class, + () -> restCompatPlugin.validateCompatibleHandlers(7, restHandler(prevVersion), restHandler(Version.fromString("8.0.0"))) + ); + + List restHandlers = restCompatPlugin.validateCompatibleHandlers(7, restHandler(prevVersion), restHandler(prevVersion)); + assertThat(restHandlers, Matchers.hasSize(2)); + } + + private RestHandler restHandler(final Version version) { + return new RestHandler() { + @Override + public Version compatibleWithVersion() { + return version; + } + + @Override + public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { + + } + }; + } +} diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index eebbf16064acd..d91ccd9c1bdbf 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -21,7 +21,7 @@ import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.test.RestIntegTestTask -apply plugin: 'elasticsearch.rest-resources' +apply plugin: 'elasticsearch.yaml-rest-test' /* TODOs: diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java similarity index 100% rename from modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java rename to modules/transport-netty4/src/yamlRestTest/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java diff --git a/modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yml b/modules/transport-netty4/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml similarity index 100% rename from modules/transport-netty4/src/test/resources/rest-api-spec/test/10_basic.yml rename to modules/transport-netty4/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index dcfaf2548e053..108cbc719e08a 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -86,26 +86,6 @@ public CollationFieldType(String name, Collator collator) { this(name, true, true, collator, Collections.emptyMap()); } - protected CollationFieldType(CollationFieldType ref) { - super(ref); - this.collator = ref.collator; - } - - @Override - public CollationFieldType clone() { - return new CollationFieldType(this); - } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(collator, ((CollationFieldType) o).collator); - } - - @Override - public int hashCode() { - return 31 * super.hashCode() + Objects.hashCode(collator); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java index 5e91524ee8282..c1c86bbb8f858 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/CollationFieldTypeTests.java @@ -37,17 +37,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; -public class CollationFieldTypeTests extends FieldTypeTestCase { +public class CollationFieldTypeTests extends FieldTypeTestCase{ private static final Collator DEFAULT_COLLATOR = Collator.getInstance(ULocale.ROOT).freeze(); - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new CollationFieldType(name, true, true, DEFAULT_COLLATOR, meta); - } - public void testIsFieldWithinQuery() throws IOException { CollationFieldType ft = new CollationFieldType("field", DEFAULT_COLLATOR); // current impl ignores args and shourd always return INTERSECTS diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 0ba08a21295e8..94fd537f44846 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -521,10 +521,6 @@ public AnnotatedTextFieldType(String name, Map meta) { super(name, true, meta); } - protected AnnotatedTextFieldType(AnnotatedTextFieldType ref) { - super(ref); - } - public void setIndexAnalyzer(NamedAnalyzer delegate, int positionIncrementGap) { if(delegate.analyzer() instanceof AnnotationAnalyzerWrapper){ // Already wrapped the Analyzer with an AnnotationAnalyzer @@ -536,10 +532,6 @@ public void setIndexAnalyzer(NamedAnalyzer delegate, int positionIncrementGap) { } } - public AnnotatedTextFieldType clone() { - return new AnnotatedTextFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java index f00518a373ebb..3671c21678b96 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java @@ -29,16 +29,11 @@ import java.io.IOException; import java.util.Collections; -import java.util.Map; -public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new AnnotatedTextFieldMapper.AnnotatedTextFieldType(name, meta); - } +public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase { public void testIntervals() throws IOException { - MappedFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + MappedFieldType ft = new AnnotatedTextFieldMapper.AnnotatedTextFieldType("field", Collections.emptyMap()); NamedAnalyzer a = new NamedAnalyzer("name", AnalyzerScope.INDEX, new StandardAnalyzer()); IntervalsSource source = ft.intervals("Donald Trump", 0, true, a, false); assertEquals(Intervals.phrase(Intervals.term("donald"), Intervals.term("trump")), source); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 0dd9b5fe76fc2..4be96ad4be03a 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -96,20 +96,11 @@ public Murmur3FieldType(String name, Map meta) { super(name, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } - protected Murmur3FieldType(Murmur3FieldType ref) { - super(ref); - } - @Override public String typeName() { return CONTENT_TYPE; } - @Override - public Murmur3FieldType clone() { - return new Murmur3FieldType(this); - } - @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 58d3bf21594ab..51032981a9eef 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -249,7 +249,7 @@ public void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryS @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { if (SnapshotsService.useShardGenerations(repositoryMetaVersion) == false) { listener = delayedListener(listener); } diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 6178ed53e5da3..8fa8763d8027f 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -609,7 +609,7 @@ void assertTotalHits(int expectedTotalHits, Map response) { assertEquals(response.toString(), expectedTotalHits, actualTotalHits); } - int extractTotalHits(Map response) { + static int extractTotalHits(Map response) { return (Integer) XContentMapValues.extractValue("hits.total.value", response); } @@ -1385,11 +1385,12 @@ public void testResize() throws Exception { } } - private void assertNumHits(String index, int numHits, int totalShards) throws IOException { + public static void assertNumHits(String index, int numHits, int totalShards) throws IOException { Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); assertNoFailures(resp); assertThat(XContentMapValues.extractValue("_shards.total", resp), equalTo(totalShards)); assertThat(XContentMapValues.extractValue("_shards.successful", resp), equalTo(totalShards)); assertThat(extractTotalHits(resp), equalTo(numHits)); } + } diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml index 455ada0c1ec1e..8f9b0ce1545b3 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml @@ -3,47 +3,6 @@ - skip: features: allowed_warnings - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - do: - allowed_warnings: - - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" - indices.put_index_template: - name: my-template2 - body: - index_patterns: [simple-data-stream2] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - - do: - indices.create_data_stream: - name: simple-data-stream2 - - - do: - indices.rollover: - alias: "simple-data-stream2" - - do: indices.create: index: single_doc_index @@ -123,18 +82,6 @@ nested2: type: keyword doc_values: false - - - do: - indices.create: - index: closed_index - body: - aliases: - aliased_closed_index: {} - - - do: - indices.close: - index: closed_index - - do: indices.create: index: test_index diff --git a/qa/rest-compat-tests/build.gradle b/qa/rest-compat-tests/build.gradle index 056058ecf794f..78fac721fa843 100644 --- a/qa/rest-compat-tests/build.gradle +++ b/qa/rest-compat-tests/build.gradle @@ -15,6 +15,8 @@ task copyRestTestsResources(type: Copy) { dependsOn ':distribution:bwc:minor:checkoutBwcBranch' } +//copyOverrides.dependsOn(copyRestTestsResources) +processTestResources.dependsOn(copyRestTestsResources) integTest.dependsOn(copyRestTestsResources) dependencies { diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/11_basic_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/11_basic_with_types.yml new file mode 100644 index 0000000000000..4d62b24184e56 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/11_basic_with_types.yml @@ -0,0 +1,85 @@ +--- +setup: + - do: + indices.create: + include_type_name: true + index: test_index + body: + mappings: + test_type: + properties: + text: + type: text + +--- +"Get field mapping with no index and type": + + - do: + indices.get_field_mapping: + include_type_name: true + fields: text + #- match: {test_index.mappings.test_type.text.mapping.text.type: text} + - match: {test_index.mappings._doc.text.mapping.text.type: text} + +--- +"Get field mapping by index only": + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + fields: text +# - match: {test_index.mappings.test_type.text.mapping.text.type: text} + - match: {test_index.mappings._doc.text.mapping.text.type: text} + +--- +"Get field mapping by type & field": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + type: test_type + fields: text +# - match: {test_index.mappings.test_type.text.mapping.text.type: text} + - match: {test_index.mappings._doc.text.mapping.text.type: text} + +--- +"Get field mapping by type & field, with another field that doesn't exist": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + type: test_type + fields: [ text , text1 ] +#- match: {test_index.mappings.test_type.text.mapping.text.type: text} +# - is_false: test_index.mappings.test_type.text1 + - match: {test_index.mappings._doc.text.mapping.text.type: text} + - is_false: test_index.mappings._doc.text1 + +--- +"Get field mapping with include_defaults": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + type: test_type + fields: text + include_defaults: true +#- match: {test_index.mappings.test_type.text.mapping.text.type: text} +# - match: {test_index.mappings.test_type.text.mapping.text.analyzer: default} + - match: {test_index.mappings._doc.text.mapping.text.type: text} + - match: {test_index.mappings._doc.text.mapping.text.analyzer: default} + +--- +"Get field mapping should work without index specifying type and fields": + + - do: + indices.get_field_mapping: + include_type_name: true + type: test_type + fields: text +# - match: {test_index.mappings.test_type.text.mapping.text.type: text} + - match: {test_index.mappings._doc.text.mapping.text.type: text} + diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/21_missing_field_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/21_missing_field_with_types.yml new file mode 100644 index 0000000000000..da8588b1ba7d6 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/21_missing_field_with_types.yml @@ -0,0 +1,23 @@ +--- +"Return empty object if field doesn't exist, but type and index do": + + - do: + indices.create: + include_type_name: true + index: test_index + body: + mappings: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + type: test_type + fields: not_existent +# - match: { '': {}} + - match: { 'test_index.mappings': { '_doc':{}}} diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/51_field_wildcards_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/51_field_wildcards_with_types.yml new file mode 100644 index 0000000000000..f7943439cce2b --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_field_mapping/51_field_wildcards_with_types.yml @@ -0,0 +1,181 @@ +--- +setup: + - do: + indices.create: + include_type_name: true + index: test_index + body: + mappings: + test_type: + properties: + t1: + type: text + t2: + type: text + obj: + properties: + t1: + type: text + i_t1: + type: text + i_t3: + type: text + + - do: + indices.create: + include_type_name: true + index: test_index_2 + body: + mappings: + test_type_2: + properties: + t1: + type: text + t2: + type: text + obj: + properties: + t1: + type: text + i_t1: + type: text + i_t3: + type: text + +--- +"Get field mapping with * for fields": + + - do: + indices.get_field_mapping: + include_type_name: true + fields: "*" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.t2.full_name: t2 } + #- match: {test_index.mappings.test_type.obj\.t1.full_name: obj.t1 } + #- match: {test_index.mappings.test_type.obj\.i_t1.full_name: obj.i_t1 } + #- match: {test_index.mappings.test_type.obj\.i_t3.full_name: obj.i_t3 } + + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.t2.full_name: t2 } + - match: {test_index.mappings._doc.obj\.t1.full_name: obj.t1 } + - match: {test_index.mappings._doc.obj\.i_t1.full_name: obj.i_t1 } + - match: {test_index.mappings._doc.obj\.i_t3.full_name: obj.i_t3 } + +--- +"Get field mapping with t* for fields": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + fields: "t*" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.t2.full_name: t2 } + #- length: {test_index.mappings.test_type: 2} + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.t2.full_name: t2 } + - length: {test_index.mappings._doc: 2} + +--- +"Get field mapping with *t1 for fields": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + fields: "*t1" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.obj\.t1.full_name: obj.t1 } + #- match: {test_index.mappings.test_type.obj\.i_t1.full_name: obj.i_t1 } + #- length: {test_index.mappings.test_type: 3} + + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.obj\.t1.full_name: obj.t1 } + - match: {test_index.mappings._doc.obj\.i_t1.full_name: obj.i_t1 } + - length: {test_index.mappings._doc: 3} + +--- +"Get field mapping with wildcarded relative names": + + - do: + indices.get_field_mapping: + include_type_name: true + index: test_index + fields: "obj.i_*" + #- match: {test_index.mappings.test_type.obj\.i_t1.full_name: obj.i_t1 } + #- match: {test_index.mappings.test_type.obj\.i_t3.full_name: obj.i_t3 } + #- length: {test_index.mappings.test_type: 2} + + - match: {test_index.mappings._doc.obj\.i_t1.full_name: obj.i_t1 } + - match: {test_index.mappings._doc.obj\.i_t3.full_name: obj.i_t3 } + - length: {test_index.mappings._doc: 2} + +--- +"Get field mapping should work using '_all' for indices and types": + + - do: + indices.get_field_mapping: + include_type_name: true + index: _all + type: _all + fields: "t*" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.t2.full_name: t2 } + #- length: {test_index.mappings.test_type: 2} + #- match: {test_index_2.mappings.test_type_2.t1.full_name: t1 } + #- match: {test_index_2.mappings.test_type_2.t2.full_name: t2 } + #- length: {test_index_2.mappings.test_type_2: 2} + + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.t2.full_name: t2 } + - length: {test_index.mappings._doc: 2} + - match: {test_index_2.mappings._doc.t1.full_name: t1 } + - match: {test_index_2.mappings._doc.t2.full_name: t2 } + - length: {test_index_2.mappings._doc: 2} + +--- +"Get field mapping should work using '*' for indices and types": + + - do: + indices.get_field_mapping: + include_type_name: true + index: '*' + type: '*' + fields: "t*" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.t2.full_name: t2 } + #- length: {test_index.mappings.test_type: 2} + #- match: {test_index_2.mappings.test_type_2.t1.full_name: t1 } + #- match: {test_index_2.mappings.test_type_2.t2.full_name: t2 } + #- length: {test_index_2.mappings.test_type_2: 2} + + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.t2.full_name: t2 } + - length: {test_index.mappings._doc: 2} + - match: {test_index_2.mappings._doc.t1.full_name: t1 } + - match: {test_index_2.mappings._doc.t2.full_name: t2 } + - length: {test_index_2.mappings._doc: 2} + +--- +"Get field mapping should work using comma_separated values for indices and types": + + - do: + indices.get_field_mapping: + include_type_name: true + index: 'test_index,test_index_2' + type: 'test_type,test_type_2' + fields: "t*" + #- match: {test_index.mappings.test_type.t1.full_name: t1 } + #- match: {test_index.mappings.test_type.t2.full_name: t2 } + #- length: {test_index.mappings.test_type: 2} + #- match: {test_index_2.mappings.test_type_2.t1.full_name: t1 } + #- match: {test_index_2.mappings.test_type_2.t2.full_name: t2 } + #- length: {test_index_2.mappings.test_type_2: 2} + + - match: {test_index.mappings._doc.t1.full_name: t1 } + - match: {test_index.mappings._doc.t2.full_name: t2 } + - length: {test_index.mappings._doc: 2} + - match: {test_index_2.mappings._doc.t1.full_name: t1 } + - match: {test_index_2.mappings._doc.t2.full_name: t2 } + - length: {test_index_2.mappings._doc: 2} + diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_mapping/11_basic_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_mapping/11_basic_with_types.yml new file mode 100644 index 0000000000000..188fb174af9fe --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.get_mapping/11_basic_with_types.yml @@ -0,0 +1,175 @@ +--- +setup: + - do: + indices.create: + include_type_name: true + index: test_1 + body: + mappings: + doc: {} + - do: + indices.create: + include_type_name: true + index: test_2 + body: + mappings: + doc: {} +--- +"Get /{index}/_mapping with empty mappings": +#this is not working atm. index created with a type does not store that information +# hence there is no difference in this scenario and the test below. + - do: + indices.create: + index: t + + - do: + indices.get_mapping: + include_type_name: true + index: t + + - match: { t.mappings: {}} + +--- +"Get /_mapping": + + - do: + indices.get_mapping: + include_type_name: true + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + +--- +"Get /{index}/_mapping": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1 + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc + - is_false: test_2 + + +--- +"Get /{index}/_mapping/_all": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1 + type: _all + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc + - is_false: test_2 + +--- +"Get /{index}/_mapping/*": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1 + type: '*' + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc + - is_false: test_2 + +--- +"Get /{index}/_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1 + type: doc + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc + - is_false: test_2 + +--- +"Get /{index}/_mapping/{type*}": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1 + type: 'd*' + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc + - is_false: test_2 + +--- +"Get /_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + type: doc + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + +--- +"Get /_all/_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + index: _all + type: doc + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + +--- +"Get /*/_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + index: '*' + type: doc + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + +--- +"Get /index,index/_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + index: test_1,test_2 + type: doc + +# - is_true: test_1.mappings.doc + - is_true: test_1.mappings._doc +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + +--- +"Get /index*/_mapping/{type}": + + - do: + indices.get_mapping: + include_type_name: true + index: '*2' + type: doc + +# - is_true: test_2.mappings.doc + - is_true: test_2.mappings._doc + - is_false: test_1 diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/11_basic_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/11_basic_with_types.yml new file mode 100644 index 0000000000000..6b16b01b465e3 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/11_basic_with_types.yml @@ -0,0 +1,79 @@ +--- +"Test Create and update mapping": + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + include_type_name: true + index: test_index + type: test_type + body: + test_type: + properties: + text1: + type: text + analyzer: whitespace + text2: + type: text + analyzer: whitespace + subfield.text3: + type: text + + - do: + indices.get_mapping: + include_type_name: true + index: test_index + #- match: {test_index.mappings.test_type.properties.text1.type: text} + #- match: {test_index.mappings.test_type.properties.text1.analyzer: whitespace} + #- match: {test_index.mappings.test_type.properties.text2.type: text} + #- match: {test_index.mappings.test_type.properties.text2.analyzer: whitespace} + - match: {test_index.mappings._doc.properties.text1.type: text} + - match: {test_index.mappings._doc.properties.text1.analyzer: whitespace} + - match: {test_index.mappings._doc.properties.text2.type: text} + - match: {test_index.mappings._doc.properties.text2.analyzer: whitespace} + + - do: + indices.put_mapping: + include_type_name: true + index: test_index + type: test_type + body: + test_type: + properties: + text1: + type: text + analyzer: whitespace + fields: + text_raw: + type: keyword + + + - do: + indices.get_mapping: + include_type_name: true + index: test_index + #- match: {test_index.mappings.test_type.properties.text1.type: text} + # - match: {test_index.mappings.test_type.properties.subfield.properties.text3.type: text} + # - match: {test_index.mappings.test_type.properties.text1.fields.text_raw.type: keyword} + - match: {test_index.mappings._doc.properties.text1.type: text} + - match: {test_index.mappings._doc.properties.subfield.properties.text3.type: text} + - match: {test_index.mappings._doc.properties.text1.fields.text_raw.type: keyword} + +--- +"Create index with invalid mappings": + - do: + indices.create: + index: test_index + - do: + catch: /illegal_argument_exception/ + indices.put_mapping: + include_type_name: true + index: test_index + type: test_type + body: + test_type: + properties: + "": + type: keyword diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/all_path_options_with_types.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/all_path_options_with_types.yml new file mode 100644 index 0000000000000..b2ad5ef665f39 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/indices.put_mapping/all_path_options_with_types.yml @@ -0,0 +1,263 @@ +setup: + - do: + indices.create: + index: test_index1 + - do: + indices.create: + index: test_index2 + - do: + indices.create: + index: foo + + +--- +"put one mapping per index": + - do: + indices.put_mapping: + include_type_name: true + index: test_index1 + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + - do: + indices.put_mapping: + include_type_name: true + index: test_index2 + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + +# - match: { foo.mappings: {} } + - match: { 'foo.mappings': { '_doc':{}}} + +--- +"put mapping in _all index": + + - do: + indices.put_mapping: + include_type_name: true + index: _all + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {foo.mappings.test_type.properties.text.type: text} +# - match: {foo.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + + - match: {foo.mappings._doc.properties.text.type: text} + - match: {foo.mappings._doc.properties.text.analyzer: whitespace} + +--- +"put mapping in * index": + - do: + indices.put_mapping: + include_type_name: true + index: "*" + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {foo.mappings.test_type.properties.text.type: text} +# - match: {foo.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + + - match: {foo.mappings._doc.properties.text.type: text} + - match: {foo.mappings._doc.properties.text.analyzer: whitespace} + +--- +"put mapping in prefix* index": + - do: + indices.put_mapping: + include_type_name: true + index: "test_index*" + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + +# - match: { foo.mappings: {} } + - match: { 'foo.mappings': { '_doc':{}}} + +--- +"put mapping in list of indices": + - do: + indices.put_mapping: + include_type_name: true + index: [test_index1, test_index2] + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + +# - match: { foo.mappings: {} } + - match: { 'foo.mappings': { '_doc':{}}} + +--- +"put mapping with blank index": + - do: + indices.put_mapping: + include_type_name: true + type: test_type + body: + test_type: + properties: + text: + type: text + analyzer: whitespace + + - do: + indices.get_mapping: + include_type_name: true +#- match: {test_index1.mappings.test_type.properties.text.type: text} +# - match: {test_index1.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {test_index2.mappings.test_type.properties.text.type: text} +# - match: {test_index2.mappings.test_type.properties.text.analyzer: whitespace} +# +# - match: {foo.mappings.test_type.properties.text.type: text} +# - match: {foo.mappings.test_type.properties.text.analyzer: whitespace} + - match: {test_index1.mappings._doc.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.analyzer: whitespace} + + - match: {test_index2.mappings._doc.properties.text.type: text} + - match: {test_index2.mappings._doc.properties.text.analyzer: whitespace} + + - match: {foo.mappings._doc.properties.text.type: text} + - match: {foo.mappings._doc.properties.text.analyzer: whitespace} + +--- +"put mapping with missing type": + + + - do: + catch: param + indices.put_mapping: + include_type_name: true + +--- +"post a mapping with default analyzer twice": + + - do: + indices.put_mapping: + include_type_name: true + index: test_index1 + type: test_type + body: + test_type: + dynamic: false + properties: + text: + analyzer: default + type: text + + - do: + indices.put_mapping: + include_type_name: true + index: test_index1 + type: test_type + body: + test_type: + dynamic: false + properties: + text: + analyzer: default + type: text + + - do: + indices.get_mapping: + include_type_name: true +# - match: {test_index1.mappings.test_type.properties.text.type: text} + - match: {test_index1.mappings._doc.properties.text.type: text} + diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index a69d7d735719b..9d7e3e77a7739 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -207,6 +207,9 @@ --- "Component and composable template validation": + - skip: + features: allowed_warnings + - do: cluster.put_component_template: name: my-ct @@ -226,6 +229,8 @@ eggplant: true - do: + allowed_warnings: + - "index template [my-it] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-it] will take precedence during new index creation" indices.put_index_template: name: my-it body: diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/IndexingPressureRestIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/IndexingPressureRestIT.java new file mode 100644 index 0000000000000..b096107e0a327 --- /dev/null +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/IndexingPressureRestIT.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.http; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.XContentTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +import static org.elasticsearch.rest.RestStatus.CREATED; +import static org.elasticsearch.rest.RestStatus.OK; +import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; + +/** + * Test Indexing Pressure Metrics and Statistics + */ +@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0) +public class IndexingPressureRestIT extends HttpSmokeTestCase { + + private static final Settings unboundedWriteQueue = Settings.builder().put("thread_pool.write.queue_size", -1).build(); + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB") + .put(unboundedWriteQueue) + .build(); + } + + @SuppressWarnings("unchecked") + public void testIndexingPressureStats() throws IOException { + Request createRequest = new Request("PUT", "/index_name"); + createRequest.setJsonEntity("{\"settings\": {\"index\": {\"number_of_shards\": 1, \"number_of_replicas\": 1, " + + "\"write.wait_for_active_shards\": 2}}}"); + final Response indexCreatedResponse = getRestClient().performRequest(createRequest); + assertThat(indexCreatedResponse.getStatusLine().getStatusCode(), equalTo(OK.getStatus())); + + Request successfulIndexingRequest = new Request("POST", "/index_name/_doc/"); + successfulIndexingRequest.setJsonEntity("{\"x\": \"small text\"}"); + final Response indexSuccessFul = getRestClient().performRequest(successfulIndexingRequest); + assertThat(indexSuccessFul.getStatusLine().getStatusCode(), equalTo(CREATED.getStatus())); + + Request getNodeStats = new Request("GET", "/_nodes/stats/indexing_pressure"); + final Response nodeStats = getRestClient().performRequest(getNodeStats); + Map nodeStatsMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats.getEntity().getContent(), true); + ArrayList values = new ArrayList<>(((Map) nodeStatsMap.get("nodes")).values()); + assertThat(values.size(), equalTo(2)); + XContentTestUtils.JsonMapView node1 = new XContentTestUtils.JsonMapView((Map) values.get(0)); + Integer node1IndexingBytes = node1.get("indexing_pressure.total.coordinating_and_primary_bytes"); + Integer node1ReplicaBytes = node1.get("indexing_pressure.total.replica_bytes"); + Integer node1Rejections = node1.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections"); + XContentTestUtils.JsonMapView node2 = new XContentTestUtils.JsonMapView((Map) values.get(1)); + Integer node2IndexingBytes = node2.get("indexing_pressure.total.coordinating_and_primary_bytes"); + Integer node2ReplicaBytes = node2.get("indexing_pressure.total.replica_bytes"); + Integer node2Rejections = node2.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections"); + + if (node1IndexingBytes == 0) { + assertThat(node2IndexingBytes, greaterThan(0)); + assertThat(node2IndexingBytes, lessThan(1024)); + } else { + assertThat(node1IndexingBytes, greaterThan(0)); + assertThat(node1IndexingBytes, lessThan(1024)); + } + + if (node1ReplicaBytes == 0) { + assertThat(node2ReplicaBytes, greaterThan(0)); + assertThat(node2ReplicaBytes, lessThan(1024)); + } else { + assertThat(node2ReplicaBytes, equalTo(0)); + assertThat(node1ReplicaBytes, lessThan(1024)); + } + + assertThat(node1Rejections, equalTo(0)); + assertThat(node2Rejections, equalTo(0)); + + Request failedIndexingRequest = new Request("POST", "/index_name/_doc/"); + String largeString = randomAlphaOfLength(10000); + failedIndexingRequest.setJsonEntity("{\"x\": " + largeString + "}"); + ResponseException exception = expectThrows(ResponseException.class, () -> getRestClient().performRequest(failedIndexingRequest)); + assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(TOO_MANY_REQUESTS.getStatus())); + + Request getNodeStats2 = new Request("GET", "/_nodes/stats/indexing_pressure"); + final Response nodeStats2 = getRestClient().performRequest(getNodeStats2); + Map nodeStatsMap2 = XContentHelper.convertToMap(JsonXContent.jsonXContent, nodeStats2.getEntity().getContent(), + true); + ArrayList values2 = new ArrayList<>(((Map) nodeStatsMap2.get("nodes")).values()); + assertThat(values2.size(), equalTo(2)); + XContentTestUtils.JsonMapView node1AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(0)); + node1Rejections = node1AfterRejection.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections"); + XContentTestUtils.JsonMapView node2AfterRejection = new XContentTestUtils.JsonMapView((Map) values2.get(1)); + node2Rejections = node2AfterRejection.get("indexing_pressure.total.coordinating_and_primary_memory_limit_rejections"); + + if (node1Rejections == 0) { + assertThat(node2Rejections, equalTo(1)); + } else { + assertThat(node1Rejections, equalTo(1)); + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json index ef51638022beb..ce19186bea6a9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json @@ -20,8 +20,8 @@ ], "parts":{ "name":{ - "type":"string", - "description":"The name or wildcard expression of the requested data streams" + "type":"list", + "description":"A comma-separated list of data streams to get; use `*` to get all data streams" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json index 0dc5e159a4528..fcc0f1d14da2b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json @@ -44,7 +44,8 @@ "process", "thread_pool", "transport", - "discovery" + "discovery", + "indexing_pressure" ], "description":"Limit the information returned to the specified metrics" } @@ -69,7 +70,8 @@ "process", "thread_pool", "transport", - "discovery" + "discovery", + "indexing_pressure" ], "description":"Limit the information returned to the specified metrics" }, @@ -98,7 +100,8 @@ "process", "thread_pool", "transport", - "discovery" + "discovery", + "indexing_pressure" ], "description":"Limit the information returned to the specified metrics" }, @@ -146,7 +149,8 @@ "process", "thread_pool", "transport", - "discovery" + "discovery", + "indexing_pressure" ], "description":"Limit the information returned to the specified metrics" }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml index 44aa7c390e441..a4d1841ed7108 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml @@ -106,47 +106,3 @@ setup: settings: index.number_of_replicas: 0 index.number_of_shards: 6 - ---- -"Prohibit clone on data stream's write index": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - catch: bad_request - indices.clone: - index: ".ds-simple-data-stream1-000001" - target: "target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 - index.number_of_shards: 2 - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml deleted file mode 100644 index a2c5198845c10..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml +++ /dev/null @@ -1,64 +0,0 @@ ---- -"Test apis that do not supported data streams": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" - indices.put_index_template: - name: my-template - body: - index_patterns: [logs-*] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: logs-foobar - - is_true: acknowledged - - - do: - index: - index: logs-foobar - refresh: true - body: - '@timestamp': '2020-12-12' - foo: bar - - match: {_index: .ds-logs-foobar-000001} - - - do: - search: - index: logs-foobar - body: { query: { match_all: {} } } - - length: { hits.hits: 1 } - - match: { hits.hits.0._index: .ds-logs-foobar-000001 } - - match: { hits.hits.0._source.foo: 'bar' } - - - do: - catch: missing - indices.delete: - index: logs-foobar - - - do: - indices.delete_data_stream: - name: logs-foobar - - is_true: acknowledged - ---- -"APIs temporarily muted": - - skip: - version: "all" - reason: "restore to above test after data stream resolution PRs have been merged" - - - do: - catch: bad_request - indices.close: - index: logs-* diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml deleted file mode 100644 index 770b12a29fd75..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -setup: - - skip: - features: allowed_warnings - - do: - allowed_warnings: - - "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation" - indices.put_index_template: - name: logs_template - body: - index_patterns: logs-foobar - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: logs-foobar - ---- -teardown: - - do: - indices.delete_data_stream: - name: logs-foobar - ---- -"Verify get index api": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - - - do: - indices.get: - index: logs-foobar - - is_true: \.ds-logs-foobar-000001 - - is_false: logs-foobar - - match: { \.ds-logs-foobar-000001.settings.index.number_of_shards: '1' } - ---- -"Verify get mapping api": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - - - do: - indices.get_mapping: - index: logs-foobar - - is_true: \.ds-logs-foobar-000001.mappings - - is_false: \.ds-logs-foobar.mappings diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/60_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/60_data_streams.yml deleted file mode 100644 index 0766936cf0241..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/60_data_streams.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -"Data streams": - - skip: - features: allowed_warnings - version: " - 7.99.99" - reason: "change to 7.8.99 after backport" - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - indices.get_field_mapping: - index: simple-data-stream1 - fields: foo - - - is_true: \.ds-simple-data-stream1-000001 - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml index 697bd61d38d81..2fff1a42e7a6b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml @@ -115,109 +115,3 @@ - match: { indices.index_1.closed: true } - match: { indices.index_2.closed: true } - match: { indices.index_3.closed: true } - ---- -"Close write index for data stream fails": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - catch: bad_request - indices.close: - index: ".ds-simple-data-stream1-000001" - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - ---- -"Open write index for data stream opens all backing indices": - - skip: - version: " - 7.99.99" - reason: "change to - 7.8.99 after backport" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - # rollover data stream twice to create new backing indices - - do: - indices.rollover: - alias: "simple-data-stream1" - - - match: { old_index: .ds-simple-data-stream1-000001 } - - match: { new_index: .ds-simple-data-stream1-000002 } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.rollover: - alias: "simple-data-stream1" - - - match: { old_index: .ds-simple-data-stream1-000002 } - - match: { new_index: .ds-simple-data-stream1-000003 } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - indices.close: - index: ".ds-simple-data-stream1-000001,.ds-simple-data-stream1-000002" - - is_true: acknowledged - - - do: - indices.open: - index: simple-data-stream1 - - is_true: acknowledged - - # all closed backing indices should be re-opened and returned - - do: - indices.get: - index: ".ds-simple-data-stream1-*" - - - is_true: \.ds-simple-data-stream1-000001.settings - - is_true: \.ds-simple-data-stream1-000002.settings - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml index baafca5a77093..9b467ee460f6b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.resolve_index/10_basic_resolve_index.yml @@ -5,47 +5,6 @@ setup: reason: "resolve index api only supported in 7.9+" features: allowed_warnings - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - do: - allowed_warnings: - - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" - indices.put_index_template: - name: my-template2 - body: - index_patterns: [simple-data-stream2] - template: - mappings: - properties: - '@timestamp2': - type: date - data_stream: - timestamp_field: '@timestamp2' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - - do: - indices.create_data_stream: - name: simple-data-stream2 - - - do: - indices.rollover: - alias: "simple-data-stream2" - - do: indices.create: index: test_index1 @@ -74,7 +33,7 @@ setup: test_blias: {} --- -"Resolve index with indices, aliases, and data streams": +"Resolve index with indices and aliases": - skip: version: " - 7.8.99" reason: "resolve index api only supported in 7.9+" @@ -99,23 +58,7 @@ setup: - match: {aliases.1.indices.1: test_index3} - match: {aliases.2.name: test_clias} - match: {aliases.2.indices.0: test_index1} - - match: {data_streams.0.name: simple-data-stream1} - - match: {data_streams.0.backing_indices.0: .ds-simple-data-stream1-000001} - - match: {data_streams.0.timestamp_field: "@timestamp"} - - match: {data_streams.1.name: simple-data-stream2} - - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001} - - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002} - - match: {data_streams.1.timestamp_field: "@timestamp"} - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - indices.delete_data_stream: - name: simple-data-stream2 - - is_true: acknowledged + - length: {data_streams: 0} --- "Resolve index with hidden and closed indices": @@ -159,20 +102,4 @@ setup: - match: {aliases.1.indices.1: test_index3} - match: {aliases.2.name: test_clias} - match: {aliases.2.indices.0: test_index1} - - match: {data_streams.0.name: simple-data-stream1} - - match: {data_streams.0.backing_indices.0: .ds-simple-data-stream1-000001} - - match: {data_streams.0.timestamp_field: "@timestamp"} - - match: {data_streams.1.name: simple-data-stream2} - - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001} - - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002} - - match: {data_streams.1.timestamp_field: "@timestamp"} - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - indices.delete_data_stream: - name: simple-data-stream2 - - is_true: acknowledged + - length: {data_streams: 0} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shard_stores/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shard_stores/10_basic.yml index 25b3ca410b75f..1f621c2e50b9d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shard_stores/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shard_stores/10_basic.yml @@ -76,60 +76,3 @@ - match: { indices.index1.shards.0.stores.0.allocation: "primary" } - match: { indices.index2.shards.0.stores.0.allocation: "primary" } - match: { indices.index2.shards.1.stores.0.allocation: "primary" } - ---- -"Data streams test": - - skip: - version: " - 7.99.99" - reason: "change to 7.8.99 after backport" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - settings: - number_of_shards: "1" - number_of_replicas: "0" - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - # rollover data stream to create new backing index - - do: - indices.rollover: - alias: "simple-data-stream1" - - - match: { old_index: .ds-simple-data-stream1-000001 } - - match: { new_index: .ds-simple-data-stream1-000002 } - - match: { rolled_over: true } - - match: { dry_run: false } - - - do: - cluster.health: - wait_for_status: green - - - do: - indices.shard_stores: - index: simple-data-stream1 - status: "green" - - - match: { indices.\.ds-simple-data-stream1-000001.shards.0.stores.0.allocation: "primary" } - - match: { indices.\.ds-simple-data-stream1-000002.shards.0.stores.0.allocation: "primary" } - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml index 792b77fa120f5..6697984003a20 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml @@ -77,46 +77,3 @@ - match: { _index: target } - match: { _id: "1" } - match: { _source: { foo: "hello world" } } - ---- -"Prohibit shrink on data stream's write index": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - catch: bad_request - indices.shrink: - index: ".ds-simple-data-stream1-000001" - target: "target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 5ec2b6489050d..067b2bb5774c3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -205,47 +205,3 @@ setup: settings: index.number_of_replicas: 0 index.number_of_shards: 6 - ---- -"Prohibit split on data stream's write index": - - skip: - version: " - 7.8.99" - reason: "data streams only supported in 7.9+" - features: allowed_warnings - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - catch: bad_request - indices.split: - index: ".ds-simple-data-stream1-000001" - target: "target" - wait_for_active_shards: 1 - master_timeout: 10s - body: - settings: - index.number_of_replicas: 0 - index.number_of_shards: 4 - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/50_indexing_pressure.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/50_indexing_pressure.yml new file mode 100644 index 0000000000000..bf85c9e2ac1af --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/50_indexing_pressure.yml @@ -0,0 +1,28 @@ +--- +"Indexing pressure stats": + - skip: + version: " - 7.8.99" + reason: "indexing_pressure was added in 7.9" + features: [arbitrary_key] + + - do: + nodes.info: {} + - set: + nodes._arbitrary_key_: node_id + + - do: + nodes.stats: + metric: [ indexing_pressure ] + + - gte: { nodes.$node_id.indexing_pressure.total.coordinating_and_primary_bytes: 0 } + - gte: { nodes.$node_id.indexing_pressure.total.replica_bytes: 0 } + - gte: { nodes.$node_id.indexing_pressure.total.all_bytes: 0 } + - gte: { nodes.$node_id.indexing_pressure.total.coordinating_and_primary_memory_limit_rejections: 0 } + - gte: { nodes.$node_id.indexing_pressure.total.replica_memory_limit_rejections: 0 } + - gte: { nodes.$node_id.indexing_pressure.current.coordinating_and_primary_bytes: 0 } + - gte: { nodes.$node_id.indexing_pressure.current.replica_bytes: 0 } + - gte: { nodes.$node_id.indexing_pressure.current.all_bytes: 0 } + +# TODO: +# +# Change skipped version after backport diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml index 31ae8fd2690a3..6ffbde5b95e49 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml @@ -528,3 +528,48 @@ setup: - match: { profile.shards.0.aggregations.0.description: histo } - match: { profile.shards.0.aggregations.0.breakdown.collect_count: 4 } - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 } + +--- +"histogram with hard bounds": + - skip: + version: " - 7.99.99" + reason: hard_bounds were introduced in 8.0.0 + + - do: + indices.create: + index: test_3 + body: + mappings: + properties: + range: + type: long_range + + - do: + bulk: + index: test_3 + refresh: true + body: + - '{"index": {}}' + - '{"range": {"lte": 10}}' + - '{"index": {}}' + - '{"range": {"gte": 15}}' + + - do: + search: + index: test_3 + body: + size: 0 + aggs: + histo: + histogram: + field: range + interval: 1 + hard_bounds: + min: 0 + max: 20 + - match: { hits.total.value: 2 } + - length: { aggregations.histo.buckets: 21 } + - match: { aggregations.histo.buckets.0.key: 0 } + - match: { aggregations.histo.buckets.0.doc_count: 1 } + - match: { aggregations.histo.buckets.20.key: 20 } + - match: { aggregations.histo.buckets.20.doc_count: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/350_variable_width_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/350_variable_width_histogram.yml index 071e543e8a25e..47ff51dd6a1c4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/350_variable_width_histogram.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/350_variable_width_histogram.yml @@ -28,8 +28,8 @@ setup: --- "basic": - skip: - version: " - 7.99.99" - reason: added in 8.0.0 (to be backported to 7.9.0) + version: " - 7.9.99" + reason: added in 7.9.0 - do: search: body: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml new file mode 100644 index 0000000000000..729f58a223576 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/360_date_histogram.yml @@ -0,0 +1,63 @@ +setup: + - skip: + version: " - 7.1.99" + reason: calendar_interval introduced in 7.2.0 + + - do: + indices.create: + index: test_date_hist + body: + settings: + # There was a BWC issue that only showed up on empty shards. This + # test has 4 docs and 5 shards makes sure we get one empty. + number_of_shards: 5 + mappings: + properties: + range: + type: date_range + + - do: + bulk: + index: test_date_hist + refresh: true + body: + - '{"index": {}}' + - '{"range": {"gte": "2016-01-01", "lt": "2016-01-02"}}' + - '{"index": {}}' + - '{"range": {"gte": "2016-01-02", "lt": "2016-01-03"}}' + - '{"index": {}}' + - '{"range": {"gte": "2016-02-01", "lt": "2016-02-02"}}' + - '{"index": {}}' + - '{"range": {"gte": "2016-03-01", "lt": "2016-03-02"}}' + - '{"index": {}}' + - '{"range": {"gte": "2016-04-01"}}' + - '{"index": {}}' + - '{"range": {"lt": "2016-02-01"}}' + +--- +"date_histogram on range with hard bounds": + - skip: + version: " - 7.99.99" + reason: waiting for backport + + - do: + search: + body: + size: 0 + aggs: + histo: + date_histogram: + field: range + calendar_interval: month + hard_bounds: + "min": "2015-06-01" + "max": "2016-06-01" + + - match: { hits.total.value: 6 } + - length: { aggregations.histo.buckets: 13 } + - match: { aggregations.histo.buckets.0.key_as_string: "2015-06-01T00:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 1 } + - match: { aggregations.histo.buckets.8.key_as_string: "2016-02-01T00:00:00.000Z" } + - match: { aggregations.histo.buckets.8.doc_count: 1 } + - match: { aggregations.histo.buckets.12.key_as_string: "2016-06-01T00:00:00.000Z" } + - match: { aggregations.histo.buckets.12.doc_count: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 5248c02ccad96..4959272465c3e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -29,7 +29,7 @@ setup: "Request with negative from value url parameter": - skip: - version: " - 7.9.99" + version: " - 7.99.99" reason: waiting for backport - do: @@ -43,7 +43,7 @@ setup: "Request with negative from value in body": - skip: - version: " - 7.9.99" + version: " - 7.99.99" reason: waiting for backport - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml index 55db6cac72d23..c89873f2b2c6f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_shards/10_basic.yml @@ -98,41 +98,3 @@ - match: { shards.0.0.index: test_index } - match: { indices.test_index.aliases: [test_alias_no_filter]} - is_false: indices.test_index.filter - ---- -"Search shards on data streams": - - skip: - features: allowed_warnings - version: " - 7.99.99" - reason: "change to 7.8.99 after backport" - - - do: - allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" - indices.put_index_template: - name: my-template1 - body: - index_patterns: [simple-data-stream1] - template: - mappings: - properties: - '@timestamp': - type: date - data_stream: - timestamp_field: '@timestamp' - - - do: - indices.create_data_stream: - name: simple-data-stream1 - - is_true: acknowledged - - - do: - search_shards: - index: "simple-data-stream1" - - - match: { shards.0.0.index: ".ds-simple-data-stream1-000001" } - - - do: - indices.delete_data_stream: - name: simple-data-stream1 - - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml index 5ed71d83b5cbc..dd1417f032516 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.get/10_basic.yml @@ -172,7 +172,7 @@ setup: --- "Get snapshot info with metadata": - skip: - version: " - 7.9.99" + version: " - 7.99.99" reason: "8.0 changes get snapshots response format" - do: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index 3faf8831699cb..ef54c8883b07e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -24,26 +24,14 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; -import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationRequest; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.IngestTestPlugin; @@ -56,27 +44,20 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; -import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE; import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasItemInArray; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.oneOf; @@ -220,86 +201,4 @@ public void testDeleteIndexWhileIndexing() throws Exception { } } - public void testMixedAutoCreate() throws Exception { - PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo"); - createTemplateRequest.indexTemplate( - new ComposableIndexTemplate( - List.of("logs-foo*"), - new Template(null, new CompressedXContent(generateMapping("@timestamp")), null), - null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp")) - ); - client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); - - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); - - bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); - - bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-foobaz3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); - bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); - - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*"); - GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); - assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4)); - getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); - assertThat(getDataStreamsResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); - assertThat(getDataStreamsResponse.getDataStreams().get(1).getDataStream().getName(), equalTo("logs-foobaz")); - assertThat(getDataStreamsResponse.getDataStreams().get(2).getDataStream().getName(), equalTo("logs-foobaz2")); - assertThat(getDataStreamsResponse.getDataStreams().get(3).getDataStream().getName(), equalTo("logs-foobaz3")); - - GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-bar*")).actionGet(); - assertThat(getIndexResponse.getIndices(), arrayWithSize(4)); - assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barbaz")); - assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo")); - assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo2")); - assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo3")); - - DeleteDataStreamAction.Request deleteDSReq = new DeleteDataStreamAction.Request(new String[]{"*"}); - client().execute(DeleteDataStreamAction.INSTANCE, deleteDSReq).actionGet(); - DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); - client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); - } - - public void testAutoCreateV1TemplateNoDataStream() { - Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); - - PutIndexTemplateRequest v1Request = new PutIndexTemplateRequest("logs-foo"); - v1Request.patterns(List.of("logs-foo*")); - v1Request.settings(settings); - v1Request.order(Integer.MAX_VALUE); // in order to avoid number_of_replicas being overwritten by random_template - client().admin().indices().putTemplate(v1Request).actionGet(); - - BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); - BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); - - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*"); - GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); - assertThat(getDataStreamsResponse.getDataStreams(), hasSize(0)); - - GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-foobar")).actionGet(); - assertThat(getIndexResponse.getIndices(), arrayWithSize(1)); - assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-foobar")); - assertThat(getIndexResponse.getSettings().get("logs-foobar").get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS), equalTo("0")); - } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteMemoryLimitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteMemoryLimitsIT.java deleted file mode 100644 index a7cb2f5260fde..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/WriteMemoryLimitsIT.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.action.bulk; - -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.concurrent.CountDownLatch; -import java.util.stream.Stream; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.greaterThan; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 2, numClientNodes = 1) -public class WriteMemoryLimitsIT extends ESIntegTestCase { - - public static final String INDEX_NAME = "test"; - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder() - .put(super.nodeSettings(nodeOrdinal)) - // Need at least two threads because we are going to block one - .put("thread_pool.write.size", 2) - .build(); - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(MockTransportService.TestPlugin.class, InternalSettingsPlugin.class); - } - - @Override - protected int numberOfReplicas() { - return 1; - } - - @Override - protected int numberOfShards() { - return 1; - } - - public void testWriteBytesAreIncremented() throws Exception { - assertAcked(prepareCreate(INDEX_NAME, Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))); - ensureGreen(INDEX_NAME); - - IndicesStatsResponse response = client().admin().indices().prepareStats(INDEX_NAME).get(); - String primaryId = Stream.of(response.getShards()) - .map(ShardStats::getShardRouting) - .filter(ShardRouting::primary) - .findAny() - .get() - .currentNodeId(); - String replicaId = Stream.of(response.getShards()) - .map(ShardStats::getShardRouting) - .filter(sr -> sr.primary() == false) - .findAny() - .get() - .currentNodeId(); - DiscoveryNodes nodes = client().admin().cluster().prepareState().get().getState().nodes(); - String primaryName = nodes.get(primaryId).getName(); - String replicaName = nodes.get(replicaId).getName(); - String coordinatingOnlyNode = nodes.getCoordinatingOnlyNodes().iterator().next().value.getName(); - - final CountDownLatch replicationSendPointReached = new CountDownLatch(1); - final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1); - final CountDownLatch newActionsSendPointReached = new CountDownLatch(2); - final CountDownLatch latchBlockingReplication = new CountDownLatch(1); - - TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName); - final MockTransportService primaryTransportService = (MockTransportService) primaryService; - TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName); - final MockTransportService replicaTransportService = (MockTransportService) replicaService; - - primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> { - if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) { - try { - replicationSendPointReached.countDown(); - latchBlockingReplicationSend.await(); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - } - connection.sendRequest(requestId, action, request, options); - }); - - final BulkRequest bulkRequest = new BulkRequest(); - int totalRequestSize = 0; - for (int i = 0; i < 80; ++i) { - IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) - .source(Collections.singletonMap("key", randomAlphaOfLength(50))); - totalRequestSize += request.ramBytesUsed(); - assertTrue(request.ramBytesUsed() > request.source().length()); - bulkRequest.add(request); - } - - final long bulkRequestSize = bulkRequest.ramBytesUsed(); - final long bulkShardRequestSize = totalRequestSize; - - try { - final ActionFuture successFuture = client(coordinatingOnlyNode).bulk(bulkRequest); - replicationSendPointReached.await(); - - WriteMemoryLimits primaryWriteLimits = internalCluster().getInstance(WriteMemoryLimits.class, primaryName); - WriteMemoryLimits replicaWriteLimits = internalCluster().getInstance(WriteMemoryLimits.class, replicaName); - WriteMemoryLimits coordinatingWriteLimits = internalCluster().getInstance(WriteMemoryLimits.class, coordinatingOnlyNode); - - assertThat(primaryWriteLimits.getWriteBytes(), greaterThan(bulkShardRequestSize)); - assertEquals(0, primaryWriteLimits.getReplicaWriteBytes()); - assertEquals(0, replicaWriteLimits.getWriteBytes()); - assertEquals(0, replicaWriteLimits.getReplicaWriteBytes()); - assertEquals(bulkRequestSize, coordinatingWriteLimits.getWriteBytes()); - assertEquals(0, coordinatingWriteLimits.getReplicaWriteBytes()); - - ThreadPool replicaThreadPool = replicaTransportService.getThreadPool(); - // Block the replica Write thread pool - replicaThreadPool.executor(ThreadPool.Names.WRITE).execute(() -> { - try { - newActionsSendPointReached.countDown(); - latchBlockingReplication.await(); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - }); - replicaThreadPool.executor(ThreadPool.Names.WRITE).execute(() -> { - try { - newActionsSendPointReached.countDown(); - latchBlockingReplication.await(); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - }); - newActionsSendPointReached.await(); - latchBlockingReplicationSend.countDown(); - - IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) - .source(Collections.singletonMap("key", randomAlphaOfLength(50))); - final BulkRequest secondBulkRequest = new BulkRequest(); - secondBulkRequest.add(request); - - // Use the primary or the replica data node as the coordinating node this time - boolean usePrimaryAsCoordinatingNode = randomBoolean(); - final ActionFuture secondFuture; - if (usePrimaryAsCoordinatingNode) { - secondFuture = client(primaryName).bulk(secondBulkRequest); - } else { - secondFuture = client(replicaName).bulk(secondBulkRequest); - } - - final long secondBulkRequestSize = secondBulkRequest.ramBytesUsed(); - final long secondBulkShardRequestSize = request.ramBytesUsed(); - - if (usePrimaryAsCoordinatingNode) { - assertThat(primaryWriteLimits.getWriteBytes(), greaterThan(bulkShardRequestSize + secondBulkRequestSize)); - assertEquals(0, replicaWriteLimits.getWriteBytes()); - } else { - assertThat(primaryWriteLimits.getWriteBytes(), greaterThan(bulkShardRequestSize)); - assertEquals(secondBulkRequestSize, replicaWriteLimits.getWriteBytes()); - } - assertEquals(bulkRequestSize, coordinatingWriteLimits.getWriteBytes()); - assertBusy(() -> assertThat(replicaWriteLimits.getReplicaWriteBytes(), - greaterThan(bulkShardRequestSize + secondBulkShardRequestSize))); - - latchBlockingReplication.countDown(); - - successFuture.actionGet(); - secondFuture.actionGet(); - - assertEquals(0, primaryWriteLimits.getWriteBytes()); - assertEquals(0, primaryWriteLimits.getReplicaWriteBytes()); - assertEquals(0, replicaWriteLimits.getWriteBytes()); - assertEquals(0, replicaWriteLimits.getReplicaWriteBytes()); - assertEquals(0, coordinatingWriteLimits.getWriteBytes()); - assertEquals(0, coordinatingWriteLimits.getReplicaWriteBytes()); - } finally { - if (replicationSendPointReached.getCount() > 0) { - replicationSendPointReached.countDown(); - } - while (newActionsSendPointReached.getCount() > 0) { - newActionsSendPointReached.countDown(); - } - if (latchBlockingReplicationSend.getCount() > 0) { - latchBlockingReplicationSend.countDown(); - } - if (latchBlockingReplication.getCount() > 0) { - latchBlockingReplication.countDown(); - } - primaryTransportService.clearAllRules(); - } - } -} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index cb83a8833b109..605a9f66d59e2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -42,6 +42,7 @@ import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.gateway.DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; import static org.elasticsearch.test.NodeRoles.nonMasterNode; @@ -314,11 +315,15 @@ public void test3MasterNodes2Failed() throws Exception { public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); + Settings settings = Settings.builder() + .put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), true) + .build(); + logger.info("--> start mixed data and master-eligible node and bootstrap cluster"); - String masterNode = internalCluster().startNode(); // node ordinal 0 + String masterNode = internalCluster().startNode(settings); // node ordinal 0 logger.info("--> start data-only node and ensure 2 nodes stable cluster"); - String dataNode = internalCluster().startDataOnlyNode(); // node ordinal 1 + String dataNode = internalCluster().startDataOnlyNode(settings); // node ordinal 1 ensureStableCluster(2); logger.info("--> index 1 doc and ensure index is green"); @@ -332,11 +337,18 @@ public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Excepti assertThat(client().prepareGet("test", "1").execute().actionGet().isExists(), equalTo(true)); logger.info("--> stop data-only node and detach it from the old cluster"); - Settings dataNodeDataPathSettings = internalCluster().dataPathSettings(dataNode); + Settings dataNodeDataPathSettings = Settings.builder() + .put(internalCluster().dataPathSettings(dataNode), true) + .put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), true) + .build(); assertBusy(() -> internalCluster().getInstance(GatewayMetaState.class, dataNode).allPendingAsyncStatesWritten()); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); final Environment environment = TestEnvironment.newEnvironment( - Settings.builder().put(internalCluster().getDefaultSettings()).put(dataNodeDataPathSettings).build()); + Settings.builder() + .put(internalCluster().getDefaultSettings()) + .put(dataNodeDataPathSettings) + .put(AUTO_IMPORT_DANGLING_INDICES_SETTING.getKey(), true) + .build()); detachCluster(environment, false); logger.info("--> stop master-eligible node, clear its data and start it again - new cluster should form"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java new file mode 100644 index 0000000000000..43ccf33db1e00 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java @@ -0,0 +1,416 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.bulk.TransportShardBulkAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 2, numClientNodes = 1) +public class IndexingPressureIT extends ESIntegTestCase { + + // TODO: Add additional REST tests when metrics are exposed + + public static final String INDEX_NAME = "test"; + + private static final Settings unboundedWriteQueue = Settings.builder().put("thread_pool.write.queue_size", -1).build(); + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(unboundedWriteQueue) + .build(); + } + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTransportService.TestPlugin.class, InternalSettingsPlugin.class); + } + + @Override + protected int numberOfReplicas() { + return 1; + } + + @Override + protected int numberOfShards() { + return 1; + } + + public void testWriteBytesAreIncremented() throws Exception { + assertAcked(prepareCreate(INDEX_NAME, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); + + Tuple primaryReplicaNodeNames = getPrimaryReplicaNodeNames(); + String primaryName = primaryReplicaNodeNames.v1(); + String replicaName = primaryReplicaNodeNames.v2(); + String coordinatingOnlyNode = getCoordinatingOnlyNode(); + + final CountDownLatch replicationSendPointReached = new CountDownLatch(1); + final CountDownLatch latchBlockingReplicationSend = new CountDownLatch(1); + + TransportService primaryService = internalCluster().getInstance(TransportService.class, primaryName); + final MockTransportService primaryTransportService = (MockTransportService) primaryService; + TransportService replicaService = internalCluster().getInstance(TransportService.class, replicaName); + final MockTransportService replicaTransportService = (MockTransportService) replicaService; + + primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> { + if (action.equals(TransportShardBulkAction.ACTION_NAME + "[r]")) { + try { + replicationSendPointReached.countDown(); + latchBlockingReplicationSend.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + connection.sendRequest(requestId, action, request, options); + }); + + final ThreadPool replicaThreadPool = replicaTransportService.getThreadPool(); + final Releasable replicaRelease = blockReplicas(replicaThreadPool); + + final BulkRequest bulkRequest = new BulkRequest(); + int totalRequestSize = 0; + for (int i = 0; i < 80; ++i) { + IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) + .source(Collections.singletonMap("key", randomAlphaOfLength(50))); + totalRequestSize += request.ramBytesUsed(); + assertTrue(request.ramBytesUsed() > request.source().length()); + bulkRequest.add(request); + } + + final long bulkRequestSize = bulkRequest.ramBytesUsed(); + final long bulkShardRequestSize = totalRequestSize; + + try { + final ActionFuture successFuture = client(coordinatingOnlyNode).bulk(bulkRequest); + replicationSendPointReached.await(); + + IndexingPressure primaryWriteLimits = internalCluster().getInstance(IndexingPressure.class, primaryName); + IndexingPressure replicaWriteLimits = internalCluster().getInstance(IndexingPressure.class, replicaName); + IndexingPressure coordinatingWriteLimits = internalCluster().getInstance(IndexingPressure.class, coordinatingOnlyNode); + + assertThat(primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes()); + assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + + latchBlockingReplicationSend.countDown(); + + IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) + .source(Collections.singletonMap("key", randomAlphaOfLength(50))); + final BulkRequest secondBulkRequest = new BulkRequest(); + secondBulkRequest.add(request); + + // Use the primary or the replica data node as the coordinating node this time + boolean usePrimaryAsCoordinatingNode = randomBoolean(); + final ActionFuture secondFuture; + if (usePrimaryAsCoordinatingNode) { + secondFuture = client(primaryName).bulk(secondBulkRequest); + } else { + secondFuture = client(replicaName).bulk(secondBulkRequest); + } + + final long secondBulkRequestSize = secondBulkRequest.ramBytesUsed(); + final long secondBulkShardRequestSize = request.ramBytesUsed(); + + if (usePrimaryAsCoordinatingNode) { + assertThat(primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes(), + greaterThan(bulkShardRequestSize + secondBulkRequestSize)); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + } else { + assertThat(primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(secondBulkRequestSize, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + } + assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertBusy(() -> assertThat(replicaWriteLimits.getCurrentReplicaBytes(), + greaterThan(bulkShardRequestSize + secondBulkShardRequestSize))); + + replicaRelease.close(); + + successFuture.actionGet(); + secondFuture.actionGet(); + + assertEquals(0, primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + } finally { + if (replicationSendPointReached.getCount() > 0) { + replicationSendPointReached.countDown(); + } + replicaRelease.close(); + if (latchBlockingReplicationSend.getCount() > 0) { + latchBlockingReplicationSend.countDown(); + } + replicaRelease.close(); + primaryTransportService.clearAllRules(); + } + } + + public void testWriteCanBeRejectedAtCoordinatingLevel() throws Exception { + final BulkRequest bulkRequest = new BulkRequest(); + int totalRequestSize = 0; + for (int i = 0; i < 80; ++i) { + IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) + .source(Collections.singletonMap("key", randomAlphaOfLength(50))); + totalRequestSize += request.ramBytesUsed(); + assertTrue(request.ramBytesUsed() > request.source().length()); + bulkRequest.add(request); + } + + final long bulkRequestSize = bulkRequest.ramBytesUsed(); + final long bulkShardRequestSize = totalRequestSize; + restartNodesWithSettings(Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), + (long) (bulkShardRequestSize * 1.5) + "B").build()); + + assertAcked(prepareCreate(INDEX_NAME, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); + + Tuple primaryReplicaNodeNames = getPrimaryReplicaNodeNames(); + String primaryName = primaryReplicaNodeNames.v1(); + String replicaName = primaryReplicaNodeNames.v2(); + String coordinatingOnlyNode = getCoordinatingOnlyNode(); + + final ThreadPool replicaThreadPool = internalCluster().getInstance(ThreadPool.class, replicaName); + try (Releasable replicaRelease = blockReplicas(replicaThreadPool)) { + final ActionFuture successFuture = client(coordinatingOnlyNode).bulk(bulkRequest); + + IndexingPressure primaryWriteLimits = internalCluster().getInstance(IndexingPressure.class, primaryName); + IndexingPressure replicaWriteLimits = internalCluster().getInstance(IndexingPressure.class, replicaName); + IndexingPressure coordinatingWriteLimits = internalCluster().getInstance(IndexingPressure.class, coordinatingOnlyNode); + + assertBusy(() -> { + assertThat(primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertThat(replicaWriteLimits.getCurrentReplicaBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(bulkRequestSize, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + }); + + expectThrows(EsRejectedExecutionException.class, () -> { + if (randomBoolean()) { + client(coordinatingOnlyNode).bulk(bulkRequest).actionGet(); + } else if (randomBoolean()) { + client(primaryName).bulk(bulkRequest).actionGet(); + } else { + client(replicaName).bulk(bulkRequest).actionGet(); + } + }); + + replicaRelease.close(); + + successFuture.actionGet(); + + assertEquals(0, primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + } + } + + public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { + final BulkRequest bulkRequest = new BulkRequest(); + int totalRequestSize = 0; + for (int i = 0; i < 80; ++i) { + IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) + .source(Collections.singletonMap("key", randomAlphaOfLength(50))); + totalRequestSize += request.ramBytesUsed(); + assertTrue(request.ramBytesUsed() > request.source().length()); + bulkRequest.add(request); + } + final long bulkShardRequestSize = totalRequestSize; + restartNodesWithSettings(Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), + (long)(bulkShardRequestSize * 1.5) + "B").build()); + + assertAcked(prepareCreate(INDEX_NAME, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); + + Tuple primaryReplicaNodeNames = getPrimaryReplicaNodeNames(); + String primaryName = primaryReplicaNodeNames.v1(); + String replicaName = primaryReplicaNodeNames.v2(); + String coordinatingOnlyNode = getCoordinatingOnlyNode(); + + final ThreadPool replicaThreadPool = internalCluster().getInstance(ThreadPool.class, replicaName); + try (Releasable replicaRelease = blockReplicas(replicaThreadPool)) { + final ActionFuture successFuture = client(primaryName).bulk(bulkRequest); + + IndexingPressure primaryWriteLimits = internalCluster().getInstance(IndexingPressure.class, primaryName); + IndexingPressure replicaWriteLimits = internalCluster().getInstance(IndexingPressure.class, replicaName); + IndexingPressure coordinatingWriteLimits = internalCluster().getInstance(IndexingPressure.class, coordinatingOnlyNode); + + assertBusy(() -> { + assertThat(primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertThat(replicaWriteLimits.getCurrentReplicaBytes(), greaterThan(bulkShardRequestSize)); + assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + }); + + BulkResponse responses = client(coordinatingOnlyNode).bulk(bulkRequest).actionGet(); + assertTrue(responses.hasFailures()); + assertThat(responses.getItems()[0].getFailure().getCause().getCause(), instanceOf(EsRejectedExecutionException.class)); + + replicaRelease.close(); + + successFuture.actionGet(); + + assertEquals(0, primaryWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, primaryWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, replicaWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, replicaWriteLimits.getCurrentReplicaBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentCoordinatingAndPrimaryBytes()); + assertEquals(0, coordinatingWriteLimits.getCurrentReplicaBytes()); + } + } + + public void testWritesWillSucceedIfBelowThreshold() throws Exception { + restartNodesWithSettings(Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1MB").build()); + assertAcked(prepareCreate(INDEX_NAME, Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))); + ensureGreen(INDEX_NAME); + + Tuple primaryReplicaNodeNames = getPrimaryReplicaNodeNames(); + String replicaName = primaryReplicaNodeNames.v2(); + String coordinatingOnlyNode = getCoordinatingOnlyNode(); + + final ThreadPool replicaThreadPool = internalCluster().getInstance(ThreadPool.class, replicaName); + try (Releasable replicaRelease = blockReplicas(replicaThreadPool)) { + // The write limits is set to 1MB. We will send up to 800KB to stay below that threshold. + int thresholdToStopSending = 800 * 1024; + + ArrayList> responses = new ArrayList<>(); + int totalRequestSize = 0; + while (totalRequestSize < thresholdToStopSending) { + IndexRequest request = new IndexRequest(INDEX_NAME).id(UUIDs.base64UUID()) + .source(Collections.singletonMap("key", randomAlphaOfLength(500))); + totalRequestSize += request.ramBytesUsed(); + responses.add(client(coordinatingOnlyNode).index(request)); + } + + replicaRelease.close(); + + // Would throw exception if one of the operations was rejected + responses.forEach(ActionFuture::actionGet); + } + } + + private void restartNodesWithSettings(Settings settings) throws Exception { + internalCluster().fullRestart(new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + return Settings.builder().put(unboundedWriteQueue).put(settings).build(); + } + }); + } + + private String getCoordinatingOnlyNode() { + return client().admin().cluster().prepareState().get().getState().nodes().getCoordinatingOnlyNodes().iterator().next() + .value.getName(); + } + + private Tuple getPrimaryReplicaNodeNames() { + IndicesStatsResponse response = client().admin().indices().prepareStats(INDEX_NAME).get(); + String primaryId = Stream.of(response.getShards()) + .map(ShardStats::getShardRouting) + .filter(ShardRouting::primary) + .findAny() + .get() + .currentNodeId(); + String replicaId = Stream.of(response.getShards()) + .map(ShardStats::getShardRouting) + .filter(sr -> sr.primary() == false) + .findAny() + .get() + .currentNodeId(); + DiscoveryNodes nodes = client().admin().cluster().prepareState().get().getState().nodes(); + String primaryName = nodes.get(primaryId).getName(); + String replicaName = nodes.get(replicaId).getName(); + return new Tuple<>(primaryName, replicaName); + } + + private Releasable blockReplicas(ThreadPool threadPool) { + final CountDownLatch blockReplication = new CountDownLatch(1); + final int threads = threadPool.info(ThreadPool.Names.WRITE).getMax(); + final CountDownLatch pointReached = new CountDownLatch(threads); + for (int i = 0; i< threads; ++i) { + threadPool.executor(ThreadPool.Names.WRITE).execute(() -> { + try { + pointReached.countDown(); + blockReplication.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + }); + } + + return () -> { + if (blockReplication.getCount() > 0) { + blockReplication.countDown(); + } + }; + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index 8873e9254e61a..8796662907f83 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -144,7 +144,7 @@ public void testCompletionMultiField() throws Exception { assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); - assertThat(aField.size(), equalTo(6)); + assertThat(aField.size(), equalTo(2)); assertThat(aField.get("type").toString(), equalTo("completion")); assertThat(aField.get("fields"), notNullValue()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 103fc00e9179d..5d299467fa682 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices; import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; @@ -28,12 +27,10 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.search.MultiSearchRequestBuilder; @@ -46,7 +43,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -665,18 +661,10 @@ static GetMappingsRequestBuilder getMapping(String... indices) { return client().admin().indices().prepareGetMappings(indices); } - static PutMappingRequestBuilder putMapping(String source, String... indices) { - return client().admin().indices().preparePutMapping(indices).setSource(source, XContentType.JSON); - } - static GetSettingsRequestBuilder getSettings(String... indices) { return client().admin().indices().prepareGetSettings(indices); } - static UpdateSettingsRequestBuilder updateSettings(Settings.Builder settings, String... indices) { - return client().admin().indices().prepareUpdateSettings(indices).setSettings(settings); - } - private static CreateSnapshotRequestBuilder snapshot(String name, String... indices) { return client().admin().cluster().prepareCreateSnapshot("dummy-repo", name).setWaitForCompletion(true).setIndices(indices); } @@ -688,15 +676,11 @@ private static RestoreSnapshotRequestBuilder restore(String name, String... indi .setIndices(indices); } - static ClusterHealthRequestBuilder health(String... indices) { - return client().admin().cluster().prepareHealth(indices); - } - - private static void verify(ActionRequestBuilder requestBuilder, boolean fail) { + private static void verify(ActionRequestBuilder requestBuilder, boolean fail) { verify(requestBuilder, fail, 0); } - private static void verify(ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) { + private static void verify(ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) { if (fail) { if (requestBuilder instanceof MultiSearchRequestBuilder) { MultiSearchResponse multiSearchResponse = ((MultiSearchRequestBuilder) requestBuilder).get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index df8c85d9b51b2..a09dec1860af2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -24,10 +24,18 @@ import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; +import java.io.IOException; import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; public class ComposableTemplateIT extends ESIntegTestCase { @@ -83,4 +91,23 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2)).get(); } + + public void testUsageOfDataStreamFails() throws IOException { + // Exception that would happen if a unknown field is provided in a composable template: + // The thrown exception will be used to compare against the exception that is thrown when providing + // a composable template with a data stream definition. + String content = "{\"index_patterns\":[\"logs-*-*\"],\"my_field\":\"bla\"}"; + XContentParser parser = + XContentHelper.createParser(xContentRegistry(), null, new BytesArray(content), XContentType.JSON); + Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); + + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, + null, new ComposableIndexTemplate.DataStreamTemplate("@timestamp")); + Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); + Exception actualException = (Exception) e.getCause(); + assertThat(actualException.getMessage(), + equalTo(expectedException.getMessage().replace("[1:32] ", "").replace("my_field", "data_stream"))); + assertThat(actualException.getMessage(), equalTo("[index_template] unknown field [data_stream]")); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 89f649afc769c..0b6d12d59726a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -38,7 +38,7 @@ import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; +import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; @@ -1084,7 +1084,7 @@ public void testSingleValueFieldWithExtendedBounds() throws Exception { .dateHistogramInterval(DateHistogramInterval.days(interval)) .minDocCount(0) // when explicitly specifying a format, the extended bounds should be defined by the same format - .extendedBounds(new ExtendedBounds(format(boundsMin, pattern), format(boundsMax, pattern))) + .extendedBounds(new LongBounds(format(boundsMin, pattern), format(boundsMax, pattern))) .format(pattern)) .get(); @@ -1152,7 +1152,7 @@ public void testSingleValueFieldWithExtendedBoundsTimezone() throws Exception { .from("now/d").to("now/d").includeLower(true).includeUpper(true).timeZone(timezone.getId())) .addAggregation( dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.hours(1)) - .timeZone(timezone).minDocCount(0).extendedBounds(new ExtendedBounds("now/d", "now/d+23h")) + .timeZone(timezone).minDocCount(0).extendedBounds(new LongBounds("now/d", "now/d+23h")) ).get(); assertSearchResponse(response); @@ -1205,7 +1205,7 @@ public void testSingleValueFieldWithExtendedBoundsOffset() throws Exception { .addAggregation( dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.days(1)) .offset("+6h").minDocCount(0) - .extendedBounds(new ExtendedBounds("2016-01-01T06:00:00Z", "2016-01-08T08:00:00Z")) + .extendedBounds(new LongBounds("2016-01-01T06:00:00Z", "2016-01-08T08:00:00Z")) ).get(); assertSearchResponse(response); @@ -1377,7 +1377,7 @@ public void testFormatIndexUnmapped() throws InterruptedException, ExecutionExce SearchResponse response = client().prepareSearch(indexDateUnmapped) .addAggregation( dateHistogram("histo").field("dateField").dateHistogramInterval(DateHistogramInterval.MONTH).format("yyyy-MM") - .minDocCount(0).extendedBounds(new ExtendedBounds("2018-01", "2018-01"))) + .minDocCount(0).extendedBounds(new LongBounds("2018-01", "2018-01"))) .get(); assertSearchResponse(response); Histogram histo = response.getAggregations().get("histo"); @@ -1433,7 +1433,7 @@ public void testDSTEndTransition() throws Exception { .setQuery(new MatchNoneQueryBuilder()) .addAggregation(dateHistogram("histo").field("date").timeZone(ZoneId.of("Europe/Oslo")) .calendarInterval(DateHistogramInterval.HOUR).minDocCount(0).extendedBounds( - new ExtendedBounds("2015-10-25T02:00:00.000+02:00", "2015-10-25T04:00:00.000+01:00"))) + new LongBounds("2015-10-25T02:00:00.000+02:00", "2015-10-25T04:00:00.000+01:00"))) .get(); Histogram histo = response.getAggregations().get("histo"); @@ -1450,7 +1450,7 @@ public void testDSTEndTransition() throws Exception { .setQuery(new MatchNoneQueryBuilder()) .addAggregation(dateHistogram("histo").field("date").timeZone(ZoneId.of("Europe/Oslo")) .dateHistogramInterval(DateHistogramInterval.HOUR).minDocCount(0).extendedBounds( - new ExtendedBounds("2015-10-25T02:00:00.000+02:00", "2015-10-25T04:00:00.000+01:00"))) + new LongBounds("2015-10-25T02:00:00.000+02:00", "2015-10-25T04:00:00.000+01:00"))) .get(); histo = response.getAggregations().get("histo"); @@ -1647,4 +1647,23 @@ public void testDateKeyFormatting() { assertThat(buckets.get(1).getKeyAsString(), equalTo("2012-02-01T00:00:00.000-07:00")); assertThat(buckets.get(2).getKeyAsString(), equalTo("2012-03-01T00:00:00.000-07:00")); } + + public void testHardBoundsOnDates() { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(dateHistogram("histo") + .field("date") + .calendarInterval(DateHistogramInterval.DAY) + .hardBounds(new LongBounds("2012-02-01T00:00:00.000", "2012-03-03T00:00:00.000")) + ) + .get(); + + assertSearchResponse(response); + + InternalDateHistogram histogram = response.getAggregations().get("histo"); + List buckets = histogram.getBuckets(); + assertThat(buckets.size(), equalTo(30)); + assertThat(buckets.get(1).getKeyAsString(), equalTo("2012-02-03T00:00:00.000Z")); + assertThat(buckets.get(29).getKeyAsString(), equalTo("2012-03-02T00:00:00.000Z")); + } + } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java index 5a8e0cc6a17bf..4283872a74003 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; +import org.elasticsearch.search.aggregations.bucket.histogram.DoubleBounds; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; import org.elasticsearch.search.aggregations.metrics.Avg; @@ -1195,6 +1196,58 @@ public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound assertMultiSortResponse(expectedKeys, BucketOrder.aggregation("avg_l", true)); } + public void testInvalidBounds() { + SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("empty_bucket_idx") + .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).hardBounds(new DoubleBounds(0.0, 10.0)) + .extendedBounds(3, 20)).get()); + assertThat(e.toString(), containsString("Extended bounds have to be inside hard bounds, hard bounds")); + + e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("empty_bucket_idx") + .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).hardBounds(new DoubleBounds(3.0, null)) + .extendedBounds(0, 20)).get()); + assertThat(e.toString(), containsString("Extended bounds have to be inside hard bounds, hard bounds")); + } + + public void testHardBounds() throws Exception { + assertAcked(prepareCreate("test").setMapping("d", "type=double").get()); + indexRandom(true, + client().prepareIndex("test").setId("1").setSource("d", -0.6), + client().prepareIndex("test").setId("2").setSource("d", 0.5), + client().prepareIndex("test").setId("3").setSource("d", 0.1)); + + SearchResponse r = client().prepareSearch("test") + .addAggregation(histogram("histo").field("d").interval(0.1).hardBounds(new DoubleBounds(0.0, null))) + .get(); + assertSearchResponse(r); + + Histogram histogram = r.getAggregations().get("histo"); + List buckets = histogram.getBuckets(); + assertEquals(5, buckets.size()); + assertEquals(0.1, (double) buckets.get(0).getKey(), 0.01d); + assertEquals(0.5, (double) buckets.get(4).getKey(), 0.01d); + + r = client().prepareSearch("test") + .addAggregation(histogram("histo").field("d").interval(0.1).hardBounds(new DoubleBounds(null, 0.0))) + .get(); + assertSearchResponse(r); + + histogram = r.getAggregations().get("histo"); + buckets = histogram.getBuckets(); + assertEquals(1, buckets.size()); + assertEquals(-0.6, (double) buckets.get(0).getKey(), 0.01d); + + r = client().prepareSearch("test") + .addAggregation(histogram("histo").field("d").interval(0.1).hardBounds(new DoubleBounds(0.0, 3.0))) + .get(); + assertSearchResponse(r); + + histogram = r.getAggregations().get("histo"); + buckets = histogram.getBuckets(); + assertEquals(1, buckets.size()); + assertEquals(0.1, (double) buckets.get(0).getKey(), 0.01d); + + } + private void assertMultiSortResponse(long[] expectedKeys, BucketOrder... order) { SearchResponse response = client() .prepareSearch("sort_idx") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java new file mode 100644 index 0000000000000..6e0c84f4732a9 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -0,0 +1,1316 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.snapshots; + +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; +import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.SnapshotDeletionsInProgress; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; +import org.elasticsearch.discovery.AbstractDisruptionTestCase; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.RepositoryException; +import org.elasticsearch.repositories.ShardGenerations; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.disruption.NetworkDisruption; +import org.elasticsearch.test.transport.MockTransportService; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class ConcurrentSnapshotsIT extends AbstractSnapshotIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(MockTransportService.TestPlugin.class, MockRepository.Plugin.class); + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)) + .put(AbstractDisruptionTestCase.DEFAULT_SETTINGS) + .build(); + } + + public void testLongRunningSnapshotAllowsConcurrentSnapshot() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-slow"); + + final ActionFuture createSlowFuture = + startFullSnapshotBlockedOnDataNode("slow-snapshot", repoName, dataNode); + + final String dataNode2 = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + final String indexFast = "index-fast"; + createIndexWithContent(indexFast, dataNode2, dataNode); + + assertSuccessful(client().admin().cluster().prepareCreateSnapshot(repoName, "fast-snapshot") + .setIndices(indexFast).setWaitForCompletion(true).execute()); + + assertThat(createSlowFuture.isDone(), is(false)); + unblockNode(repoName, dataNode); + + assertSuccessful(createSlowFuture); + } + + public void testDeletesAreBatched() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + + createIndex("foo"); + ensureGreen(); + + final int numSnapshots = randomIntBetween(1, 4); + final PlainActionFuture> allSnapshotsDone = PlainActionFuture.newFuture(); + final ActionListener snapshotsListener = new GroupedActionListener<>(allSnapshotsDone, numSnapshots); + final Collection snapshotNames = new HashSet<>(); + for (int i = 0; i < numSnapshots; i++) { + final String snapshot = "snap-" + i; + snapshotNames.add(snapshot); + client().admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true) + .execute(snapshotsListener); + } + final Collection snapshotResponses = allSnapshotsDone.get(); + for (CreateSnapshotResponse snapshotResponse : snapshotResponses) { + assertThat(snapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS)); + } + + createIndexWithContent("index-slow"); + + final ActionFuture createSlowFuture = + startFullSnapshotBlockedOnDataNode("blocked-snapshot", repoName, dataNode); + + final Collection> deleteFutures = new ArrayList<>(); + while (snapshotNames.isEmpty() == false) { + final Collection toDelete = randomSubsetOf(snapshotNames); + if (toDelete.isEmpty()) { + continue; + } + snapshotNames.removeAll(toDelete); + final StepListener future = new StepListener<>(); + client().admin().cluster().prepareDeleteSnapshot(repoName, toDelete.toArray(Strings.EMPTY_ARRAY)).execute(future); + deleteFutures.add(future); + } + + assertThat(createSlowFuture.isDone(), is(false)); + + final long repoGenAfterInitialSnapshots = getRepositoryData(repoName).getGenId(); + assertThat(repoGenAfterInitialSnapshots, is(numSnapshots - 1L)); + unblockNode(repoName, dataNode); + + final SnapshotInfo slowSnapshotInfo = assertSuccessful(createSlowFuture); + + logger.info("--> waiting for batched deletes to finish"); + final PlainActionFuture> allDeletesDone = new PlainActionFuture<>(); + final ActionListener deletesListener = new GroupedActionListener<>(allDeletesDone, deleteFutures.size()); + for (StepListener deleteFuture : deleteFutures) { + deleteFuture.whenComplete(deletesListener::onResponse, deletesListener::onFailure); + } + allDeletesDone.get(); + + logger.info("--> verifying repository state"); + final RepositoryData repositoryDataAfterDeletes = getRepositoryData(repoName); + // One increment for snapshot, one for all the deletes + assertThat(repositoryDataAfterDeletes.getGenId(), is(repoGenAfterInitialSnapshots + 2)); + assertThat(repositoryDataAfterDeletes.getSnapshotIds(), contains(slowSnapshotInfo.snapshotId())); + } + + public void testBlockedRepoDoesNotBlockOtherRepos() throws Exception { + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String blockedRepoName = "test-repo-blocked"; + final String otherRepoName = "test-repo"; + createRepository(blockedRepoName, "mock"); + createRepository(otherRepoName, "fs"); + createIndex("foo"); + ensureGreen(); + createIndexWithContent("index-slow"); + + final ActionFuture createSlowFuture = + startAndBlockFailingFullSnapshot(blockedRepoName, "blocked-snapshot"); + + client().admin().cluster().prepareCreateSnapshot(otherRepoName, "snapshot") + .setIndices("does-not-exist-*") + .setWaitForCompletion(false).get(); + + unblockNode(blockedRepoName, internalCluster().getMasterName()); + expectThrows(SnapshotException.class, createSlowFuture::actionGet); + + assertBusy(() -> assertThat(currentSnapshots(otherRepoName), empty()), 30L, TimeUnit.SECONDS); + } + + public void testMultipleReposAreIndependent() throws Exception { + internalCluster().startMasterOnlyNode(); + // We're blocking a some of the snapshot threads when we block the first repo below so we have to make sure we have enough threads + // left for the second concurrent snapshot. + final String dataNode = startDataNodeWithLargeSnapshotPool(); + final String blockedRepoName = "test-repo-blocked"; + final String otherRepoName = "test-repo"; + createRepository(blockedRepoName, "mock"); + createRepository(otherRepoName, "fs"); + createIndexWithContent("test-index"); + + final ActionFuture createSlowFuture = + startFullSnapshotBlockedOnDataNode("blocked-snapshot", blockedRepoName, dataNode); + + logger.info("--> waiting for concurrent snapshot(s) to finish"); + createNSnapshots(otherRepoName, randomIntBetween(1, 5)); + + unblockNode(blockedRepoName, dataNode); + assertSuccessful(createSlowFuture); + } + + public void testMultipleReposAreIndependent2() throws Exception { + internalCluster().startMasterOnlyNode(); + // We're blocking a some of the snapshot threads when we block the first repo below so we have to make sure we have enough threads + // left for the second repository's concurrent operations. + final String dataNode = startDataNodeWithLargeSnapshotPool(); + final String blockedRepoName = "test-repo-blocked"; + final String otherRepoName = "test-repo"; + createRepository(blockedRepoName, "mock"); + createRepository(otherRepoName, "fs"); + createIndexWithContent("test-index"); + + final ActionFuture createSlowFuture = + startFullSnapshotBlockedOnDataNode("blocked-snapshot", blockedRepoName, dataNode); + + logger.info("--> waiting for concurrent snapshot(s) to finish"); + createNSnapshots(otherRepoName, randomIntBetween(1, 5)); + assertAcked(startDelete(otherRepoName, "*").get()); + + unblockNode(blockedRepoName, dataNode); + assertSuccessful(createSlowFuture); + } + + public void testMultipleReposAreIndependent3() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); + internalCluster().startDataOnlyNode(); + final String blockedRepoName = "test-repo-blocked"; + final String otherRepoName = "test-repo"; + createRepository(blockedRepoName, "mock"); + createRepository(otherRepoName, "fs"); + createIndexWithContent("test-index"); + + createFullSnapshot( blockedRepoName, "blocked-snapshot"); + blockNodeOnAnyFiles(blockedRepoName, masterNode); + final ActionFuture slowDeleteFuture = startDelete(blockedRepoName, "*"); + + logger.info("--> waiting for concurrent snapshot(s) to finish"); + createNSnapshots(otherRepoName, randomIntBetween(1, 5)); + assertAcked(startDelete(otherRepoName, "*").get()); + + unblockNode(blockedRepoName, masterNode); + assertAcked(slowDeleteFuture.actionGet()); + } + + public void testSnapshotRunsAfterInProgressDelete() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + + ensureGreen(); + createIndexWithContent("index-test"); + + final String firstSnapshot = "first-snapshot"; + createFullSnapshot(repoName, firstSnapshot); + + blockMasterFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture deleteFuture = startDelete(repoName, firstSnapshot); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "second-snapshot"); + + unblockNode(repoName, masterNode); + final UncategorizedExecutionException ex = expectThrows(UncategorizedExecutionException.class, deleteFuture::actionGet); + assertThat(ex.getRootCause(), instanceOf(IOException.class)); + + assertSuccessful(snapshotFuture); + } + + public void testAbortOneOfMultipleSnapshots() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + final String firstIndex = "index-one"; + createIndexWithContent(firstIndex); + + final String firstSnapshot = "snapshot-one"; + final ActionFuture firstSnapshotResponse = + startFullSnapshotBlockedOnDataNode(firstSnapshot, repoName, dataNode); + + final String dataNode2 = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + final String secondIndex = "index-two"; + createIndexWithContent(secondIndex, dataNode2, dataNode); + + final String secondSnapshot = "snapshot-two"; + final ActionFuture secondSnapshotResponse = startFullSnapshot(repoName, secondSnapshot); + + logger.info("--> wait for snapshot on second data node to finish"); + awaitClusterState(state -> { + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + return snapshotsInProgress.entries().size() == 2 && snapshotHasCompletedShard(secondSnapshot, snapshotsInProgress); + }); + + final ActionFuture deleteSnapshotsResponse = startDelete(repoName, firstSnapshot); + awaitNDeletionsInProgress(1); + + logger.info("--> start third snapshot"); + final ActionFuture thirdSnapshotResponse = client().admin().cluster() + .prepareCreateSnapshot(repoName, "snapshot-three").setIndices(secondIndex).setWaitForCompletion(true).execute(); + + assertThat(firstSnapshotResponse.isDone(), is(false)); + assertThat(secondSnapshotResponse.isDone(), is(false)); + + unblockNode(repoName, dataNode); + final SnapshotInfo firstSnapshotInfo = firstSnapshotResponse.get().getSnapshotInfo(); + assertThat(firstSnapshotInfo.state(), is(SnapshotState.FAILED)); + assertThat(firstSnapshotInfo.reason(), is("Snapshot was aborted by deletion")); + + final SnapshotInfo secondSnapshotInfo = assertSuccessful(secondSnapshotResponse); + final SnapshotInfo thirdSnapshotInfo = assertSuccessful(thirdSnapshotResponse); + + assertThat(deleteSnapshotsResponse.get().isAcknowledged(), is(true)); + + logger.info("--> verify that the first snapshot is gone"); + assertThat(client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(repoName), + containsInAnyOrder(secondSnapshotInfo, thirdSnapshotInfo)); + } + + public void testCascadedAborts() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + + final String firstSnapshot = "snapshot-one"; + final ActionFuture firstSnapshotResponse = + startFullSnapshotBlockedOnDataNode(firstSnapshot, repoName, dataNode); + + final String dataNode2 = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + createIndexWithContent("index-two", dataNode2, dataNode); + + final String secondSnapshot = "snapshot-two"; + final ActionFuture secondSnapshotResponse = startFullSnapshot(repoName, secondSnapshot); + + logger.info("--> wait for snapshot on second data node to finish"); + awaitClusterState(state -> { + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + return snapshotsInProgress.entries().size() == 2 && snapshotHasCompletedShard(secondSnapshot, snapshotsInProgress); + }); + + final ActionFuture deleteSnapshotsResponse = startDelete(repoName, firstSnapshot); + awaitNDeletionsInProgress(1); + + final ActionFuture thirdSnapshotResponse = startFullSnapshot(repoName, "snapshot-three"); + + assertThat(firstSnapshotResponse.isDone(), is(false)); + assertThat(secondSnapshotResponse.isDone(), is(false)); + + logger.info("--> waiting for all three snapshots to show up as in-progress"); + assertBusy(() -> assertThat(currentSnapshots(repoName), hasSize(3)), 30L, TimeUnit.SECONDS); + + final ActionFuture allDeletedResponse = startDelete(repoName, "*"); + + logger.info("--> waiting for second and third snapshot to finish"); + assertBusy(() -> { + assertThat(currentSnapshots(repoName), hasSize(1)); + final SnapshotsInProgress snapshotsInProgress = clusterService().state().custom(SnapshotsInProgress.TYPE); + assertThat(snapshotsInProgress.entries().get(0).state(), is(SnapshotsInProgress.State.ABORTED)); + }, 30L, TimeUnit.SECONDS); + + unblockNode(repoName, dataNode); + + logger.info("--> verify all snapshots were aborted"); + assertThat(firstSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); + assertThat(secondSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); + assertThat(thirdSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); + + logger.info("--> verify both deletes have completed"); + assertAcked(deleteSnapshotsResponse.get()); + assertAcked(allDeletedResponse.get()); + + logger.info("--> verify that all snapshots are gone"); + assertThat(client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(repoName), empty()); + } + + public void testMasterFailOverWithQueuedDeletes() throws Exception { + internalCluster().startMasterOnlyNodes(3); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + + final String firstIndex = "index-one"; + createIndexWithContent(firstIndex); + + final String firstSnapshot = "snapshot-one"; + blockDataNode(repoName, dataNode); + final ActionFuture firstSnapshotResponse = startFullSnapshotFromNonMasterClient(repoName, firstSnapshot); + waitForBlock(dataNode, repoName, TimeValue.timeValueSeconds(30L)); + + final String dataNode2 = internalCluster().startDataOnlyNode(); + ensureStableCluster(5); + final String secondIndex = "index-two"; + createIndexWithContent(secondIndex, dataNode2, dataNode); + + final String secondSnapshot = "snapshot-two"; + final ActionFuture secondSnapshotResponse = startFullSnapshot(repoName, secondSnapshot); + + logger.info("--> wait for snapshot on second data node to finish"); + awaitClusterState(state -> { + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + return snapshotsInProgress.entries().size() == 2 && snapshotHasCompletedShard(secondSnapshot, snapshotsInProgress); + }); + + final ActionFuture firstDeleteFuture = startDeleteFromNonMasterClient(repoName, firstSnapshot); + awaitNDeletionsInProgress(1); + + blockDataNode(repoName, dataNode2); + final ActionFuture snapshotThreeFuture = startFullSnapshotFromNonMasterClient(repoName, "snapshot-three"); + waitForBlock(dataNode2, repoName, TimeValue.timeValueSeconds(30L)); + + assertThat(firstSnapshotResponse.isDone(), is(false)); + assertThat(secondSnapshotResponse.isDone(), is(false)); + + logger.info("--> waiting for all three snapshots to show up as in-progress"); + assertBusy(() -> assertThat(currentSnapshots(repoName), hasSize(3)), 30L, TimeUnit.SECONDS); + + final ActionFuture deleteAllSnapshots = startDeleteFromNonMasterClient(repoName, "*"); + logger.info("--> wait for delete to be enqueued in cluster state"); + awaitClusterState(state -> { + final SnapshotDeletionsInProgress deletionsInProgress = state.custom(SnapshotDeletionsInProgress.TYPE); + return deletionsInProgress.getEntries().size() == 1 && deletionsInProgress.getEntries().get(0).getSnapshots().size() == 3; + }); + + logger.info("--> waiting for second snapshot to finish and the other two snapshots to become aborted"); + assertBusy(() -> { + assertThat(currentSnapshots(repoName), hasSize(2)); + for (SnapshotsInProgress.Entry entry + : clusterService().state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries()) { + assertThat(entry.state(), is(SnapshotsInProgress.State.ABORTED)); + assertThat(entry.snapshot().getSnapshotId().getName(), not(secondSnapshot)); + } + }, 30L, TimeUnit.SECONDS); + + logger.info("--> stopping current master node"); + internalCluster().stopCurrentMasterNode(); + + unblockNode(repoName, dataNode); + unblockNode(repoName, dataNode2); + + assertAcked(firstDeleteFuture.get()); + assertAcked(deleteAllSnapshots.get()); + expectThrows(SnapshotException.class, snapshotThreeFuture::actionGet); + + logger.info("--> verify that all snapshots are gone and no more work is left in the cluster state"); + assertBusy(() -> { + assertThat(client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(repoName), empty()); + final ClusterState state = clusterService().state(); + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); + assertThat(snapshotsInProgress.entries(), empty()); + final SnapshotDeletionsInProgress snapshotDeletionsInProgress = state.custom(SnapshotDeletionsInProgress.TYPE); + assertThat(snapshotDeletionsInProgress.getEntries(), empty()); + }, 30L, TimeUnit.SECONDS); + } + + public void testAssertMultipleSnapshotsAndPrimaryFailOver() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + + final String testIndex = "index-one"; + createIndex(testIndex, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1).build()); + ensureYellow(testIndex); + indexDoc(testIndex, "some_id", "foo", "bar"); + + blockDataNode(repoName, dataNode); + final ActionFuture firstSnapshotResponse = startFullSnapshotFromMasterClient(repoName, "snapshot-one"); + waitForBlock(dataNode, repoName, TimeValue.timeValueSeconds(30L)); + + internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + ensureGreen(testIndex); + + final String secondSnapshot = "snapshot-two"; + final ActionFuture secondSnapshotResponse = startFullSnapshotFromMasterClient(repoName, secondSnapshot); + + internalCluster().restartNode(dataNode, InternalTestCluster.EMPTY_CALLBACK); + + assertThat(firstSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); + assertThat(secondSnapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); + } + + public void testQueuedDeletesWithFailures() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + blockMasterFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture firstDeleteFuture = startDelete(repoName, "*"); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNSnapshotsInProgress(1); + + final ActionFuture secondDeleteFuture = startDelete(repoName, "*"); + awaitNDeletionsInProgress(2); + + unblockNode(repoName, masterNode); + expectThrows(UncategorizedExecutionException.class, firstDeleteFuture::actionGet); + + // Second delete works out cleanly since the repo is unblocked now + assertThat(secondDeleteFuture.get().isAcknowledged(), is(true)); + // Snapshot should have been aborted + assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); + + assertThat(client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(repoName), empty()); + } + + public void testQueuedDeletesWithOverlap() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final ActionFuture firstDeleteFuture = startAndBlockOnDeleteSnapshot(repoName, "*"); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, "snapshot-queued"); + awaitNSnapshotsInProgress(1); + + final ActionFuture secondDeleteFuture = startDelete(repoName, "*"); + awaitNDeletionsInProgress(2); + + unblockNode(repoName, masterNode); + assertThat(firstDeleteFuture.get().isAcknowledged(), is(true)); + + // Second delete works out cleanly since the repo is unblocked now + assertThat(secondDeleteFuture.get().isAcknowledged(), is(true)); + // Snapshot should have been aborted + assertThat(snapshotFuture.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); + + assertThat(client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots(repoName), empty()); + } + + public void testQueuedOperationsOnMasterRestart() throws Exception { + internalCluster().startMasterOnlyNodes(3); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + startAndBlockOnDeleteSnapshot(repoName, "*"); + + client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-three").setWaitForCompletion(false).get(); + + startDelete(repoName, "*"); + awaitNDeletionsInProgress(2); + + internalCluster().stopCurrentMasterNode(); + ensureStableCluster(3); + + awaitNoMoreRunningOperations(); + } + + public void testQueuedOperationsOnMasterDisconnect() throws Exception { + internalCluster().startMasterOnlyNodes(3); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final String masterNode = internalCluster().getMasterName(); + final NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT); + internalCluster().setDisruptionScheme(networkDisruption); + + blockNodeOnAnyFiles(repoName, masterNode); + ActionFuture firstDeleteFuture = client(masterNode).admin().cluster() + .prepareDeleteSnapshot(repoName, "*").execute(); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + final ActionFuture createThirdSnapshot = client(masterNode).admin().cluster() + .prepareCreateSnapshot(repoName, "snapshot-three").setWaitForCompletion(true).execute(); + awaitNSnapshotsInProgress(1); + + final ActionFuture secondDeleteFuture = + client(masterNode).admin().cluster().prepareDeleteSnapshot(repoName, "*").execute(); + awaitNDeletionsInProgress(2); + + networkDisruption.startDisrupting(); + ensureStableCluster(3, dataNode); + unblockNode(repoName, masterNode); + networkDisruption.stopDisrupting(); + + logger.info("--> make sure all failing requests get a response"); + expectThrows(RepositoryException.class, firstDeleteFuture::actionGet); + expectThrows(RepositoryException.class, secondDeleteFuture::actionGet); + expectThrows(SnapshotException.class, createThirdSnapshot::actionGet); + + awaitNoMoreRunningOperations(); + } + + public void testQueuedOperationsOnMasterDisconnectAndRepoFailure() throws Exception { + internalCluster().startMasterOnlyNodes(3); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final String masterNode = internalCluster().getMasterName(); + final NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT); + internalCluster().setDisruptionScheme(networkDisruption); + + blockMasterFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture firstFailedSnapshotFuture = + startFullSnapshotFromMasterClient(repoName, "failing-snapshot-1"); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + final ActionFuture secondFailedSnapshotFuture = + startFullSnapshotFromMasterClient(repoName, "failing-snapshot-2"); + awaitNSnapshotsInProgress(2); + + final ActionFuture failedDeleteFuture = + client(masterNode).admin().cluster().prepareDeleteSnapshot(repoName, "*").execute(); + awaitNDeletionsInProgress(1); + + networkDisruption.startDisrupting(); + ensureStableCluster(3, dataNode); + unblockNode(repoName, masterNode); + networkDisruption.stopDisrupting(); + + logger.info("--> make sure all failing requests get a response"); + expectThrows(SnapshotException.class, firstFailedSnapshotFuture::actionGet); + expectThrows(SnapshotException.class, secondFailedSnapshotFuture::actionGet); + expectThrows(RepositoryException.class, failedDeleteFuture::actionGet); + + awaitNoMoreRunningOperations(); + } + + public void testQueuedOperationsAndBrokenRepoOnMasterFailOver() throws Exception { + disableRepoConsistencyCheck("This test corrupts the repository on purpose"); + + internalCluster().startMasterOnlyNodes(3); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final long generation = getRepositoryData(repoName).getGenId(); + + startAndBlockOnDeleteSnapshot(repoName, "*"); + + corruptIndexN(repoPath, generation); + + client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-three").setWaitForCompletion(false).get(); + + final ActionFuture deleteFuture = startDeleteFromNonMasterClient(repoName, "*"); + awaitNDeletionsInProgress(2); + + internalCluster().stopCurrentMasterNode(); + ensureStableCluster(3); + + awaitNoMoreRunningOperations(); + expectThrows(RepositoryException.class, deleteFuture::actionGet); + } + + public void testQueuedSnapshotOperationsAndBrokenRepoOnMasterFailOver() throws Exception { + disableRepoConsistencyCheck("This test corrupts the repository on purpose"); + + internalCluster().startMasterOnlyNodes(3); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final long generation = getRepositoryData(repoName).getGenId(); + final String masterNode = internalCluster().getMasterName(); + blockNodeOnAnyFiles(repoName, masterNode); + final ActionFuture snapshotThree = startFullSnapshotFromNonMasterClient(repoName, "snapshot-three"); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + corruptIndexN(repoPath, generation); + + final ActionFuture snapshotFour = startFullSnapshotFromNonMasterClient(repoName, "snapshot-four"); + internalCluster().stopCurrentMasterNode(); + ensureStableCluster(3); + + awaitNoMoreRunningOperations(); + expectThrows(ElasticsearchException.class, snapshotThree::actionGet); + expectThrows(ElasticsearchException.class, snapshotFour::actionGet); + } + + public void testQueuedSnapshotOperationsAndBrokenRepoOnMasterFailOver2() throws Exception { + disableRepoConsistencyCheck("This test corrupts the repository on purpose"); + + internalCluster().startMasterOnlyNodes(3); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final long generation = getRepositoryData(repoName).getGenId(); + final String masterNode = internalCluster().getMasterName(); + blockMasterFromFinalizingSnapshotOnIndexFile(repoName); + final ActionFuture snapshotThree = startFullSnapshotFromNonMasterClient(repoName, "snapshot-three"); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + corruptIndexN(repoPath, generation); + + final ActionFuture snapshotFour = startFullSnapshotFromNonMasterClient(repoName, "snapshot-four"); + awaitNSnapshotsInProgress(2); + + final NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT); + internalCluster().setDisruptionScheme(networkDisruption); + networkDisruption.startDisrupting(); + ensureStableCluster(3, dataNode); + unblockNode(repoName, masterNode); + networkDisruption.stopDisrupting(); + awaitNoMoreRunningOperations(); + expectThrows(ElasticsearchException.class, snapshotThree::actionGet); + expectThrows(ElasticsearchException.class, snapshotFour::actionGet); + } + + public void testQueuedSnapshotOperationsAndBrokenRepoOnMasterFailOverMultipleRepos() throws Exception { + disableRepoConsistencyCheck("This test corrupts the repository on purpose"); + + internalCluster().startMasterOnlyNodes(3, LARGE_SNAPSHOT_POOL_SETTINGS); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + + final String masterNode = internalCluster().getMasterName(); + + final String blockedRepoName = "repo-blocked"; + createRepository(blockedRepoName, "mock"); + createNSnapshots(blockedRepoName, randomIntBetween(1, 5)); + blockNodeOnAnyFiles(blockedRepoName, masterNode); + final ActionFuture deleteFuture = startDeleteFromNonMasterClient(blockedRepoName, "*"); + waitForBlock(masterNode, blockedRepoName, TimeValue.timeValueSeconds(30L)); + final ActionFuture createBlockedSnapshot = + startFullSnapshotFromNonMasterClient(blockedRepoName, "queued-snapshot"); + + final long generation = getRepositoryData(repoName).getGenId(); + blockNodeOnAnyFiles(repoName, masterNode); + final ActionFuture snapshotThree = startFullSnapshotFromNonMasterClient(repoName, "snapshot-three"); + waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(30L)); + + corruptIndexN(repoPath, generation); + + final ActionFuture snapshotFour = startFullSnapshotFromNonMasterClient(repoName, "snapshot-four"); + internalCluster().stopCurrentMasterNode(); + ensureStableCluster(3); + + awaitNoMoreRunningOperations(); + expectThrows(ElasticsearchException.class, snapshotThree::actionGet); + expectThrows(ElasticsearchException.class, snapshotFour::actionGet); + assertAcked(deleteFuture.get()); + expectThrows(ElasticsearchException.class, createBlockedSnapshot::actionGet); + } + + public void testMultipleSnapshotsQueuedAfterDelete() throws Exception { + final String masterNode = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(1, 5)); + + final ActionFuture deleteFuture = startAndBlockOnDeleteSnapshot(repoName, "*"); + final ActionFuture snapshotThree = startFullSnapshot(repoName, "snapshot-three"); + final ActionFuture snapshotFour = startFullSnapshot(repoName, "snapshot-four"); + + unblockNode(repoName, masterNode); + + assertSuccessful(snapshotThree); + assertSuccessful(snapshotFour); + assertAcked(deleteFuture.get()); + } + + public void testQueuedSnapshotsWaitingForShardReady() throws Exception { + internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNodes(2); + final String repoName = "test-repo"; + createRepository(repoName, "fs"); + + final String testIndex = "test-idx"; + // Create index on two nodes and make sure each node has a primary by setting no replicas + assertAcked(prepareCreate(testIndex, 2, indexSettingsNoReplicas(between(2, 10)))); + + ensureGreen(testIndex); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + indexDoc(testIndex, Integer.toString(i), "foo", "bar" + i); + } + refresh(); + assertThat(client().prepareSearch(testIndex).setSize(0).get().getHits().getTotalHits().value, equalTo(100L)); + + logger.info("--> start relocations"); + allowNodes(testIndex, 1); + + logger.info("--> wait for relocations to start"); + assertBusy(() -> assertThat( + client().admin().cluster().prepareHealth(testIndex).execute().actionGet().getRelocatingShards(), greaterThan(0)), + 1L, TimeUnit.MINUTES); + + logger.info("--> start two snapshots"); + final String snapshotOne = "snap-1"; + final String snapshotTwo = "snap-2"; + final ActionFuture snapOneResponse = client().admin().cluster() + .prepareCreateSnapshot(repoName, snapshotOne).setWaitForCompletion(false).setIndices(testIndex).execute(); + final ActionFuture snapTwoResponse = client().admin().cluster() + .prepareCreateSnapshot(repoName, snapshotTwo).setWaitForCompletion(false).setIndices(testIndex).execute(); + + snapOneResponse.get(); + snapTwoResponse.get(); + logger.info("--> wait for snapshot to complete"); + for (String snapshot : Arrays.asList(snapshotOne, snapshotTwo)) { + SnapshotInfo snapshotInfo = waitForCompletion(repoName, snapshot, TimeValue.timeValueSeconds(600)); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.shardFailures().size(), equalTo(0)); + } + } + + public void testBackToBackQueuedDeletes() throws Exception { + final String masterName = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + final List snapshots = createNSnapshots(repoName, 2); + final String snapshotOne = snapshots.get(0); + final String snapshotTwo = snapshots.get(1); + + final ActionFuture deleteSnapshotOne = startAndBlockOnDeleteSnapshot(repoName, snapshotOne); + final ActionFuture deleteSnapshotTwo = startDelete(repoName, snapshotTwo); + awaitNDeletionsInProgress(2); + + unblockNode(repoName, masterName); + assertAcked(deleteSnapshotOne.get()); + assertAcked(deleteSnapshotTwo.get()); + + final RepositoryData repositoryData = getRepositoryData(repoName); + assertThat(repositoryData.getSnapshotIds(), empty()); + // Two snapshots and two distinct delete operations move us 4 steps from -1 to 3 + assertThat(repositoryData.getGenId(), is(3L)); + } + + public void testQueuedOperationsAfterFinalizationFailure() throws Exception { + internalCluster().startMasterOnlyNodes(3); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + + final List snapshotNames = createNSnapshots(repoName, randomIntBetween(2, 5)); + + final ActionFuture snapshotThree = startAndBlockFailingFullSnapshot(repoName, "snap-other"); + + final String masterName = internalCluster().getMasterName(); + + final String snapshotOne = snapshotNames.get(0); + final ActionFuture deleteSnapshotOne = startDelete(repoName, snapshotOne); + awaitNDeletionsInProgress(1); + + unblockNode(repoName, masterName); + + expectThrows(SnapshotException.class, snapshotThree::actionGet); + assertAcked(deleteSnapshotOne.get()); + } + + public void testStartDeleteDuringFinalizationCleanup() throws Exception { + final String masterName = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + createNSnapshots(repoName, randomIntBetween(1, 5)); + final String snapshotName = "snap-name"; + blockMasterFromDeletingIndexNFile(repoName); + final ActionFuture snapshotFuture = startFullSnapshot(repoName, snapshotName); + waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L)); + final ActionFuture deleteFuture = startDelete(repoName, snapshotName); + awaitNDeletionsInProgress(1); + unblockNode(repoName, masterName); + assertSuccessful(snapshotFuture); + assertAcked(deleteFuture.get(30L, TimeUnit.SECONDS)); + } + + public void testEquivalentDeletesAreDeduplicated() throws Exception { + final String masterName = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + createNSnapshots(repoName, randomIntBetween(1, 5)); + + blockMasterFromDeletingIndexNFile(repoName); + final int deletes = randomIntBetween(2, 10); + final List> deleteResponses = new ArrayList<>(deletes); + for (int i = 0; i < deletes; ++i) { + deleteResponses.add(client().admin().cluster().prepareDeleteSnapshot(repoName, "*").execute()); + } + waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L)); + awaitNDeletionsInProgress(1); + for (ActionFuture deleteResponse : deleteResponses) { + assertFalse(deleteResponse.isDone()); + } + awaitNDeletionsInProgress(1); + unblockNode(repoName, masterName); + for (ActionFuture deleteResponse : deleteResponses) { + assertAcked(deleteResponse.get()); + } + } + + public void testMasterFailoverOnFinalizationLoop() throws Exception { + internalCluster().startMasterOnlyNodes(3); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + final NetworkDisruption networkDisruption = isolateMasterDisruption(NetworkDisruption.DISCONNECT); + internalCluster().setDisruptionScheme(networkDisruption); + + final List snapshotNames = createNSnapshots(repoName, randomIntBetween(2, 5)); + final String masterName = internalCluster().getMasterName(); + blockMasterFromDeletingIndexNFile(repoName); + final ActionFuture snapshotThree = startFullSnapshotFromMasterClient(repoName, "snap-other"); + waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L)); + + final String snapshotOne = snapshotNames.get(0); + final ActionFuture deleteSnapshotOne = startDelete(repoName, snapshotOne); + awaitNDeletionsInProgress(1); + networkDisruption.startDisrupting(); + ensureStableCluster(3, dataNode); + + unblockNode(repoName, masterName); + networkDisruption.stopDisrupting(); + ensureStableCluster(4); + + assertSuccessful(snapshotThree); + try { + deleteSnapshotOne.actionGet(); + } catch (RepositoryException re) { + // ignored + } + awaitNoMoreRunningOperations(); + } + + public void testStatusMultipleSnapshotsMultipleRepos() throws Exception { + internalCluster().startMasterOnlyNode(); + // We're blocking a some of the snapshot threads when we block the first repo below so we have to make sure we have enough threads + // left for the second concurrent snapshot. + final String dataNode = startDataNodeWithLargeSnapshotPool(); + final String blockedRepoName = "test-repo-blocked-1"; + final String otherBlockedRepoName = "test-repo-blocked-2"; + createRepository(blockedRepoName, "mock"); + createRepository(otherBlockedRepoName, "mock"); + createIndexWithContent("test-index"); + + final ActionFuture createSlowFuture1 = + startFullSnapshotBlockedOnDataNode("blocked-snapshot", blockedRepoName, dataNode); + final ActionFuture createSlowFuture2 = + startFullSnapshotBlockedOnDataNode("blocked-snapshot-2", blockedRepoName, dataNode); + final ActionFuture createSlowFuture3 = + startFullSnapshotBlockedOnDataNode("other-blocked-snapshot", otherBlockedRepoName, dataNode); + awaitNSnapshotsInProgress(3); + + assertSnapshotStatusCountOnRepo("_all", 3); + assertSnapshotStatusCountOnRepo(blockedRepoName, 2); + assertSnapshotStatusCountOnRepo(otherBlockedRepoName, 1); + + unblockNode(blockedRepoName, dataNode); + awaitNSnapshotsInProgress(1); + assertSnapshotStatusCountOnRepo("_all", 1); + assertSnapshotStatusCountOnRepo(blockedRepoName, 0); + assertSnapshotStatusCountOnRepo(otherBlockedRepoName, 1); + + unblockNode(otherBlockedRepoName, dataNode); + assertSuccessful(createSlowFuture1); + assertSuccessful(createSlowFuture2); + assertSuccessful(createSlowFuture3); + } + + public void testInterleavedAcrossMultipleRepos() throws Exception { + internalCluster().startMasterOnlyNode(); + // We're blocking a some of the snapshot threads when we block the first repo below so we have to make sure we have enough threads + // left for the second concurrent snapshot. + final String dataNode = startDataNodeWithLargeSnapshotPool(); + final String blockedRepoName = "test-repo-blocked-1"; + final String otherBlockedRepoName = "test-repo-blocked-2"; + createRepository(blockedRepoName, "mock"); + createRepository(otherBlockedRepoName, "mock"); + createIndexWithContent("test-index"); + + final ActionFuture createSlowFuture1 = + startFullSnapshotBlockedOnDataNode("blocked-snapshot", blockedRepoName, dataNode); + final ActionFuture createSlowFuture2 = + startFullSnapshotBlockedOnDataNode("blocked-snapshot-2", blockedRepoName, dataNode); + final ActionFuture createSlowFuture3 = + startFullSnapshotBlockedOnDataNode("other-blocked-snapshot", otherBlockedRepoName, dataNode); + awaitNSnapshotsInProgress(3); + unblockNode(blockedRepoName, dataNode); + unblockNode(otherBlockedRepoName, dataNode); + + assertSuccessful(createSlowFuture1); + assertSuccessful(createSlowFuture2); + assertSuccessful(createSlowFuture3); + } + + public void testMasterFailoverAndMultipleQueuedUpSnapshotsAcrossTwoRepos() throws Exception { + disableRepoConsistencyCheck("This test corrupts the repository on purpose"); + + internalCluster().startMasterOnlyNodes(3, LARGE_SNAPSHOT_POOL_SETTINGS); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final String otherRepoName = "other-test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", repoPath); + createRepository(otherRepoName, "mock"); + createIndexWithContent("index-one"); + createNSnapshots(repoName, randomIntBetween(2, 5)); + final int countOtherRepo = randomIntBetween(2, 5); + createNSnapshots(otherRepoName, countOtherRepo); + + corruptIndexN(repoPath, getRepositoryData(repoName).getGenId()); + + blockMasterFromFinalizingSnapshotOnIndexFile(repoName); + blockMasterFromFinalizingSnapshotOnIndexFile(otherRepoName); + + client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-blocked-1").setWaitForCompletion(false).get(); + client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-blocked-2").setWaitForCompletion(false).get(); + client().admin().cluster().prepareCreateSnapshot(otherRepoName, "snapshot-other-blocked-1").setWaitForCompletion(false).get(); + client().admin().cluster().prepareCreateSnapshot(otherRepoName, "snapshot-other-blocked-2").setWaitForCompletion(false).get(); + + awaitNSnapshotsInProgress(4); + final String initialMaster = internalCluster().getMasterName(); + waitForBlock(initialMaster, repoName, TimeValue.timeValueSeconds(30L)); + waitForBlock(initialMaster, otherRepoName, TimeValue.timeValueSeconds(30L)); + + internalCluster().stopCurrentMasterNode(); + ensureStableCluster(3, dataNode); + awaitNoMoreRunningOperations(); + + final RepositoryData repositoryData = getRepositoryData(otherRepoName); + assertThat(repositoryData.getSnapshotIds(), hasSize(countOtherRepo + 2)); + } + + public void testConcurrentOperationsLimit() throws Exception { + final String masterName = internalCluster().startMasterOnlyNode(); + internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "mock"); + createIndexWithContent("index-test"); + + final int limitToTest = randomIntBetween(1, 3); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put( + SnapshotsService.MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING.getKey(), limitToTest).build()).get()); + + final List snapshotNames = createNSnapshots(repoName, limitToTest + 1); + blockNodeOnAnyFiles(repoName, masterName); + int blockedSnapshots = 0; + boolean blockedDelete = false; + final List> snapshotFutures = new ArrayList<>(); + ActionFuture deleteFuture = null; + for (int i = 0; i < limitToTest; ++i) { + if (blockedDelete || randomBoolean()) { + snapshotFutures.add(startFullSnapshot(repoName, "snap-" + i)); + ++blockedSnapshots; + } else { + blockedDelete = true; + deleteFuture = startDelete(repoName, randomFrom(snapshotNames)); + } + } + awaitNSnapshotsInProgress(blockedSnapshots); + if (blockedDelete) { + awaitNDeletionsInProgress(1); + } + waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L)); + + final String expectedFailureMessage = "Cannot start another operation, already running [" + limitToTest + + "] operations and the current limit for concurrent snapshot operations is set to [" + limitToTest + "]"; + final ConcurrentSnapshotExecutionException csen1 = expectThrows(ConcurrentSnapshotExecutionException.class, + () -> client().admin().cluster().prepareCreateSnapshot(repoName, "expected-to-fail").execute().actionGet()); + assertThat(csen1.getMessage(), containsString(expectedFailureMessage)); + if (blockedDelete == false || limitToTest == 1) { + final ConcurrentSnapshotExecutionException csen2 = expectThrows(ConcurrentSnapshotExecutionException.class, + () -> client().admin().cluster().prepareDeleteSnapshot(repoName, "*").execute().actionGet()); + assertThat(csen2.getMessage(), containsString(expectedFailureMessage)); + } + + unblockNode(repoName, masterName); + if (deleteFuture != null) { + assertAcked(deleteFuture.get()); + } + for (ActionFuture snapshotFuture : snapshotFutures) { + assertSuccessful(snapshotFuture); + } + } + + public void testConcurrentSnapshotWorksWithOldVersionRepo() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + final Path repoPath = randomRepoPath(); + createRepository(repoName, "mock", Settings.builder().put(BlobStoreRepository.CACHE_REPOSITORY_DATA.getKey(), false) + .put("location", repoPath)); + initWithSnapshotVersion(repoName, repoPath, SnapshotsService.OLD_SNAPSHOT_FORMAT); + + createIndexWithContent("index-slow"); + + final ActionFuture createSlowFuture = + startFullSnapshotBlockedOnDataNode("slow-snapshot", repoName, dataNode); + + final String dataNode2 = internalCluster().startDataOnlyNode(); + ensureStableCluster(3); + final String indexFast = "index-fast"; + createIndexWithContent(indexFast, dataNode2, dataNode); + + final ActionFuture createFastSnapshot = + client().admin().cluster().prepareCreateSnapshot(repoName, "fast-snapshot").setWaitForCompletion(true).execute(); + + assertThat(createSlowFuture.isDone(), is(false)); + unblockNode(repoName, dataNode); + + assertSuccessful(createFastSnapshot); + assertSuccessful(createSlowFuture); + + final RepositoryData repositoryData = getRepositoryData(repoName); + assertThat(repositoryData.shardGenerations(), is(ShardGenerations.EMPTY)); + } + + private static String startDataNodeWithLargeSnapshotPool() { + return internalCluster().startDataOnlyNode(LARGE_SNAPSHOT_POOL_SETTINGS); + } + + private static void assertSnapshotStatusCountOnRepo(String otherBlockedRepoName, int count) { + final SnapshotsStatusResponse snapshotsStatusResponse = + client().admin().cluster().prepareSnapshotStatus(otherBlockedRepoName).get(); + final List snapshotStatuses = snapshotsStatusResponse.getSnapshots(); + assertThat(snapshotStatuses, hasSize(count)); + } + + private List createNSnapshots(String repoName, int count) { + final List snapshotNames = new ArrayList<>(count); + final String prefix = "snap-" + UUIDs.randomBase64UUID(random()).toLowerCase(Locale.ROOT) + "-"; + for (int i = 0; i < count; i++) { + final String name = prefix + i; + createFullSnapshot(repoName, name); + snapshotNames.add(name); + } + logger.info("--> created {} in [{}]", snapshotNames, repoName); + return snapshotNames; + } + + private void awaitNoMoreRunningOperations() throws Exception { + awaitNoMoreRunningOperations(internalCluster().getMasterName()); + } + + private ActionFuture startDeleteFromNonMasterClient(String repoName, String snapshotName) { + logger.info("--> deleting snapshot [{}] from repo [{}] from non master client", snapshotName, repoName); + return internalCluster().nonMasterClient().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).execute(); + } + + private ActionFuture startDelete(String repoName, String snapshotName) { + logger.info("--> deleting snapshot [{}] from repo [{}]", snapshotName, repoName); + return client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).execute(); + } + + private ActionFuture startFullSnapshotFromNonMasterClient(String repoName, String snapshotName) { + logger.info("--> creating full snapshot [{}] to repo [{}] from non master client", snapshotName, repoName); + return internalCluster().nonMasterClient().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true).execute(); + } + + private ActionFuture startFullSnapshotFromMasterClient(String repoName, String snapshotName) { + logger.info("--> creating full snapshot [{}] to repo [{}] from master client", snapshotName, repoName); + return internalCluster().masterClient().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true).execute(); + } + + private ActionFuture startFullSnapshot(String repoName, String snapshotName) { + logger.info("--> creating full snapshot [{}] to repo [{}]", snapshotName, repoName); + return client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).execute(); + } + + private void awaitClusterState(Predicate statePredicate) throws Exception { + awaitClusterState(internalCluster().getMasterName(), statePredicate); + } + + // Large snapshot pool settings to set up nodes for tests involving multiple repositories that need to have enough + // threads so that blocking some threads on one repository doesn't block other repositories from doing work + private static final Settings LARGE_SNAPSHOT_POOL_SETTINGS = Settings.builder() + .put("thread_pool.snapshot.core", 5).put("thread_pool.snapshot.max", 5).build(); + + private static final Settings SINGLE_SHARD_NO_REPLICA = indexSettingsNoReplicas(1).build(); + + private void createIndexWithContent(String indexName) { + createIndexWithContent(indexName, SINGLE_SHARD_NO_REPLICA); + } + + private void createIndexWithContent(String indexName, String nodeInclude, String nodeExclude) { + createIndexWithContent(indexName, indexSettingsNoReplicas(1) + .put("index.routing.allocation.include._name", nodeInclude) + .put("index.routing.allocation.exclude._name", nodeExclude).build()); + } + + private void createIndexWithContent(String indexName, Settings indexSettings) { + logger.info("--> creating index [{}]", indexName); + createIndex(indexName, indexSettings); + ensureGreen(indexName); + indexDoc(indexName, "some_id", "foo", "bar"); + } + + private static boolean snapshotHasCompletedShard(String snapshot, SnapshotsInProgress snapshotsInProgress) { + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { + if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) { + for (ObjectCursor shard : entry.shards().values()) { + if (shard.value.state().completed()) { + return true; + } + } + } + } + return false; + } + + private static SnapshotInfo assertSuccessful(ActionFuture future) throws Exception { + final SnapshotInfo snapshotInfo = future.get().getSnapshotInfo(); + assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); + return snapshotInfo; + } + + private void corruptIndexN(Path repoPath, long generation) throws IOException { + logger.info("--> corrupting [index-{}] in [{}]", generation, repoPath); + Path indexNBlob = repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + generation); + assertFileExists(indexNBlob); + Files.write(indexNBlob, randomByteArrayOfLength(1), StandardOpenOption.TRUNCATE_EXISTING); + } + + private void awaitNDeletionsInProgress(int count) throws Exception { + logger.info("--> wait for [{}] deletions to show up in the cluster state", count); + awaitClusterState(state -> + state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY).getEntries().size() == count); + } + + private void awaitNSnapshotsInProgress(int count) throws Exception { + logger.info("--> wait for [{}] snapshots to show up in the cluster state", count); + awaitClusterState(state -> + state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries().size() == count); + } + + private static List currentSnapshots(String repoName) { + return client().admin().cluster().prepareGetSnapshots(repoName).setSnapshots(GetSnapshotsRequest.CURRENT_SNAPSHOT) + .get().getSnapshots(repoName); + } + + private ActionFuture startAndBlockOnDeleteSnapshot(String repoName, String snapshotName) + throws InterruptedException { + final String masterName = internalCluster().getMasterName(); + blockNodeOnAnyFiles(repoName, masterName); + final ActionFuture fut = startDelete(repoName, snapshotName); + waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L)); + return fut; + } + + private ActionFuture startAndBlockFailingFullSnapshot(String blockedRepoName, String snapshotName) + throws InterruptedException { + blockMasterFromFinalizingSnapshotOnIndexFile(blockedRepoName); + final ActionFuture fut = startFullSnapshot(blockedRepoName, snapshotName); + waitForBlock(internalCluster().getMasterName(), blockedRepoName, TimeValue.timeValueSeconds(30L)); + return fut; + } + + private ActionFuture startFullSnapshotBlockedOnDataNode(String snapshotName, String repoName, String dataNode) + throws InterruptedException { + blockDataNode(repoName, dataNode); + final ActionFuture fut = startFullSnapshot(repoName, snapshotName); + waitForBlock(dataNode, repoName, TimeValue.timeValueSeconds(30L)); + return fut; + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 37eed98ad8532..0555434a03584 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -1250,6 +1250,19 @@ public void onRequestSent(DiscoveryNode node, long requestId, String action, Tra assertThat(snapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED)); } + public void testPartialSnapshotAllShardsMissing() throws Exception { + internalCluster().startMasterOnlyNode(); + final String dataNode = internalCluster().startDataOnlyNode(); + final String repoName = "test-repo"; + createRepository(repoName, "fs"); + createIndex("some-index"); + stopNode(dataNode); + ensureStableCluster(1); + final CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(repoName, "test-snap") + .setPartial(true).setWaitForCompletion(true).get(); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), is(SnapshotState.PARTIAL)); + } + private long calculateTotalFilesSize(List files) { return files.stream().mapToLong(f -> { try { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java deleted file mode 100644 index 9f437cdf70d35..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MinThreadsSnapshotRestoreIT.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.snapshots; - -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.repositories.RepositoriesService; -import org.elasticsearch.snapshots.mockstore.MockRepository; - -import java.util.Collection; -import java.util.Collections; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; - -/** - * Tests for snapshot/restore that require at least 2 threads available - * in the thread pool (for example, tests that use the mock repository that - * block on master). - */ -public class MinThreadsSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)) - .put("thread_pool.snapshot.core", 2) - .put("thread_pool.snapshot.max", 2) - .build(); - } - - @Override - protected Collection> nodePlugins() { - return Collections.singleton(MockRepository.Plugin.class); - } - - public void testConcurrentSnapshotDeletionsNotAllowed() throws Exception { - logger.info("--> creating repository"); - final String repo = "test-repo"; - assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings( - Settings.builder() - .put("location", randomRepoPath()) - .put("random", randomAlphaOfLength(10)) - .put("wait_after_unblock", 200)).get()); - - logger.info("--> snapshot twice"); - final String index = "test-idx1"; - assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); - for (int i = 0; i < 10; i++) { - indexDoc(index, Integer.toString(i), "foo", "bar" + i); - } - refresh(); - final String snapshot1 = "test-snap1"; - client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get(); - final String index2 = "test-idx2"; - assertAcked(prepareCreate(index2, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); - for (int i = 0; i < 10; i++) { - indexDoc(index2, Integer.toString(i), "foo", "bar" + i); - } - refresh(); - final String snapshot2 = "test-snap2"; - client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get(); - - String blockedNode = internalCluster().getMasterName(); - ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true); - logger.info("--> start deletion of first snapshot"); - ActionFuture future = - client().admin().cluster().prepareDeleteSnapshot(repo, snapshot2).execute(); - logger.info("--> waiting for block to kick in on node [{}]", blockedNode); - waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10)); - - logger.info("--> try deleting the second snapshot, should fail because the first deletion is in progress"); - try { - client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get(); - fail("should not be able to delete snapshots concurrently"); - } catch (ConcurrentSnapshotExecutionException e) { - assertThat(e.getMessage(), containsString("cannot delete - another snapshot is currently being deleted")); - } - - logger.info("--> unblocking blocked node [{}]", blockedNode); - unblockNode(repo, blockedNode); - - logger.info("--> wait until first snapshot is finished"); - assertAcked(future.actionGet()); - - logger.info("--> delete second snapshot, which should now work"); - client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).get(); - assertTrue(client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).isEmpty()); - } - - public void testSnapshottingWithInProgressDeletionNotAllowed() throws Exception { - logger.info("--> creating repository"); - final String repo = "test-repo"; - assertAcked(client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings( - Settings.builder() - .put("location", randomRepoPath()) - .put("random", randomAlphaOfLength(10)) - .put("wait_after_unblock", 200)).get()); - - logger.info("--> snapshot"); - final String index = "test-idx"; - assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0))); - for (int i = 0; i < 10; i++) { - indexDoc(index, Integer.toString(i), "foo", "bar" + i); - } - refresh(); - final String snapshot1 = "test-snap1"; - client().admin().cluster().prepareCreateSnapshot(repo, snapshot1).setWaitForCompletion(true).get(); - - String blockedNode = internalCluster().getMasterName(); - ((MockRepository)internalCluster().getInstance(RepositoriesService.class, blockedNode).repository(repo)).blockOnDataFiles(true); - logger.info("--> start deletion of snapshot"); - ActionFuture future = client().admin().cluster().prepareDeleteSnapshot(repo, snapshot1).execute(); - logger.info("--> waiting for block to kick in on node [{}]", blockedNode); - waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10)); - - logger.info("--> try creating a second snapshot, should fail because the deletion is in progress"); - final String snapshot2 = "test-snap2"; - try { - client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get(); - fail("should not be able to create a snapshot while another is being deleted"); - } catch (ConcurrentSnapshotExecutionException e) { - assertThat(e.getMessage(), containsString("cannot snapshot while a snapshot deletion is in-progress")); - } - - logger.info("--> unblocking blocked node [{}]", blockedNode); - unblockNode(repo, blockedNode); - - logger.info("--> wait until snapshot deletion is finished"); - assertAcked(future.actionGet()); - - logger.info("--> creating second snapshot, which should now work"); - client().admin().cluster().prepareCreateSnapshot(repo, snapshot2).setWaitForCompletion(true).get(); - assertEquals(1, client().admin().cluster().prepareGetSnapshots(repo).setSnapshots("_all").get().getSnapshots(repo).size()); - } -} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 83ea4b9c3b10c..5546e5f188113 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -35,7 +34,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -50,7 +48,6 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotsInProgress.State; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; @@ -75,7 +72,6 @@ import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.DataStreamIT; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.ingest.IngestTestPlugin; @@ -127,7 +123,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertRequestBuilderThrows; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -2329,56 +2324,6 @@ public void testRecreateBlocksOnRestore() throws Exception { } } - public void testDeleteDataStreamDuringSnapshot() throws Exception { - Client client = client(); - - createRepository("test-repo", "mock", Settings.builder() - .put("location", randomRepoPath()).put("compress", randomBoolean()) - .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) - .put("block_on_data", true)); - - - String dataStream = "datastream"; - DataStreamIT.putComposableIndexTemplate("dst", "@timestamp", List.of(dataStream)); - - logger.info("--> indexing some data"); - for (int i = 0; i < 100; i++) { - client.prepareIndex(dataStream) - .setOpType(DocWriteRequest.OpType.CREATE) - .setId(Integer.toString(i)) - .setSource(Collections.singletonMap("@timestamp", "2020-12-12")) - .execute().actionGet(); - } - refresh(); - assertDocCount(dataStream, 100L); - - logger.info("--> snapshot"); - ActionFuture future = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap") - .setIndices(dataStream).setWaitForCompletion(true).setPartial(false).execute(); - logger.info("--> wait for block to kick in"); - waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1)); - - // non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed - try { - logger.info("--> delete index while non-partial snapshot is running"); - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{dataStream})).actionGet(); - fail("Expected deleting index to fail during snapshot"); - } catch (SnapshotInProgressException e) { - assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: ["+dataStream)); - } finally { - logger.info("--> unblock all data nodes"); - unblockAllDataNodes("test-repo"); - } - logger.info("--> waiting for snapshot to finish"); - CreateSnapshotResponse createSnapshotResponse = future.get(); - - logger.info("Snapshot successfully completed"); - SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); - assertThat(snapshotInfo.state(), equalTo((SnapshotState.SUCCESS))); - assertThat(snapshotInfo.dataStreams(), contains(dataStream)); - assertThat(snapshotInfo.indices(), contains(DataStream.getDefaultBackingIndexName(dataStream, 1))); - } - public void testCloseOrDeleteIndexDuringSnapshot() throws Exception { disableRepoConsistencyCheck("This test intentionally leaves a broken repository"); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 1106d6aca46b4..ee295a0edc671 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.discovery.DiscoveryStats; import org.elasticsearch.http.HttpStats; +import org.elasticsearch.index.stats.IndexingPressureStats; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; import org.elasticsearch.ingest.IngestStats; @@ -38,7 +39,6 @@ import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.monitor.process.ProcessStats; import org.elasticsearch.node.AdaptiveSelectionStats; -import org.elasticsearch.script.ScriptCacheStats; import org.elasticsearch.script.ScriptStats; import org.elasticsearch.threadpool.ThreadPoolStats; import org.elasticsearch.transport.TransportStats; @@ -83,9 +83,6 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private ScriptStats scriptStats; - @Nullable - private ScriptCacheStats scriptCacheStats; - @Nullable private DiscoveryStats discoveryStats; @@ -95,6 +92,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment { @Nullable private AdaptiveSelectionStats adaptiveSelectionStats; + @Nullable + private IndexingPressureStats indexingPressureStats; + public NodeStats(StreamInput in) throws IOException { super(in); timestamp = in.readVLong(); @@ -113,10 +113,10 @@ public NodeStats(StreamInput in) throws IOException { discoveryStats = in.readOptionalWriteable(DiscoveryStats::new); ingestStats = in.readOptionalWriteable(IngestStats::new); adaptiveSelectionStats = in.readOptionalWriteable(AdaptiveSelectionStats::new); - if (in.getVersion().onOrAfter(Version.V_7_8_0)) { - scriptCacheStats = in.readOptionalWriteable(ScriptCacheStats::new); + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + indexingPressureStats = in.readOptionalWriteable(IndexingPressureStats::new); } else { - scriptCacheStats = null; + indexingPressureStats = null; } } @@ -128,7 +128,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats @Nullable DiscoveryStats discoveryStats, @Nullable IngestStats ingestStats, @Nullable AdaptiveSelectionStats adaptiveSelectionStats, - @Nullable ScriptCacheStats scriptCacheStats) { + @Nullable IndexingPressureStats indexingPressureStats) { super(node); this.timestamp = timestamp; this.indices = indices; @@ -144,7 +144,7 @@ public NodeStats(DiscoveryNode node, long timestamp, @Nullable NodeIndicesStats this.discoveryStats = discoveryStats; this.ingestStats = ingestStats; this.adaptiveSelectionStats = adaptiveSelectionStats; - this.scriptCacheStats = scriptCacheStats; + this.indexingPressureStats = indexingPressureStats; } public long getTimestamp() { @@ -240,8 +240,8 @@ public AdaptiveSelectionStats getAdaptiveSelectionStats() { } @Nullable - public ScriptCacheStats getScriptCacheStats() { - return scriptCacheStats; + public IndexingPressureStats getIndexingPressureStats() { + return indexingPressureStats; } @Override @@ -266,8 +266,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(discoveryStats); out.writeOptionalWriteable(ingestStats); out.writeOptionalWriteable(adaptiveSelectionStats); - if (out.getVersion().onOrAfter(Version.V_7_8_0)) { - out.writeOptionalWriteable(scriptCacheStats); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + out.writeOptionalWriteable(indexingPressureStats); } } @@ -332,8 +332,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (getAdaptiveSelectionStats() != null) { getAdaptiveSelectionStats().toXContent(builder, params); } - if (getScriptCacheStats() != null) { - getScriptCacheStats().toXContent(builder, params); + if (getIndexingPressureStats() != null) { + getIndexingPressureStats().toXContent(builder, params); } return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index 35e307a3cb393..d0b76291c0e1c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -220,7 +220,8 @@ public enum Metric { DISCOVERY("discovery"), INGEST("ingest"), ADAPTIVE_SELECTION("adaptive_selection"), - SCRIPT_CACHE("script_cache"); + SCRIPT_CACHE("script_cache"), + INDEXING_PRESSURE("indexing_pressure"),; private String metricName; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index 2ce3cb2b03e1a..5446f752a5303 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -157,6 +157,11 @@ public NodesStatsRequestBuilder setScriptCache(boolean scriptCache) { return this; } + public NodesStatsRequestBuilder setIndexingPressure(boolean indexingPressure) { + addOrRemoveMetric(indexingPressure, NodesStatsRequest.Metric.INDEXING_PRESSURE); + return this; + } + /** * Helper method for adding metrics to a request */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index db5011042f4e1..3b9d6c3ad9f6d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -84,7 +84,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest, Task task) NodesStatsRequest.Metric.DISCOVERY.containedIn(metrics), NodesStatsRequest.Metric.INGEST.containedIn(metrics), NodesStatsRequest.Metric.ADAPTIVE_SELECTION.containedIn(metrics), - NodesStatsRequest.Metric.SCRIPT_CACHE.containedIn(metrics)); + NodesStatsRequest.Metric.SCRIPT_CACHE.containedIn(metrics), + NodesStatsRequest.Metric.INDEXING_PRESSURE.containedIn(metrics)); } public static class NodeStatsRequest extends TransportRequest { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index 678c7ca426009..b3fcb36dad24c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -196,6 +196,7 @@ private void buildResponse(SnapshotsInProgress snapshotsInProgress, SnapshotsSta break; case INIT: case WAITING: + case QUEUED: stage = SnapshotIndexShardStage.STARTED; break; case SUCCESS: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 7a8d7a40dcb75..a8440de35d262 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -98,7 +98,7 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest, Task task) { NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false); NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, - true, true, true, false, true, false, false, false, false, false, true, false, false); + true, true, true, false, true, false, false, false, false, false, true, false, false, false); List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java index 646365758a471..57f7c855a6f29 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -21,8 +21,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -53,7 +55,7 @@ private CreateDataStreamAction() { super(NAME, AcknowledgedResponse::new); } - public static class Request extends AcknowledgedRequest { + public static class Request extends AcknowledgedRequest implements IndicesRequest { private final String name; @@ -93,6 +95,16 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(name); } + + @Override + public String[] indices() { + return new String[]{name}; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } } public static class TransportAction extends TransportMasterNodeAction { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java index 6df84a1df466c..bf056896576d0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamAction.java @@ -20,11 +20,12 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -71,9 +72,9 @@ private DeleteDataStreamAction() { super(NAME, AcknowledgedResponse::new); } - public static class Request extends MasterNodeRequest { + public static class Request extends MasterNodeRequest implements IndicesRequest.Replaceable { - private final String[] names; + private String[] names; public Request(String[] names) { this.names = Objects.requireNonNull(names); @@ -111,6 +112,29 @@ public boolean equals(Object o) { public int hashCode() { return Arrays.hashCode(names); } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and + // a data stream's backing indices are retrieved from its metadata + return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; + } } public static class TransportAction extends TransportMasterNodeAction { @@ -176,16 +200,6 @@ static ClusterState removeDataStream(MetadataDeleteIndexService deleteIndexServi snapshottingDataStreams.addAll(SnapshotsService.snapshottingDataStreams(currentState, dataStreams)); } - if (dataStreams.isEmpty()) { - // if only a match-all pattern was specified and no data streams were found because none exist, do not - // fail with data stream missing exception - if (request.names.length == 1 && Regex.isMatchAllPattern(request.names[0])) { - return currentState; - } - throw new ResourceNotFoundException("data_streams matching [" + Strings.arrayToCommaDelimitedString(request.names) + - "] not found"); - } - if (snapshottingDataStreams.isEmpty() == false) { throw new SnapshotInProgressException("Cannot delete data streams that are being snapshotted: " + snapshottingDataStreams + ". Try again after snapshot finishes or cancel the currently running snapshot."); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java index ade719acad6f8..a69ad25c4635a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamAction.java @@ -20,12 +20,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; import org.elasticsearch.cluster.AbstractDiffable; @@ -43,7 +44,6 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -54,10 +54,12 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class GetDataStreamAction extends ActionType { @@ -68,12 +70,12 @@ private GetDataStreamAction() { super(NAME, Response::new); } - public static class Request extends MasterNodeReadRequest { + public static class Request extends MasterNodeReadRequest implements IndicesRequest.Replaceable { - private final String name; + private String[] names; - public Request(String name) { - this.name = name; + public Request(String[] names) { + this.names = names; } @Override @@ -83,13 +85,13 @@ public ActionRequestValidationException validate() { public Request(StreamInput in) throws IOException { super(in); - this.name = in.readOptionalString(); + this.names = in.readOptionalStringArray(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalString(name); + out.writeOptionalStringArray(names); } @Override @@ -97,12 +99,35 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(name, request.name); + return Arrays.equals(names, request.names); } @Override public int hashCode() { - return Objects.hash(name); + return Arrays.hashCode(names); + } + + @Override + public String[] indices() { + return names; + } + + @Override + public IndicesOptions indicesOptions() { + // this doesn't really matter since data stream name resolution isn't affected by IndicesOptions and + // a data stream's backing indices are retrieved from its metadata + return IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); + } + + @Override + public boolean includeDataStreams() { + return true; + } + + @Override + public IndicesRequest indices(String... indices) { + this.names = indices; + return this; } } @@ -261,7 +286,7 @@ protected Response read(StreamInput in) throws IOException { @Override protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { - List dataStreams = getDataStreams(state, request); + List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); List dataStreamInfos = new ArrayList<>(dataStreams.size()); for (DataStream dataStream : dataStreams) { String indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false); @@ -280,26 +305,14 @@ protected void masterOperation(Task task, Request request, ClusterState state, listener.onResponse(new Response(dataStreamInfos)); } - static List getDataStreams(ClusterState clusterState, Request request) { + static List getDataStreams(ClusterState clusterState, IndexNameExpressionResolver iner, Request request) { + List results = iner.dataStreamNames(clusterState, request.indicesOptions(), request.names); Map dataStreams = clusterState.metadata().dataStreams(); - // return all data streams if no name was specified - final String requestedName = request.name == null ? "*" : request.name; - - final List results = new ArrayList<>(); - if (Regex.isSimpleMatchPattern(requestedName)) { - for (Map.Entry entry : dataStreams.entrySet()) { - if (Regex.simpleMatch(requestedName, entry.getKey())) { - results.add(entry.getValue()); - } - } - } else if (dataStreams.containsKey(request.name)) { - results.add(dataStreams.get(request.name)); - } else { - throw new ResourceNotFoundException("data_stream matching [" + request.name + "] not found"); - } - results.sort(Comparator.comparing(DataStream::getName)); - return results; + return results.stream() + .map(dataStreams::get) + .sorted(Comparator.comparing(DataStream::getName)) + .collect(Collectors.toList()); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index a6358e388a63e..392c78d08ed39 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -102,6 +102,8 @@ public FieldMappingMetadata fieldMappings(String index, String field) { } return indexMapping.get(field); } + public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; + public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -109,9 +111,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); - + boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); if (indexEntry.getValue() != null) { + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major && includeTypeName) { + builder.startObject(MapperService.SINGLE_MAPPING_NAME); + } addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major && includeTypeName) { + builder.endObject(); + } } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 36b9cf165dae1..f1e1c59092d65 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -34,6 +34,9 @@ import java.io.IOException; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.INCLUDE_TYPE_NAME_PARAMETER; + public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); @@ -100,7 +103,14 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (final ObjectObjectCursor indexEntry : getMappings()) { builder.startObject(indexEntry.key); - if (indexEntry.value != null) { + boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major + && includeTypeName) { + builder.startObject(MAPPINGS.getPreferredName()); + builder.field(MapperService.SINGLE_MAPPING_NAME, indexEntry.value.sourceAsMap()); + builder.endObject(); + } else if (indexEntry.value != null) { builder.field(MAPPINGS.getPreferredName(), indexEntry.value.sourceAsMap()); } else { builder.startObject(MAPPINGS.getPreferredName()).endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 9821fc264c306..3500bad94476a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -68,6 +68,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexClosedException; @@ -112,21 +113,21 @@ public class TransportBulkAction extends HandledTransportAction docWriteReque @Override protected void doExecute(Task task, BulkRequest bulkRequest, ActionListener listener) { long indexingBytes = bulkRequest.ramBytesUsed(); - final Releasable releasable = writeMemoryLimits.markWriteOperationStarted(indexingBytes); + final Releasable releasable = indexingPressure.markIndexingOperationStarted(indexingBytes); final ActionListener releasingListener = ActionListener.runBefore(listener, releasable::close); try { doInternalExecute(task, bulkRequest, releasingListener); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index d1c77df56b790..dd53d54f75434 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -56,6 +56,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.get.GetResult; @@ -93,9 +94,9 @@ public class TransportShardBulkAction extends TransportWriteAction writeBytes.getAndAdd(-bytes); - } - - public long getWriteBytes() { - return writeBytes.get(); - } - - public Releasable markReplicaWriteStarted(long bytes) { - replicaWriteBytes.getAndAdd(bytes); - return () -> replicaWriteBytes.getAndAdd(-bytes); - } - - public long getReplicaWriteBytes() { - return replicaWriteBytes.get(); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 10d565c335a77..72d0e2802f26b 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -701,6 +701,6 @@ public long getAutoGeneratedTimestamp() { @Override public long ramBytesUsed() { - return SHALLOW_SIZE + RamUsageEstimator.sizeOf(id) + (source == null ? 0 : source.ramBytesUsed()); + return SHALLOW_SIZE + RamUsageEstimator.sizeOf(id) + (source == null ? 0 : source.length()); } } diff --git a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java index 74ddcf54b3212..47f287ba0133f 100644 --- a/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java +++ b/server/src/main/java/org/elasticsearch/action/resync/TransportResyncReplicationAction.java @@ -20,7 +20,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.ReplicationOperation; import org.elasticsearch.action.support.replication.ReplicationResponse; @@ -57,11 +57,11 @@ public class TransportResyncReplicationAction extends TransportWriteAction extends TransportReplicationAction { - private final boolean forceExecutionOnPrimary; - private final WriteMemoryLimits writeMemoryLimits; + private final boolean forceExecution; + private final IndexingPressure indexingPressure; private final String executor; protected TransportWriteAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction, ActionFilters actionFilters, Writeable.Reader request, Writeable.Reader replicaRequest, String executor, boolean forceExecutionOnPrimary, - WriteMemoryLimits writeMemoryLimits) { + IndexingPressure indexingPressure) { // We pass ThreadPool.Names.SAME to the super class as we control the dispatching to the // ThreadPool.Names.WRITE thread pool in this class. super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, request, replicaRequest, ThreadPool.Names.SAME, true, forceExecutionOnPrimary); this.executor = executor; - this.forceExecutionOnPrimary = forceExecutionOnPrimary; - this.writeMemoryLimits = writeMemoryLimits; + this.forceExecution = forceExecutionOnPrimary; + this.indexingPressure = indexingPressure; } @Override protected Releasable checkOperationLimits(Request request) { - return writeMemoryLimits.markWriteOperationStarted(primaryOperationSize(request)); + return indexingPressure.markIndexingOperationStarted(primaryOperationSize(request), forceExecution); } @Override @@ -90,7 +90,7 @@ protected Releasable checkPrimaryLimits(Request request, boolean rerouteWasLocal if (rerouteWasLocal) { return () -> {}; } else { - return writeMemoryLimits.markWriteOperationStarted(primaryOperationSize(request)); + return indexingPressure.markIndexingOperationStarted(primaryOperationSize(request), forceExecution); } } @@ -100,7 +100,7 @@ protected long primaryOperationSize(Request request) { @Override protected Releasable checkReplicaLimits(ReplicaRequest request) { - return writeMemoryLimits.markReplicaWriteStarted(replicaOperationSize(request)); + return indexingPressure.markReplicaOperationStarted(replicaOperationSize(request), forceExecution); } protected long replicaOperationSize(ReplicaRequest request) { @@ -156,7 +156,7 @@ protected void doRun() { @Override public boolean isForceExecution() { - return forceExecutionOnPrimary; + return forceExecution; } }); } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java index 0d8c2c8a58822..364c9b623e59f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotDeletionsInProgress.java @@ -21,21 +21,25 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState.Custom; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryOperation; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; /** * A class that represents the snapshot deletions that are in progress in the cluster. @@ -51,6 +55,8 @@ public class SnapshotDeletionsInProgress extends AbstractNamedDiffable i private SnapshotDeletionsInProgress(List entries) { this.entries = entries; + assert entries.size() == entries.stream().map(Entry::uuid).distinct().count() : "Found duplicate UUIDs in entries " + entries; + assert assertNoConcurrentDeletionsForSameRepository(entries); } public static SnapshotDeletionsInProgress of(List entries) { @@ -61,7 +67,18 @@ public static SnapshotDeletionsInProgress of(List entries) { + final Set activeRepositories = new HashSet<>(); + for (Entry entry : entries) { + if (entry.state() == State.STARTED) { + final boolean added = activeRepositories.add(entry.repository()); + assert added : "Found multiple running deletes for a single repository in " + entries; + } + } + return true; } /** @@ -83,13 +100,20 @@ public SnapshotDeletionsInProgress withAddedEntry(Entry entry) { } /** - * Returns a new instance of {@link SnapshotDeletionsInProgress} which removes - * the given entry from the invoking instance. + * Returns a new instance of {@link SnapshotDeletionsInProgress} that has the entry with the given {@code deleteUUID} removed from its + * entries. */ - public SnapshotDeletionsInProgress withRemovedEntry(Entry entry) { - List entries = new ArrayList<>(getEntries()); - entries.remove(entry); - return SnapshotDeletionsInProgress.of(entries); + public SnapshotDeletionsInProgress withRemovedEntry(String deleteUUID) { + List updatedEntries = new ArrayList<>(entries.size() - 1); + boolean removed = false; + for (Entry entry : entries) { + if (entry.uuid().equals(deleteUUID)) { + removed = true; + } else { + updatedEntries.add(entry); + } + } + return removed ? SnapshotDeletionsInProgress.of(updatedEntries) : this; } /** @@ -183,17 +207,23 @@ public String toString() { public static final class Entry implements Writeable, RepositoryOperation { private final List snapshots; private final String repoName; + private final State state; private final long startTime; private final long repositoryStateId; + private final String uuid; + + public Entry(List snapshots, String repoName, long startTime, long repositoryStateId, State state) { + this(snapshots, repoName, startTime, repositoryStateId, state, UUIDs.randomBase64UUID()); + } - public Entry(List snapshots, String repoName, long startTime, long repositoryStateId) { + private Entry(List snapshots, String repoName, long startTime, long repositoryStateId, State state, String uuid) { this.snapshots = snapshots; assert snapshots.size() == new HashSet<>(snapshots).size() : "Duplicate snapshot ids in " + snapshots; this.repoName = repoName; this.startTime = startTime; this.repositoryStateId = repositoryStateId; - assert repositoryStateId > RepositoryData.EMPTY_REPO_GEN : - "Can't delete based on an empty or unknown repository generation but saw [" + repositoryStateId + "]"; + this.state = state; + this.uuid = uuid; } public Entry(StreamInput in) throws IOException { @@ -201,6 +231,43 @@ public Entry(StreamInput in) throws IOException { this.snapshots = in.readList(SnapshotId::new); this.startTime = in.readVLong(); this.repositoryStateId = in.readLong(); + if (in.getVersion().onOrAfter(SnapshotsService.FULL_CONCURRENCY_VERSION)) { + this.state = State.readFrom(in); + this.uuid = in.readString(); + } else { + this.state = State.STARTED; + this.uuid = IndexMetadata.INDEX_UUID_NA_VALUE; + } + } + + public Entry started() { + assert state == State.WAITING; + return new Entry(snapshots, repository(), startTime, repositoryStateId, State.STARTED, uuid); + } + + public Entry withAddedSnapshots(Collection newSnapshots) { + assert state == State.WAITING; + final Collection updatedSnapshots = new HashSet<>(snapshots); + if (updatedSnapshots.addAll(newSnapshots) == false) { + return this; + } + return new Entry(List.copyOf(updatedSnapshots), repository(), startTime, repositoryStateId, State.WAITING, uuid); + } + + public Entry withSnapshots(Collection snapshots) { + return new Entry(List.copyOf(snapshots), repository(), startTime, repositoryStateId, state, uuid); + } + + public Entry withRepoGen(long repoGen) { + return new Entry(snapshots, repository(), startTime, repoGen, state, uuid); + } + + public State state() { + return state; + } + + public String uuid() { + return uuid; } public List getSnapshots() { @@ -226,12 +293,14 @@ public boolean equals(Object o) { return repoName.equals(that.repoName) && snapshots.equals(that.snapshots) && startTime == that.startTime - && repositoryStateId == that.repositoryStateId; + && repositoryStateId == that.repositoryStateId + && state == that.state + && uuid.equals(that.uuid); } @Override public int hashCode() { - return Objects.hash(snapshots, repoName, startTime, repositoryStateId); + return Objects.hash(snapshots, repoName, startTime, repositoryStateId, state, uuid); } @Override @@ -240,6 +309,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(snapshots); out.writeVLong(startTime); out.writeLong(repositoryStateId); + if (out.getVersion().onOrAfter(SnapshotsService.FULL_CONCURRENCY_VERSION)) { + state.writeTo(out); + out.writeString(uuid); + } } @Override @@ -251,5 +324,47 @@ public String repository() { public long repositoryStateId() { return repositoryStateId; } + + @Override + public String toString() { + return "SnapshotDeletionsInProgress.Entry[[" + uuid + "][" + state + "]" + snapshots + "]"; + } + } + + public enum State implements Writeable { + + /** + * Delete is waiting to execute because there are snapshots and or a delete operation that has to complete before this delete may + * run. + */ + WAITING((byte) 0), + + /** + * Delete is physically executing on the repository. + */ + STARTED((byte) 1); + + private final byte value; + + State(byte value) { + this.value = value; + } + + public static State readFrom(StreamInput in) throws IOException { + final byte value = in.readByte(); + switch (value) { + case 0: + return WAITING; + case 1: + return STARTED; + default: + throw new IllegalArgumentException("No snapshot delete state for value [" + value + "]"); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte(value); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 71e49ee88498c..0c9a7cc91e97d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -151,6 +152,18 @@ public Entry(Entry entry, ImmutableOpenMap shards) this(entry, entry.state, shards, entry.failure); } + public Entry withRepoGen(long newRepoGen) { + assert newRepoGen > repositoryStateId : "Updated repository generation [" + newRepoGen + + "] must be higher than current generation [" + repositoryStateId + "]"; + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, newRepoGen, shards, failure, + userMetadata, version); + } + + public Entry withShards(ImmutableOpenMap shards) { + return new Entry(snapshot, includeGlobalState, partial, state, indices, dataStreams, startTime, repositoryStateId, shards, + failure, userMetadata, version); + } + @Override public String repository() { return snapshot.getRepository(); @@ -308,7 +321,16 @@ public static boolean completed(ObjectContainer shards) { } public static class ShardSnapshotStatus { + + /** + * Shard snapshot status for shards that are waiting for another operation to finish before they can be assigned to a node. + */ + public static final ShardSnapshotStatus UNASSIGNED_QUEUED = + new SnapshotsInProgress.ShardSnapshotStatus(null, ShardState.QUEUED, null); + private final ShardState state; + + @Nullable private final String nodeId; @Nullable @@ -321,17 +343,23 @@ public ShardSnapshotStatus(String nodeId, String generation) { this(nodeId, ShardState.INIT, generation); } - public ShardSnapshotStatus(String nodeId, ShardState state, String generation) { + public ShardSnapshotStatus(@Nullable String nodeId, ShardState state, @Nullable String generation) { this(nodeId, state, null, generation); } - public ShardSnapshotStatus(String nodeId, ShardState state, String reason, String generation) { + public ShardSnapshotStatus(@Nullable String nodeId, ShardState state, String reason, @Nullable String generation) { this.nodeId = nodeId; this.state = state; this.reason = reason; this.generation = generation; + assert assertConsistent(); + } + + private boolean assertConsistent() { // If the state is failed we have to have a reason for this failure assert state.failed() == false || reason != null; + assert (state != ShardState.INIT && state != ShardState.WAITING) || nodeId != null : "Null node id for state [" + state + "]"; + return true; } public ShardSnapshotStatus(StreamInput in) throws IOException { @@ -349,10 +377,12 @@ public ShardState state() { return state; } + @Nullable public String nodeId() { return nodeId; } + @Nullable public String generation() { return this.generation; } @@ -361,6 +391,15 @@ public String reason() { return reason; } + /** + * Checks if this shard snapshot is actively executing. + * A shard is defined as actively executing if it either is in a state that may write to the repository + * ({@link ShardState#INIT} or {@link ShardState#ABORTED}) or about to write to it in state {@link ShardState#WAITING}. + */ + public boolean isActive() { + return state == ShardState.INIT || state == ShardState.ABORTED || state == ShardState.WAITING; + } + public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(nodeId); out.writeByte(state.value); @@ -438,6 +477,19 @@ public static State fromValue(byte value) { private final List entries; + private static boolean assertConsistentEntries(List entries) { + final Map> assignedShardsByRepo = new HashMap<>(); + for (Entry entry : entries) { + for (ObjectObjectCursor shard : entry.shards()) { + if (shard.value.isActive()) { + assert assignedShardsByRepo.computeIfAbsent(entry.repository(), k -> new HashSet<>()).add(shard.key) : + "Found duplicate shard assignments in " + entries; + } + } + } + return true; + } + public static SnapshotsInProgress of(List entries) { if (entries.isEmpty()) { return EMPTY; @@ -447,6 +499,7 @@ public static SnapshotsInProgress of(List entries) { private SnapshotsInProgress(List entries) { this.entries = entries; + assert assertConsistentEntries(entries); } public List entries() { @@ -581,7 +634,14 @@ public enum ShardState { FAILED((byte) 3, true, true), ABORTED((byte) 4, false, true), MISSING((byte) 5, true, true), - WAITING((byte) 6, false, false); + /** + * Shard snapshot is waiting for the primary to snapshot to become available. + */ + WAITING((byte) 6, false, false), + /** + * Shard snapshot is waiting for another shard snapshot for the same shard and to the same repository to finish. + */ + QUEUED((byte) 7, false, false); private final byte value; @@ -617,6 +677,8 @@ public static ShardState fromValue(byte value) { return MISSING; case 6: return WAITING; + case 7: + return QUEUED; default: throw new IllegalArgumentException("No shard snapshot state for value [" + value + "]"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java index 585dee0732b98..482cff11ea9c5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java @@ -327,16 +327,16 @@ public void handleException(TransportException exp) { failureCountSinceLastSuccess++; final String reason; - if (failureCountSinceLastSuccess >= followerCheckRetryCount) { - logger.debug(() -> new ParameterizedMessage("{} failed too many times", FollowerChecker.this), exp); - reason = "followers check retry count exceeded"; - } else if (exp instanceof ConnectTransportException + if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) { logger.debug(() -> new ParameterizedMessage("{} disconnected", FollowerChecker.this), exp); reason = "disconnected"; } else if (exp.getCause() instanceof NodeHealthCheckFailureException) { logger.debug(() -> new ParameterizedMessage("{} health check failed", FollowerChecker.this), exp); reason = "health check failed"; + } else if (failureCountSinceLastSuccess >= followerCheckRetryCount) { + logger.debug(() -> new ParameterizedMessage("{} failed too many times", FollowerChecker.this), exp); + reason = "followers check retry count exceeded"; } else { logger.debug(() -> new ParameterizedMessage("{} failed, retrying", FollowerChecker.this), exp); scheduleNextWakeUp(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 583d687b05dae..290d4b0097ab1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import java.io.IOException; import java.util.List; @@ -276,10 +275,10 @@ public String getTimestampField() { } /** - * @return a mapping snippet for a backing index with `_timestamp` meta field mapper properly configured. + * @return a mapping snippet for a backing index with `_data_stream_timestamp` meta field mapper properly configured. */ public Map getDataSteamMappingSnippet() { - return Map.of(MapperService.SINGLE_MAPPING_NAME, Map.of(TimestampFieldMapper.NAME, Map.of("path", timestampField))); + return Map.of(MapperService.SINGLE_MAPPING_NAME, Map.of("_data_stream_timestamp", Map.of("path", timestampField))); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 6a0997329662c..3816c335193cd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -34,12 +34,16 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.TimestampFieldMapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class MetadataCreateDataStreamService { @@ -168,12 +172,16 @@ public static ComposableIndexTemplate lookupTemplateForDataStream(String dataStr return composableIndexTemplate; } - public static void validateTimestampFieldMapping(String timestampFieldName, MapperService mapperService) { - TimestampFieldMapper fieldMapper = (TimestampFieldMapper) mapperService.documentMapper().mappers().getMapper("_timestamp"); - assert fieldMapper != null : "[_timestamp] meta field mapper must exist"; + public static void validateTimestampFieldMapping(String timestampFieldName, MapperService mapperService) throws IOException { + MetadataFieldMapper fieldMapper = + (MetadataFieldMapper) mapperService.documentMapper().mappers().getMapper("_data_stream_timestamp"); + assert fieldMapper != null : "[_data_stream_timestamp] meta field mapper must exist"; - if (timestampFieldName.equals(fieldMapper.getPath()) == false) { - throw new IllegalArgumentException("[_timestamp] meta field doesn't point to data stream timestamp field [" + + Map parsedTemplateMapping = + MapperService.parseMapping(NamedXContentRegistry.EMPTY, mapperService.documentMapper().mappingSource().string()); + String configuredPath = ObjectPath.eval("_doc._data_stream_timestamp.path", parsedTemplateMapping); + if (timestampFieldName.equals(configuredPath) == false) { + throw new IllegalArgumentException("[_data_stream_timestamp] meta field doesn't point to data stream timestamp field [" + timestampFieldName + "]"); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 5f659fd9b59bc..452a24d1b5df5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -50,6 +50,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; @@ -1124,6 +1125,17 @@ private static void validateCompositeTemplate(final ClusterState state, // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed + + if (template.getDataStreamTemplate() != null) { + // If there is no _data_stream meta field mapper and a data stream should be created then + // fail as if the data_stream field can't be parsed: + if (tempIndexService.mapperService().isMetadataField("_data_stream_timestamp") == false) { + // Fail like a parsing expection, since we will be moving data_stream template out of server module and + // then we would fail with the same error message, like we do here. + throw new XContentParseException("[index_template] unknown field [data_stream]"); + } + } + List mappings = collectMappings(stateWithIndex, templateName, indexName ); try { MapperService mapperService = tempIndexService.mapperService(); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java index 84972baf6b761..d39e7752a2dc3 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryParser.java @@ -20,16 +20,25 @@ package org.elasticsearch.common.geo; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.support.MapXContentParser; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.StandardValidator; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownText; import java.io.IOException; import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; /** * An utility class with a geometry parser methods supporting different shape representation formats @@ -38,11 +47,13 @@ public final class GeometryParser { private final GeoJson geoJsonParser; private final WellKnownText wellKnownTextParser; + private final boolean ignoreZValue; public GeometryParser(boolean rightOrientation, boolean coerce, boolean ignoreZValue) { GeometryValidator validator = new StandardValidator(ignoreZValue); geoJsonParser = new GeoJson(rightOrientation, coerce, validator); wellKnownTextParser = new WellKnownText(coerce, validator); + this.ignoreZValue = ignoreZValue; } /** @@ -109,4 +120,58 @@ public XContentBuilder toXContent(Geometry geometry, XContentBuilder builder, To } throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates"); } + + /** + * Parses the value as a {@link Geometry}. The following types of values are supported: + *

+ * Object: has to contain either lat and lon or geohash fields + *

+ * String: expected to be in "latitude, longitude" format, a geohash or WKT + *

+ * Array: two or more elements, the first element is longitude, the second is latitude, the rest is ignored if ignoreZValue is true + *

+ * Json structure: valid geojson definition + */ + public Geometry parseGeometry(Object value) throws ElasticsearchParseException { + if (value instanceof List) { + List values = (List) value; + if (values.size() == 2 && values.get(0) instanceof Number) { + GeoPoint point = GeoUtils.parseGeoPoint(values, ignoreZValue); + return new Point(point.lon(), point.lat()); + } else { + List geometries = new ArrayList<>(values.size()); + for (Object object : values) { + geometries.add(parseGeometry(object)); + } + return new GeometryCollection<>(geometries); + } + } + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("null_value", value), null)) { + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + if (isPoint(value)) { + GeoPoint point = GeoUtils.parseGeoPoint(parser, new GeoPoint(), ignoreZValue); + return new Point(point.lon(), point.lat()); + } else { + return parse(parser); + } + + } catch (IOException | ParseException ex) { + throw new ElasticsearchParseException("error parsing geometry ", ex); + } + } + + private boolean isPoint(Object value) { + // can we do this better? + if (value instanceof Map) { + Map map = (Map) value; + return map.containsKey("lat") && map.containsKey("lon"); + } else if (value instanceof String) { + String string = (String) value; + return Character.isDigit(string.charAt(0)) || string.indexOf('(') == -1; + } + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 8ef92c1a4e458..4ede1bfdc05d1 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -21,6 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; @@ -108,6 +109,7 @@ import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ProxyConnectionStrategy; import org.elasticsearch.transport.RemoteClusterService; @@ -371,9 +373,6 @@ public void apply(Settings value, Settings current, Settings previous) { NetworkService.TCP_RECEIVE_BUFFER_SIZE, IndexSettings.QUERY_STRING_ANALYZE_WILDCARD, IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD, - ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING, - ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, - ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING, ScriptService.SCRIPT_CACHE_SIZE_SETTING, ScriptService.SCRIPT_CACHE_EXPIRE_SETTING, ScriptService.SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING, @@ -486,9 +485,11 @@ public void apply(Settings value, Settings current, Settings previous) { LagDetector.CLUSTER_FOLLOWER_LAG_TIMEOUT_SETTING, HandshakingTransportAddressConnector.PROBE_CONNECT_TIMEOUT_SETTING, HandshakingTransportAddressConnector.PROBE_HANDSHAKE_TIMEOUT_SETTING, + SnapshotsService.MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, FsHealthService.ENABLED_SETTING, FsHealthService.REFRESH_INTERVAL_SETTING, - FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING); + FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING, + IndexingPressure.MAX_INDEXING_BYTES); static List> BUILT_IN_SETTING_UPGRADERS = Collections.emptyList(); diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java index 9cc5cf7bd8188..2444080744912 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ReleasableLock.java @@ -21,6 +21,7 @@ import org.elasticsearch.Assertions; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.engine.EngineException; import java.util.concurrent.locks.Lock; @@ -57,6 +58,19 @@ public ReleasableLock acquire() throws EngineException { return this; } + /** + * Try acquiring lock, returning null if unable to acquire lock within timeout. + */ + public ReleasableLock tryAcquire(TimeValue timeout) throws InterruptedException { + boolean locked = lock.tryLock(timeout.duration(), timeout.timeUnit()); + if (locked) { + assert addCurrentThread(); + return this; + } else { + return null; + } + } + private boolean addCurrentThread() { final Integer current = holdingThreads.get(); holdingThreads.set(current == null ? 1 : current + 1); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index 05f8cb3a2fc45..dfbb507365fa9 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -332,6 +332,16 @@ public static String nodeStringValue(Object node, String defaultValue) { return node.toString(); } + /** + * Returns the {@link Object#toString} value of its input, or {@code null} if the input is null + */ + public static String nodeStringValue(Object node) { + if (node == null) { + return null; + } + return node.toString(); + } + public static float nodeFloatValue(Object node, float defaultValue) { if (node == null) { return defaultValue; diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index e4da0885b4618..fbfdf0e700e0d 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -1044,16 +1044,7 @@ public Set findAllShardIds(final Index index) throws IOException { final Set shardIds = new HashSet<>(); final String indexUniquePathId = index.getUUID(); for (final NodePath nodePath : nodePaths) { - Path location = nodePath.indicesPath; - if (Files.isDirectory(location)) { - try (DirectoryStream indexStream = Files.newDirectoryStream(location)) { - for (Path indexPath : indexStream) { - if (indexUniquePathId.equals(indexPath.getFileName().toString())) { - shardIds.addAll(findAllShardsForIndex(indexPath, index)); - } - } - } - } + shardIds.addAll(findAllShardsForIndex(nodePath.indicesPath.resolve(indexUniquePathId), index)); } return shardIds; } diff --git a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java index ddabdfc2861dc..c0d42bc9fc62b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java +++ b/server/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java @@ -63,7 +63,7 @@ public class DanglingIndicesState implements ClusterStateListener { */ public static final Setting AUTO_IMPORT_DANGLING_INDICES_SETTING = Setting.boolSetting( "gateway.auto_import_dangling_indices", - true, + false, Setting.Property.NodeScope, Setting.Property.Deprecated ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexingPressure.java b/server/src/main/java/org/elasticsearch/index/IndexingPressure.java new file mode 100644 index 0000000000000..9c8fb83fe4ffc --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/IndexingPressure.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.index.stats.IndexingPressureStats; + +import java.util.concurrent.atomic.AtomicLong; + +public class IndexingPressure { + + public static final Setting MAX_INDEXING_BYTES = + Setting.memorySizeSetting("indexing_pressure.memory.limit", "10%", Setting.Property.NodeScope); + + private final AtomicLong currentCoordinatingAndPrimaryBytes = new AtomicLong(0); + private final AtomicLong currentReplicaBytes = new AtomicLong(0); + private final AtomicLong totalCoordinatingAndPrimaryBytes = new AtomicLong(0); + private final AtomicLong totalReplicaBytes = new AtomicLong(0); + private final AtomicLong coordinatingAndPrimaryRejections = new AtomicLong(0); + private final AtomicLong replicaRejections = new AtomicLong(0); + + private final long primaryAndCoordinatingLimits; + private final long replicaLimits; + + public IndexingPressure(Settings settings) { + this.primaryAndCoordinatingLimits = MAX_INDEXING_BYTES.get(settings).getBytes(); + this.replicaLimits = (long) (this.primaryAndCoordinatingLimits * 1.5); + } + + public Releasable markIndexingOperationStarted(long bytes) { + return markIndexingOperationStarted(bytes, false); + } + + public Releasable markIndexingOperationStarted(long bytes, boolean forceExecution) { + long writeBytes = this.currentCoordinatingAndPrimaryBytes.addAndGet(bytes); + long replicaWriteBytes = this.currentReplicaBytes.get(); + long totalBytes = writeBytes + replicaWriteBytes; + if (forceExecution == false && totalBytes > primaryAndCoordinatingLimits) { + long bytesWithoutOperation = writeBytes - bytes; + long totalBytesWithoutOperation = totalBytes - bytes; + this.currentCoordinatingAndPrimaryBytes.getAndAdd(-bytes); + this.coordinatingAndPrimaryRejections.getAndIncrement(); + throw new EsRejectedExecutionException("rejected execution of operation [" + + "coordinating_and_primary_bytes=" + bytesWithoutOperation + ", " + + "replica_bytes=" + replicaWriteBytes + ", " + + "all_bytes=" + totalBytesWithoutOperation + ", " + + "operation_bytes=" + bytes + ", " + + "max_coordinating_and_primary_bytes=" + primaryAndCoordinatingLimits + "]", false); + } + totalCoordinatingAndPrimaryBytes.getAndAdd(bytes); + return () -> this.currentCoordinatingAndPrimaryBytes.getAndAdd(-bytes); + } + + public Releasable markReplicaOperationStarted(long bytes, boolean forceExecution) { + long replicaWriteBytes = this.currentReplicaBytes.getAndAdd(bytes); + if (forceExecution == false && replicaWriteBytes > replicaLimits) { + long replicaBytesWithoutOperation = replicaWriteBytes - bytes; + this.currentReplicaBytes.getAndAdd(-bytes); + this.replicaRejections.getAndIncrement(); + throw new EsRejectedExecutionException("rejected execution of replica operation [" + + "replica_bytes=" + replicaBytesWithoutOperation + ", " + + "replica_operation_bytes=" + bytes + ", " + + "max_replica_bytes=" + replicaLimits + "]", false); + } + totalReplicaBytes.getAndAdd(bytes); + return () -> this.currentReplicaBytes.getAndAdd(-bytes); + } + + public long getCurrentCoordinatingAndPrimaryBytes() { + return currentCoordinatingAndPrimaryBytes.get(); + } + + public long getCurrentReplicaBytes() { + return currentReplicaBytes.get(); + } + + public long getTotalCoordinatingAndPrimaryBytes() { + return totalCoordinatingAndPrimaryBytes.get(); + } + + public long getTotalReplicaBytes() { + return totalReplicaBytes.get(); + } + + public IndexingPressureStats stats() { + return new IndexingPressureStats(totalCoordinatingAndPrimaryBytes.get(), totalReplicaBytes.get(), + currentCoordinatingAndPrimaryBytes.get(), currentReplicaBytes.get(), coordinatingAndPrimaryRejections.get(), + replicaRejections.get()); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 50f0c7a8c5441..0dbfc5070706d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -188,10 +188,6 @@ protected AbstractGeometryFieldType(String name, boolean indexed, boolean hasDoc super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } - protected AbstractGeometryFieldType(AbstractGeometryFieldType ref) { - super(ref); - } - public void setGeometryQueryBuilder(QueryProcessor geometryQueryBuilder) { this.geometryQueryBuilder = geometryQueryBuilder; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java index 68682ffc8c798..5dfda3f527a5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -117,10 +117,6 @@ public abstract static class AbstractPointGeometryFieldType protected AbstractPointGeometryFieldType(String name, boolean indexed, boolean hasDocValues, Map meta) { super(name, indexed, hasDocValues, meta); } - - protected AbstractPointGeometryFieldType(AbstractPointGeometryFieldType ref) { - super(ref); - } } protected AbstractPointGeometryFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 0de1a30ab2d27..5ee6a19be93aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -32,7 +32,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; /** * Base class for {@link GeoShapeFieldMapper} and {@link LegacyGeoShapeFieldMapper} @@ -161,23 +160,6 @@ protected AbstractShapeGeometryFieldType(String name, boolean isSearchable, bool super(name, isSearchable, hasDocValues, meta); } - protected AbstractShapeGeometryFieldType(AbstractShapeGeometryFieldType ref) { - super(ref); - this.orientation = ref.orientation; - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - AbstractShapeGeometryFieldType that = (AbstractShapeGeometryFieldType) o; - return orientation == that.orientation; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), orientation); - } - public Orientation orientation() { return this.orientation; } public void setOrientation(Orientation orientation) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 503703ec3f863..0a61d12cc24dd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -20,9 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectArrayList; -import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; @@ -48,42 +46,38 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.TypeParsers.parseField; - -public class BinaryFieldMapper extends FieldMapper { +public class BinaryFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "binary"; - public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); - - static { - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - } + private static BinaryFieldMapper toType(FieldMapper in) { + return (BinaryFieldMapper) in; } - public static class Builder extends FieldMapper.Builder { + public static class Builder extends ParametrizedFieldMapper.Builder { + + private final Parameter stored = Parameter.boolParam("store", false, m -> toType(m).stored, false); + private final Parameter hasDocValues = Parameter.boolParam("doc_values", false, m -> toType(m).hasDocValues, false); + private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - hasDocValues = false; - builder = this; + this(name, false); + } + + public Builder(String name, boolean hasDocValues) { + super(name); + this.hasDocValues.setValue(hasDocValues); } @Override - public BinaryFieldMapper build(BuilderContext context) { - return new BinaryFieldMapper(name, fieldType, new BinaryFieldType(buildFullName(context), hasDocValues, meta), - multiFieldsBuilder.build(this, context), copyTo); + public List> getParameters() { + return List.of(meta, stored, hasDocValues); } @Override - public Builder index(boolean index) { - if (index) { - throw new MapperParsingException("Binary field [" + name() + "] cannot be indexed"); - } - return builder; + public BinaryFieldMapper build(BuilderContext context) { + return new BinaryFieldMapper(name, new BinaryFieldType(buildFullName(context), hasDocValues.getValue(), meta.getValue()), + multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -92,7 +86,7 @@ public static class TypeParser implements Mapper.TypeParser { public BinaryFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(name); - parseField(builder, name, node, parserContext); + builder.parse(name, parserContext, node); return builder; } } @@ -107,16 +101,6 @@ public BinaryFieldType(String name) { this(name, true, Collections.emptyMap()); } - protected BinaryFieldType(BinaryFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new BinaryFieldType(this); - } - - @Override public String typeName() { return CONTENT_TYPE; @@ -167,14 +151,19 @@ public Query termQuery(Object value, QueryShardContext context) { } } - protected BinaryFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + private final boolean stored; + private final boolean hasDocValues; + + protected BinaryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, Builder builder) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.stored = builder.stored.getValue(); + this.hasDocValues = builder.hasDocValues.getValue(); } @Override protected void parseCreateField(ParseContext context) throws IOException { - if (!fieldType.stored() && !fieldType().hasDocValues()) { + if (stored == false && hasDocValues == false) { return; } byte[] value = context.parseExternalValue(byte[].class); @@ -188,11 +177,11 @@ protected void parseCreateField(ParseContext context) throws IOException { if (value == null) { return; } - if (fieldType.stored()) { + if (stored) { context.doc().add(new StoredField(fieldType().name(), value)); } - if (fieldType().hasDocValues()) { + if (hasDocValues) { CustomBinaryDocValuesField field = (CustomBinaryDocValuesField) context.doc().getByKey(fieldType().name()); if (field == null) { field = new CustomBinaryDocValuesField(fieldType().name(), value); @@ -210,18 +199,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected boolean indexedByDefault() { - return false; - } - - @Override - protected boolean docValuesByDefault() { - return false; - } - - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new BinaryFieldMapper.Builder(simpleName()).init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 9a3894bea5e74..f5d33845ef05c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -30,7 +31,6 @@ import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -42,16 +42,13 @@ import java.io.IOException; import java.time.ZoneId; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.TypeParsers.parseField; - /** * A field mapper for boolean fields. */ -public class BooleanFieldMapper extends FieldMapper { +public class BooleanFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "boolean"; @@ -71,25 +68,36 @@ public static class Values { public static final BytesRef FALSE = new BytesRef("F"); } - public static class Builder extends FieldMapper.Builder { + private static BooleanFieldMapper toType(FieldMapper in) { + return (BooleanFieldMapper) in; + } + + public static class Builder extends ParametrizedFieldMapper.Builder { + + private final Parameter docValues = Parameter.boolParam("doc_values", false, m -> toType(m).hasDocValues, true); + private final Parameter indexed = Parameter.boolParam("index", false, m -> toType(m).indexed, true); + private final Parameter stored = Parameter.boolParam("store", false, m -> toType(m).stored, false); + + private final Parameter nullValue + = new Parameter<>("null_value", false, null, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), m -> toType(m).nullValue); - private Boolean nullValue; + private final Parameter boost = Parameter.floatParam("boost", true, m -> m.fieldType().boost(), 1.0f); + private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - this.builder = this; + super(name); } - public Builder nullValue(Boolean nullValue) { - this.nullValue = nullValue; - return builder; + @Override + protected List> getParameters() { + return List.of(meta, boost, docValues, indexed, nullValue, stored); } @Override public BooleanFieldMapper build(BuilderContext context) { - return new BooleanFieldMapper(name, fieldType, - new BooleanFieldType(buildFullName(context), indexed, hasDocValues, meta), - multiFieldsBuilder.build(this, context), copyTo, nullValue); + MappedFieldType ft = new BooleanFieldType(buildFullName(context), indexed.getValue(), docValues.getValue(), meta.getValue()); + ft.setBoost(boost.getValue()); + return new BooleanFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -98,19 +106,7 @@ public static class TypeParser implements Mapper.TypeParser { public BooleanFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder(name); - parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = entry.getKey(); - Object propNode = entry.getValue(); - if (propName.equals("null_value")) { - if (propNode == null) { - throw new MapperParsingException("Property [null_value] cannot be null."); - } - builder.nullValue(XContentMapValues.nodeBooleanValue(propNode, name + ".null_value")); - iterator.remove(); - } - } + builder.parse(name, parserContext, node); return builder; } } @@ -125,15 +121,6 @@ public BooleanFieldType(String name) { this(name, true, true, Collections.emptyMap()); } - protected BooleanFieldType(BooleanFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new BooleanFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -217,11 +204,17 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } private final Boolean nullValue; - - protected BooleanFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - MultiFields multiFields, CopyTo copyTo, Boolean nullValue) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.nullValue = nullValue; + private final boolean indexed; + private final boolean hasDocValues; + private final boolean stored; + + protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, Builder builder) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.nullValue = builder.nullValue.getValue(); + this.stored = builder.stored.getValue(); + this.indexed = builder.indexed.getValue(); + this.hasDocValues = builder.docValues.getValue(); } @Override @@ -231,7 +224,7 @@ public BooleanFieldType fieldType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - if (fieldType().isSearchable() == false && !fieldType.stored() && !fieldType().hasDocValues()) { + if (indexed == false && stored == false && hasDocValues == false) { return; } @@ -250,10 +243,13 @@ protected void parseCreateField(ParseContext context) throws IOException { if (value == null) { return; } - if (fieldType().isSearchable() || fieldType.stored()) { - context.doc().add(new Field(fieldType().name(), value ? "T" : "F", fieldType)); + if (indexed) { + context.doc().add(new Field(fieldType().name(), value ? "T" : "F", Defaults.FIELD_TYPE)); } - if (fieldType().hasDocValues()) { + if (stored) { + context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F")); + } + if (hasDocValues) { context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); } else { createFieldNamesField(context); @@ -261,8 +257,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - // TODO ban updating null values + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); } @Override @@ -270,11 +266,4 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - if (includeDefaults || nullValue != null) { - builder.field("null_value", nullValue); - } - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 1b863c3051965..047fab17fc1fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -33,14 +33,11 @@ import org.apache.lucene.search.suggest.document.SuggestField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.NumberType; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -55,14 +52,10 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import static org.elasticsearch.index.mapper.TypeParsers.parseMultiField; - /** * Mapper for completion field. The field values are indexed as a weighted FST for * fast auto-completion/search-as-you-type functionality.
@@ -84,7 +77,7 @@ * This field can also be extended to add search criteria to suggestions * for query-time filtering and boosting (see {@link ContextMappings} */ -public class CompletionFieldMapper extends FieldMapper { +public class CompletionFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "completion"; /** @@ -92,6 +85,11 @@ public class CompletionFieldMapper extends FieldMapper { */ static final int COMPLETION_CONTEXTS_LIMIT = 10; + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), defaultAnalyzer).init(this); + } + public static class Defaults { public static final FieldType FIELD_TYPE = new FieldType(); static { @@ -108,20 +106,101 @@ public static class Defaults { } public static class Fields { - // Mapping field names - public static final ParseField ANALYZER = new ParseField("analyzer"); - public static final ParseField SEARCH_ANALYZER = new ParseField("search_analyzer"); - public static final ParseField PRESERVE_SEPARATORS = new ParseField("preserve_separators"); - public static final ParseField PRESERVE_POSITION_INCREMENTS = new ParseField("preserve_position_increments"); - public static final ParseField TYPE = new ParseField("type"); - public static final ParseField CONTEXTS = new ParseField("contexts"); - public static final ParseField MAX_INPUT_LENGTH = new ParseField("max_input_length", "max_input_len"); // Content field names public static final String CONTENT_FIELD_NAME_INPUT = "input"; public static final String CONTENT_FIELD_NAME_WEIGHT = "weight"; public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts"; } + private static CompletionFieldMapper toType(FieldMapper in) { + return (CompletionFieldMapper) in; + } + + /** + * Builder for {@link CompletionFieldMapper} + */ + public static class Builder extends ParametrizedFieldMapper.Builder { + + private final Parameter analyzer; + private final Parameter searchAnalyzer; + private final Parameter preserveSeparators = Parameter.boolParam("preserve_separators", false, + m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS); + private final Parameter preservePosInc = Parameter.boolParam("preserve_position_increments", false, + m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS); + private final Parameter contexts = new Parameter<>("contexts", false, null, + (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> toType(m).contexts) + .setSerializer((b, n, c) -> { + if (c == null) { + return; + } + b.startArray(n); + c.toXContent(b, ToXContent.EMPTY_PARAMS); + b.endArray(); + }); + private final Parameter maxInputLength = Parameter.intParam("max_input_length", true, + m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH) + .addDeprecatedName("max_input_len") + .setValidator(Builder::validateInputLength); + private final Parameter> meta = Parameter.metaParam(); + + private final NamedAnalyzer defaultAnalyzer; + + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Builder.class); + + /** + * @param name of the completion field to build + */ + public Builder(String name, NamedAnalyzer defaultAnalyzer) { + super(name); + this.defaultAnalyzer = defaultAnalyzer; + this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, defaultAnalyzer); + this.searchAnalyzer = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, defaultAnalyzer); + } + + private static void validateInputLength(int maxInputLength) { + if (maxInputLength <= 0) { + throw new IllegalArgumentException("[max_input_length] must be > 0 but was [" + maxInputLength + "]"); + } + } + + @Override + protected List> getParameters() { + return List.of(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, contexts, maxInputLength, meta); + } + + @Override + public CompletionFieldMapper build(BuilderContext context) { + checkCompletionContextsLimit(context); + NamedAnalyzer completionAnalyzer = new NamedAnalyzer(this.searchAnalyzer.getValue().name(), AnalyzerScope.INDEX, + new CompletionAnalyzer(this.searchAnalyzer.getValue(), preserveSeparators.getValue(), preservePosInc.getValue())); + + CompletionFieldType ft + = new CompletionFieldType(buildFullName(context), completionAnalyzer, meta.getValue()); + ft.setContextMappings(contexts.getValue()); + ft.setPreservePositionIncrements(preservePosInc.getValue()); + ft.setPreserveSep(preserveSeparators.getValue()); + ft.setIndexAnalyzer(analyzer.getValue()); + return new CompletionFieldMapper(name, ft, defaultAnalyzer, + multiFieldsBuilder.build(this, context), copyTo.build(), this); + } + + private void checkCompletionContextsLimit(BuilderContext context) { + if (this.contexts.getValue() != null && this.contexts.getValue().size() > COMPLETION_CONTEXTS_LIMIT) { + if (context.indexCreatedVersion().onOrAfter(Version.V_8_0_0)) { + throw new IllegalArgumentException( + "Limit of completion field contexts [" + COMPLETION_CONTEXTS_LIMIT + "] has been exceeded"); + } else { + deprecationLogger.deprecate("excessive_completion_contexts", + "You have defined more than [" + COMPLETION_CONTEXTS_LIMIT + "] completion contexts" + + " in the mapping for index [" + context.indexSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME) + "]. " + + "The maximum allowed number of completion contexts in a mapping will be limited to " + + "[" + COMPLETION_CONTEXTS_LIMIT + "] starting in version [8.0]."); + } + } + } + + } + public static final Set ALLOWED_CONTENT_FIELD_NAMES = Sets.newHashSet(Fields.CONTENT_FIELD_NAME_INPUT, Fields.CONTENT_FIELD_NAME_WEIGHT, Fields.CONTENT_FIELD_NAME_CONTEXTS); @@ -130,60 +209,11 @@ public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - CompletionFieldMapper.Builder builder = new CompletionFieldMapper.Builder(name); - NamedAnalyzer indexAnalyzer = null; - NamedAnalyzer searchAnalyzer = null; - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - if (fieldName.equals("type")) { - continue; - } - if (Fields.ANALYZER.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - indexAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); - iterator.remove(); - } else if (Fields.SEARCH_ANALYZER.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - searchAnalyzer = getNamedAnalyzer(parserContext, fieldNode.toString()); - iterator.remove(); - } else if (Fields.PRESERVE_SEPARATORS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - builder.preserveSeparators(Boolean.parseBoolean(fieldNode.toString())); - iterator.remove(); - } else if (Fields.PRESERVE_POSITION_INCREMENTS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - builder.preservePositionIncrements(Boolean.parseBoolean(fieldNode.toString())); - iterator.remove(); - } else if (Fields.MAX_INPUT_LENGTH.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - builder.maxInputLength(Integer.parseInt(fieldNode.toString())); - iterator.remove(); - } else if (Fields.CONTEXTS.match(fieldName, LoggingDeprecationHandler.INSTANCE)) { - builder.contextMappings(ContextMappings.load(fieldNode, parserContext.indexVersionCreated())); - iterator.remove(); - } else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) { - iterator.remove(); - } - } - - if (indexAnalyzer == null) { - if (searchAnalyzer != null) { - throw new MapperParsingException("analyzer on completion field [" + name + "] must be set when search_analyzer is set"); - } - indexAnalyzer = searchAnalyzer = parserContext.getIndexAnalyzers().get("simple"); - } else if (searchAnalyzer == null) { - searchAnalyzer = indexAnalyzer; - } - - builder.indexAnalyzer(indexAnalyzer); - builder.searchAnalyzer(searchAnalyzer); + CompletionFieldMapper.Builder builder + = new CompletionFieldMapper.Builder(name, parserContext.getIndexAnalyzers().get("simple")); + builder.parse(name, parserContext, node); return builder; } - - private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) { - NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(name); - if (analyzer == null) { - throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]"); - } - return analyzer; - } } public static final class CompletionFieldType extends TermBasedFieldType { @@ -194,24 +224,9 @@ public static final class CompletionFieldType extends TermBasedFieldType { private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; private ContextMappings contextMappings = null; - public CompletionFieldType(String name, FieldType luceneFieldType, - NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map meta) { + public CompletionFieldType(String name, NamedAnalyzer searchAnalyzer, Map meta) { super(name, true, false, - new TextSearchInfo(luceneFieldType, null, searchAnalyzer, searchQuoteAnalyzer), meta); - } - - public CompletionFieldType(String name) { - this(name, Defaults.FIELD_TYPE, - new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)), - new NamedAnalyzer("completion", AnalyzerScope.INDEX, new CompletionAnalyzer(Lucene.STANDARD_ANALYZER)), - Collections.emptyMap()); - } - - private CompletionFieldType(CompletionFieldType ref) { - super(ref); - this.contextMappings = ref.contextMappings; - this.preserveSep = ref.preserveSep; - this.preservePositionIncrements = ref.preservePositionIncrements; + new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), meta); } public void setPreserveSep(boolean preserveSep) { @@ -252,14 +267,6 @@ public ContextMappings getContextMappings() { return contextMappings; } - public boolean preserveSep() { - return preserveSep; - } - - public boolean preservePositionIncrements() { - return preservePositionIncrements; - } - /** * @return postings format to use for this field-type */ @@ -302,33 +309,6 @@ public CompletionQuery fuzzyQuery(String value, Fuzziness fuzziness, int nonFuzz unicodeAware, maxExpansions); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - CompletionFieldType that = (CompletionFieldType) o; - - if (preserveSep != that.preserveSep) return false; - if (preservePositionIncrements != that.preservePositionIncrements) return false; - return Objects.equals(contextMappings, that.contextMappings); - - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), - preserveSep, - preservePositionIncrements, - contextMappings); - } - - @Override - public CompletionFieldType clone() { - return new CompletionFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -336,105 +316,24 @@ public String typeName() { } - /** - * Builder for {@link CompletionFieldMapper} - */ - public static class Builder extends FieldMapper.Builder { - - private int maxInputLength = Defaults.DEFAULT_MAX_INPUT_LENGTH; - private ContextMappings contextMappings = null; - private boolean preserveSeparators = Defaults.DEFAULT_PRESERVE_SEPARATORS; - private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; - - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(Builder.class); - - /** - * @param name of the completion field to build - */ - public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - builder = this; - } - - /** - * @param maxInputLength maximum expected prefix length - * NOTE: prefixes longer than this will - * be truncated - */ - public Builder maxInputLength(int maxInputLength) { - if (maxInputLength <= 0) { - throw new IllegalArgumentException(Fields.MAX_INPUT_LENGTH.getPreferredName() - + " must be > 0 but was [" + maxInputLength + "]"); - } - this.maxInputLength = maxInputLength; - return this; - } - - /** - * Add context mapping to this field - * @param contextMappings see {@link ContextMappings#load(Object, Version)} - */ - public Builder contextMappings(ContextMappings contextMappings) { - this.contextMappings = contextMappings; - return this; - } - - public Builder preserveSeparators(boolean preserveSeparators) { - this.preserveSeparators = preserveSeparators; - return this; - } - - public Builder preservePositionIncrements(boolean preservePositionIncrements) { - this.preservePositionIncrements = preservePositionIncrements; - return this; - } - - @Override - public CompletionFieldMapper build(BuilderContext context) { - checkCompletionContextsLimit(context); - NamedAnalyzer searchAnalyzer = new NamedAnalyzer(this.searchAnalyzer.name(), AnalyzerScope.INDEX, - new CompletionAnalyzer(this.searchAnalyzer, preserveSeparators, preservePositionIncrements)); - - CompletionFieldType ft - = new CompletionFieldType(buildFullName(context), this.fieldType, searchAnalyzer, searchAnalyzer, meta); - ft.setContextMappings(contextMappings); - ft.setPreservePositionIncrements(preservePositionIncrements); - ft.setPreserveSep(preserveSeparators); - ft.setIndexAnalyzer(indexAnalyzer); - return new CompletionFieldMapper(name, this.fieldType, ft, - multiFieldsBuilder.build(this, context), copyTo, maxInputLength); - } - - private void checkCompletionContextsLimit(BuilderContext context) { - if (this.contextMappings != null && this.contextMappings.size() > COMPLETION_CONTEXTS_LIMIT) { - if (context.indexCreatedVersion().onOrAfter(Version.V_8_0_0)) { - throw new IllegalArgumentException( - "Limit of completion field contexts [" + COMPLETION_CONTEXTS_LIMIT + "] has been exceeded"); - } else { - deprecationLogger.deprecate("excessive_completion_contexts", - "You have defined more than [" + COMPLETION_CONTEXTS_LIMIT + "] completion contexts" + - " in the mapping for index [" + context.indexSettings().get(IndexMetadata.SETTING_INDEX_PROVIDED_NAME) + "]. " + - "The maximum allowed number of completion contexts in a mapping will be limited to " + - "[" + COMPLETION_CONTEXTS_LIMIT + "] starting in version [8.0]."); - } - } - } - - @Override - public Builder index(boolean index) { - if (index == false) { - throw new MapperParsingException("Completion field type must be indexed"); - } - return builder; - } - } - - private int maxInputLength; - - public CompletionFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - MultiFields multiFields, CopyTo copyTo, int maxInputLength) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.maxInputLength = maxInputLength; + private final int maxInputLength; + private final boolean preserveSeparators; + private final boolean preservePosInc; + private final NamedAnalyzer defaultAnalyzer; + private final NamedAnalyzer analyzer; + private final NamedAnalyzer searchAnalyzer; + private final ContextMappings contexts; + + public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, NamedAnalyzer defaultAnalyzer, + MultiFields multiFields, CopyTo copyTo, Builder builder) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.defaultAnalyzer = defaultAnalyzer; + this.maxInputLength = builder.maxInputLength.getValue(); + this.preserveSeparators = builder.preserveSeparators.getValue(); + this.preservePosInc = builder.preservePosInc.getValue(); + this.analyzer = builder.analyzer.getValue(); + this.searchAnalyzer = builder.searchAnalyzer.getValue(); + this.contexts = builder.contexts.getValue(); } @Override @@ -640,28 +539,6 @@ public String toString() { } } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(simpleName()) - .field(Fields.TYPE.getPreferredName(), CONTENT_TYPE); - builder.field(Fields.ANALYZER.getPreferredName(), fieldType().indexAnalyzer().name()); - if (fieldType().indexAnalyzer().name().equals(fieldType().getTextSearchInfo().getSearchAnalyzer().name()) == false) { - builder.field(Fields.SEARCH_ANALYZER.getPreferredName(), fieldType().getTextSearchInfo().getSearchAnalyzer().name()); - } - builder.field(Fields.PRESERVE_SEPARATORS.getPreferredName(), fieldType().preserveSep()); - builder.field(Fields.PRESERVE_POSITION_INCREMENTS.getPreferredName(), fieldType().preservePositionIncrements()); - builder.field(Fields.MAX_INPUT_LENGTH.getPreferredName(), this.maxInputLength); - - if (fieldType().hasContextMappings()) { - builder.startArray(Fields.CONTEXTS.getPreferredName()); - fieldType().getContextMappings().toXContent(builder, params); - builder.endArray(); - } - - multiFields.toXContent(builder, params); - return builder.endObject(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { // no-op @@ -672,23 +549,4 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - CompletionFieldType c = (CompletionFieldType)other.fieldType(); - - if (fieldType().preservePositionIncrements != c.preservePositionIncrements) { - conflicts.add("mapper [" + name() + "] has different [preserve_position_increments] values"); - } - if (fieldType().preserveSep != c.preserveSep) { - conflicts.add("mapper [" + name() + "] has different [preserve_separators] values"); - } - if (fieldType().hasContextMappings() != c.hasContextMappings()) { - conflicts.add("mapper [" + name() + "] has different [context_mappings] values"); - } else if (fieldType().hasContextMappings() && fieldType().contextMappings.equals(c.contextMappings) == false) { - conflicts.add("mapper [" + name() + "] has different [context_mappings] values"); - } - - this.maxInputLength = ((CompletionFieldMapper)other).maxInputLength; - } - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index e7f7503a50920..6d7b5847884bd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -45,10 +45,6 @@ public ConstantFieldType(String name, Map meta) { super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } - public ConstantFieldType(ConstantFieldType other) { - super(other); - } - @Override public final boolean isSearchable() { return true; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java index 7f3e26312ad87..015293864e55b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java @@ -45,6 +45,13 @@ public ContentPath(int offset) { this.index = 0; } + public ContentPath(String path) { + this.sb = new StringBuilder(); + this.offset = 0; + this.index = 0; + add(path); + } + public void add(String name) { path[index++] = name; if (index == path.length) { // expand if needed diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 69de862ac54ab..5d80d18d7df61 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -64,6 +64,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.function.LongSupplier; import static org.elasticsearch.common.time.DateUtils.toLong; @@ -286,7 +287,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (propName.equals("format")) { builder.format(propNode.toString()); iterator.remove(); - } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) { + } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { iterator.remove(); } } @@ -311,32 +312,6 @@ public DateFieldType(String name) { this(name, true, true, DEFAULT_DATE_TIME_FORMATTER, Resolution.MILLISECONDS, Collections.emptyMap()); } - DateFieldType(DateFieldType other) { - super(other); - this.dateTimeFormatter = other.dateTimeFormatter; - this.dateMathParser = other.dateMathParser; - this.resolution = other.resolution; - } - - @Override - public MappedFieldType clone() { - return new DateFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) { - return false; - } - DateFieldType that = (DateFieldType) o; - return Objects.equals(dateTimeFormatter, that.dateTimeFormatter) && Objects.equals(resolution, that.resolution); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), dateTimeFormatter, resolution); - } - @Override public String typeName() { return resolution.type(); @@ -490,6 +465,14 @@ public Relation isFieldWithinQuery(IndexReader reader, } } + @Override + public Function pointReaderIfPossible() { + if (isSearchable()) { + return resolution()::parsePointAsMillis; + } + return null; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 070a192edacda..9d21c4a76b766 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.AbstractXContentParser; @@ -292,7 +293,7 @@ protected FieldMapper clone() { } @Override - public final FieldMapper merge(Mapper mergeWith) { + public FieldMapper merge(Mapper mergeWith) { FieldMapper merged = clone(); List conflicts = new ArrayList<>(); if (mergeWith instanceof FieldMapper == false) { @@ -487,7 +488,7 @@ public static String termVectorOptionsToString(FieldType fieldType) { protected abstract String contentType(); - public static class MultiFields { + public static class MultiFields implements Iterable { public static MultiFields empty() { return new MultiFields(ImmutableOpenMap.of()); @@ -502,8 +503,29 @@ public Builder add(Mapper.Builder builder) { return this; } + public Builder add(Mapper mapper) { + mapperBuilders.put(mapper.simpleName(), new Mapper.Builder(mapper.simpleName()) { + @Override + public Mapper build(BuilderContext context) { + return mapper; + } + }); + return this; + } + + public Builder update(Mapper toMerge, ContentPath contentPath) { + if (mapperBuilders.containsKey(toMerge.simpleName()) == false) { + add(toMerge); + } else { + Mapper.Builder builder = mapperBuilders.get(toMerge.simpleName()); + Mapper existing = builder.build(new BuilderContext(Settings.EMPTY, contentPath)); + add(existing.merge(toMerge)); + } + return this; + } + @SuppressWarnings("unchecked") - public MultiFields build(FieldMapper.Builder mainFieldBuilder, BuilderContext context) { + public MultiFields build(Mapper.Builder mainFieldBuilder, BuilderContext context) { if (mapperBuilders.isEmpty()) { return empty(); } else { @@ -568,6 +590,7 @@ public MultiFields merge(MultiFields mergeWith) { return new MultiFields(mappers); } + @Override public Iterator iterator() { return StreamSupport.stream(mappers.values().spliterator(), false).map((p) -> (Mapper)p.value).iterator(); } @@ -634,6 +657,11 @@ public CopyTo build() { } return new CopyTo(Collections.unmodifiableList(copyToBuilders)); } + + public void reset(CopyTo copyTo) { + copyToBuilders.clear(); + copyToBuilders.addAll(copyTo.copyToFields); + } } public List copyToFields() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index c495ee4ac7cf2..f7b28669c88db 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; -import java.util.Objects; /** * A mapper that indexes the field names of a document under _field_names. This mapper is typically useful in order @@ -129,28 +128,6 @@ public FieldNamesFieldType() { super(Defaults.NAME, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected FieldNamesFieldType(FieldNamesFieldType ref) { - super(ref); - this.enabled = ref.enabled; - } - - @Override - public FieldNamesFieldType clone() { - return new FieldNamesFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - FieldNamesFieldType that = (FieldNamesFieldType) o; - return enabled == that.enabled; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), enabled); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index d8400ba75cdaf..482996d307e64 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -172,20 +172,11 @@ public GeoPointFieldType(String name) { this(name, true, true, Collections.emptyMap()); } - GeoPointFieldType(GeoPointFieldType ref) { - super(ref); - } - @Override public String typeName() { return CONTENT_TYPE; } - @Override - public MappedFieldType clone() { - return new GeoPointFieldType(this); - } - @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 9d01ec57c8099..6ad1de5b868cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -91,15 +91,6 @@ public GeoShapeFieldType(String name, boolean indexed, boolean hasDocValues, Map super(name, indexed, hasDocValues, meta); } - protected GeoShapeFieldType(GeoShapeFieldType ref) { - super(ref); - } - - @Override - public GeoShapeFieldType clone() { - return new GeoShapeFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 0a3f0e38cc0cb..4c9fd6ab51c3f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -120,15 +120,6 @@ private IdFieldType() { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected IdFieldType(IdFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new IdFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index e93a6dfd95bed..d6ad764a596be 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -87,15 +87,6 @@ private IgnoredFieldType() { super(NAME, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected IgnoredFieldType(IgnoredFieldType ref) { - super(ref); - } - - @Override - public IgnoredFieldType clone() { - return new IgnoredFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 97e0cc66b3fff..eb689d74be316 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -87,15 +87,6 @@ private IndexFieldType() { super(NAME, Collections.emptyMap()); } - protected IndexFieldType(IndexFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new IndexFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index d836bc0284eb3..ad18a1fa94b5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -130,7 +130,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + ".ignore_malformed")); iterator.remove(); - } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) { + } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { iterator.remove(); } } @@ -148,15 +148,6 @@ public IpFieldType(String name) { this(name, true, true, Collections.emptyMap()); } - IpFieldType(IpFieldType other) { - super(other); - } - - @Override - public MappedFieldType clone() { - return new IpFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index ef1c8d9fca372..ee476631b2b86 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -256,18 +256,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); } - protected KeywordFieldType(KeywordFieldType ref) { - super(ref); - this.hasNorms = ref.hasNorms; - setEagerGlobalOrdinals(ref.eagerGlobalOrdinals()); - setIndexAnalyzer(ref.indexAnalyzer()); - } - - @Override - public KeywordFieldType clone() { - return new KeywordFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 60c93e1ce300c..5ee6f12232f27 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -50,7 +50,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; /** * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. @@ -301,44 +300,6 @@ public GeoShapeFieldType(String name) { this(name, true, true, Collections.emptyMap()); } - protected GeoShapeFieldType(GeoShapeFieldType ref) { - super(ref); - this.tree = ref.tree; - this.strategy = ref.strategy; - this.pointsOnly = ref.pointsOnly; - this.treeLevels = ref.treeLevels; - this.precisionInMeters = ref.precisionInMeters; - this.distanceErrorPct = ref.distanceErrorPct; - this.defaultDistanceErrorPct = ref.defaultDistanceErrorPct; - this.defaultPrefixTreeStrategy = ref.defaultPrefixTreeStrategy; - this.recursiveStrategy = ref.recursiveStrategy; - this.termStrategy = ref.termStrategy; - } - - @Override - public GeoShapeFieldType clone() { - return new GeoShapeFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - GeoShapeFieldType that = (GeoShapeFieldType) o; - return treeLevels == that.treeLevels && - precisionInMeters == that.precisionInMeters && - defaultDistanceErrorPct == that.defaultDistanceErrorPct && - Objects.equals(tree, that.tree) && - Objects.equals(strategy, that.strategy) && - pointsOnly == that.pointsOnly && - Objects.equals(distanceErrorPct, that.distanceErrorPct); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), tree, strategy, pointsOnly, treeLevels, precisionInMeters, distanceErrorPct, - defaultDistanceErrorPct); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 4bb854c5d98e3..15a55fb5566b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -53,6 +53,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; /** * This defines the core properties and functions to operate on a field. @@ -68,17 +69,6 @@ public abstract class MappedFieldType { private boolean eagerGlobalOrdinals; private Map meta; - protected MappedFieldType(MappedFieldType ref) { - this.name = ref.name(); - this.boost = ref.boost(); - this.isIndexed = ref.isIndexed; - this.docValues = ref.hasDocValues(); - this.indexAnalyzer = ref.indexAnalyzer(); - this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; - this.meta = ref.meta; - this.textSearchInfo = ref.textSearchInfo; - } - public MappedFieldType(String name, boolean isIndexed, boolean hasDocValues, TextSearchInfo textSearchInfo, Map meta) { setBoost(1.0f); this.name = Objects.requireNonNull(name); @@ -88,9 +78,6 @@ public MappedFieldType(String name, boolean isIndexed, boolean hasDocValues, Tex this.meta = meta; } - @Override - public abstract MappedFieldType clone(); - /** * Return a fielddata builder for this field * @@ -104,32 +91,9 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) { - return false; - } - MappedFieldType fieldType = (MappedFieldType) o; - - return boost == fieldType.boost && - docValues == fieldType.docValues && - Objects.equals(name, fieldType.name) && - Objects.equals(indexAnalyzer, fieldType.indexAnalyzer) && - Objects.equals(eagerGlobalOrdinals, fieldType.eagerGlobalOrdinals) && - Objects.equals(meta, fieldType.meta); - } - - @Override - public int hashCode() { - return Objects.hash(name, boost, docValues, indexAnalyzer, - eagerGlobalOrdinals, meta); - } - - // TODO: we need to override freeze() and add safety checks that all settings are actually set - /** Returns the name of this type, as would be specified in mapping properties */ public abstract String typeName(); - + /** Returns the field family type, as used in field capabilities */ public String familyTypeName() { return typeName(); @@ -173,6 +137,17 @@ public boolean isSearchable() { return isIndexed; } + /** + * If the field supports using the indexed data to speed up operations related to ordering of data, such as sorting or aggs, return + * a function for doing that. If it is unsupported for this field type, there is no need to override this method. + * + * @return null if the optimization cannot be applied, otherwise a function to use for the optimization + */ + @Nullable + public Function pointReaderIfPossible() { + return null; + } + /** Returns true if the field is aggregatable. * */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java index 008f17cd38199..992f9345bb7b5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeValidator.java @@ -119,7 +119,7 @@ public static void validateFieldReferences(List fieldMappers, DocumentMapper newMapper) { validateCopyTo(fieldMappers, fullPathObjectMappers, fieldTypes); validateFieldAliasTargets(fieldAliasMappers, fullPathObjectMappers); - validateTimestampFieldMapper(metadataMappers, newMapper); + validateMetadataFieldMappers(metadataMappers, newMapper); } private static void validateCopyTo(List fieldMappers, @@ -174,11 +174,9 @@ private static void validateFieldAliasTargets(List fieldAliasM } } - private static void validateTimestampFieldMapper(MetadataFieldMapper[] metadataMappers, DocumentMapper newMapper) { + private static void validateMetadataFieldMappers(MetadataFieldMapper[] metadataMappers, DocumentMapper newMapper) { for (MetadataFieldMapper metadataFieldMapper : metadataMappers) { - if (metadataFieldMapper instanceof TimestampFieldMapper) { - ((TimestampFieldMapper) metadataFieldMapper).validate(newMapper.mappers()); - } + metadataFieldMapper.validate(newMapper.mappers()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index c6be5abeb47a1..4bd2048ad0e65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -169,7 +169,7 @@ public DocumentMapperParser documentMapperParser() { /** * Parses the mappings (formatted as JSON) into a map */ - public static Map parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws Exception { + public static Map parseMapping(NamedXContentRegistry xContentRegistry, String mappingSource) throws IOException { try (XContentParser parser = XContentType.JSON.xContent() .createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, mappingSource)) { return parser.map(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 54b9aaf57027b..05b18099e7b1e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -66,6 +66,14 @@ protected MetadataFieldMapper(FieldType fieldType, MappedFieldType mappedFieldTy super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } + /** + * Called when mapping gets merged. Provides the opportunity to validate other fields a metadata field mapper + * is supposed to work with before a mapping update is completed. + */ + public void validate(DocumentFieldMappers lookup) { + // noop by default + } + /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index dc5132bd1bf95..9723de6382829 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -89,15 +89,6 @@ public static final class NestedPathFieldType extends StringFieldType { super(NestedPathFieldMapper.name(settings), true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected NestedPathFieldType(NestedPathFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new NestedPathFieldType(this); - } - @Override public String typeName() { return NAME; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6b7bd4e374c12..106609f97cdeb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -66,6 +66,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ public class NumberFieldMapper extends FieldMapper { @@ -913,16 +914,6 @@ public NumberFieldType(String name, NumberType type) { this(name, type, true, true, Collections.emptyMap()); } - private NumberFieldType(NumberFieldType other) { - super(other); - this.type = other.type; - } - - @Override - public MappedFieldType clone() { - return new NumberFieldType(this); - } - @Override public String typeName() { return type.name; @@ -971,6 +962,14 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower return query; } + @Override + public Function pointReaderIfPossible() { + if (isSearchable()) { + return this::parsePoint; + } + return null; + } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); @@ -1001,20 +1000,6 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public Number parsePoint(byte[] value) { return type.parsePoint(value); } - - @Override - public boolean equals(Object o) { - if (super.equals(o) == false) { - return false; - } - NumberFieldType that = (NumberFieldType) o; - return type == that.type; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), type); - } } private Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java new file mode 100644 index 0000000000000..ed8d8b1a7f1e6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java @@ -0,0 +1,475 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.Version; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.Set; +import java.util.function.Function; + +/** + * Defines how a particular field should be indexed and searched + * + * Configuration {@link Parameter}s for the mapper are defined on a {@link Builder} subclass, + * and returned by its {@link Builder#getParameters()} method. Merging, serialization + * and parsing of the mapper are all mediated through this set of parameters. + * + * Subclasses should implement a {@link Builder} that is returned from the + * {@link #getMergeBuilder()} method, initialised with the existing builder. + */ +public abstract class ParametrizedFieldMapper extends FieldMapper { + + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ParametrizedFieldMapper.class); + + /** + * Creates a new ParametrizedFieldMapper + */ + protected ParametrizedFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, new FieldType(), mappedFieldType, multiFields, copyTo); + } + + /** + * Returns a {@link Builder} to be used for merging and serialization + * + * Implement as follows: + * {@code return new MyBuilder(simpleName()).init(this); } + */ + public abstract ParametrizedFieldMapper.Builder getMergeBuilder(); + + @Override + public final ParametrizedFieldMapper merge(Mapper mergeWith) { + + if (mergeWith instanceof FieldMapper == false) { + throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" + + contentType() + "] to [" + mergeWith.getClass().getSimpleName() + "]"); + } + if (Objects.equals(this.getClass(), mergeWith.getClass()) == false) { + throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" + + contentType() + "] to [" + ((FieldMapper) mergeWith).contentType() + "]"); + } + + ParametrizedFieldMapper.Builder builder = getMergeBuilder(); + Conflicts conflicts = new Conflicts(name()); + builder.merge((FieldMapper) mergeWith, conflicts); + conflicts.check(); + return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name()))); + } + + private static ContentPath parentPath(String name) { + int endPos = name.lastIndexOf("."); + if (endPos == -1) { + return new ContentPath(0); + } + return new ContentPath(name.substring(0, endPos)); + } + + @Override + protected final void mergeOptions(FieldMapper other, List conflicts) { + // TODO remove when everything is parametrized + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(simpleName()); + builder.field("type", contentType()); + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + getMergeBuilder().toXContent(builder, includeDefaults); + multiFields.toXContent(builder, params); + copyTo.toXContent(builder, params); + return builder.endObject(); + } + + /** + * Serializes a parameter + */ + protected interface Serializer { + void serialize(XContentBuilder builder, String name, T value) throws IOException; + } + + /** + * A configurable parameter for a field mapper + * @param the type of the value the parameter holds + */ + public static final class Parameter { + + public final String name; + private final List deprecatedNames = new ArrayList<>(); + private final T defaultValue; + private final TriFunction parser; + private final Function initializer; + private final boolean updateable; + private boolean acceptsNull = false; + private Consumer validator = null; + private Serializer serializer = XContentBuilder::field; + private T value; + + /** + * Creates a new Parameter + * @param name the parameter name, used in parsing and serialization + * @param updateable whether the parameter can be updated with a new value during a mapping update + * @param defaultValue the default value for the parameter, used if unspecified in mappings + * @param parser a function that converts an object to a parameter value + * @param initializer a function that reads a parameter value from an existing mapper + */ + public Parameter(String name, boolean updateable, T defaultValue, + TriFunction parser, Function initializer) { + this.name = name; + this.defaultValue = defaultValue; + this.value = defaultValue; + this.parser = parser; + this.initializer = initializer; + this.updateable = updateable; + } + + /** + * Returns the current value of the parameter + */ + public T getValue() { + return value; + } + + /** + * Sets the current value of the parameter + */ + public void setValue(T value) { + this.value = value; + } + + /** + * Allows the parameter to accept a {@code null} value + */ + public Parameter acceptsNull() { + this.acceptsNull = true; + return this; + } + + /** + * Adds a deprecated parameter name. + * + * If this parameter name is encountered during parsing, a deprecation warning will + * be emitted. The parameter will be serialized with its main name. + */ + public Parameter addDeprecatedName(String deprecatedName) { + this.deprecatedNames.add(deprecatedName); + return this; + } + + /** + * Adds validation to a parameter, called after parsing and merging + */ + public Parameter setValidator(Consumer validator) { + this.validator = validator; + return this; + } + + /** + * Configure a custom serializer for this parameter + */ + public Parameter setSerializer(Serializer serializer) { + this.serializer = serializer; + return this; + } + + private void validate() { + if (validator != null) { + validator.accept(value); + } + } + + private void init(FieldMapper toInit) { + setValue(initializer.apply(toInit)); + } + + private void parse(String field, ParserContext context, Object in) { + setValue(parser.apply(field, context, in)); + } + + private void merge(FieldMapper toMerge, Conflicts conflicts) { + T value = initializer.apply(toMerge); + if (updateable == false && Objects.equals(this.value, value) == false) { + conflicts.addConflict(name, this.value.toString(), value.toString()); + } else { + setValue(value); + } + } + + private void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (includeDefaults || (Objects.equals(defaultValue, value) == false)) { + serializer.serialize(builder, name, value); + } + } + + /** + * Defines a parameter that takes the values {@code true} or {@code false} + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter boolParam(String name, boolean updateable, + Function initializer, boolean defaultValue) { + return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), initializer); + } + + /** + * Defines a parameter that takes a float value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter floatParam(String name, boolean updateable, + Function initializer, float defaultValue) { + return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeFloatValue(o), initializer); + } + + /** + * Defines a parameter that takes an integer value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter intParam(String name, boolean updateable, + Function initializer, int defaultValue) { + return new Parameter<>(name, updateable, defaultValue, (n, c, o) -> XContentMapValues.nodeIntegerValue(o), initializer); + } + + /** + * Defines a parameter that takes a string value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter stringParam(String name, boolean updateable, + Function initializer, String defaultValue) { + return new Parameter<>(name, updateable, defaultValue, + (n, c, o) -> XContentMapValues.nodeStringValue(o), initializer); + } + + /** + * Defines a parameter that takes an analyzer name + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultAnalyzer the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter analyzerParam(String name, boolean updateable, + Function initializer, + NamedAnalyzer defaultAnalyzer) { + return new Parameter<>(name, updateable, defaultAnalyzer, (n, c, o) -> { + String analyzerName = o.toString(); + NamedAnalyzer a = c.getIndexAnalyzers().get(analyzerName); + if (a == null) { + throw new IllegalArgumentException("analyzer [" + analyzerName + "] has not been configured in mappings"); + } + return a; + }, initializer).setSerializer((b, n, v) -> b.field(n, v.name())); + } + + /** + * Declares a metadata parameter + */ + public static Parameter> metaParam() { + return new Parameter<>("meta", true, Collections.emptyMap(), + (n, c, o) -> TypeParsers.parseMeta(n, o), m -> m.fieldType().meta()); + } + + } + + private static final class Conflicts { + + private final String mapperName; + private final List conflicts = new ArrayList<>(); + + Conflicts(String mapperName) { + this.mapperName = mapperName; + } + + void addConflict(String parameter, String existing, String toMerge) { + conflicts.add("Cannot update parameter [" + parameter + "] from [" + existing + "] to [" + toMerge + "]"); + } + + void check() { + if (conflicts.isEmpty()) { + return; + } + String message = "Mapper for [" + mapperName + "] conflicts with existing mapper:\n\t" + + String.join("\n\t", conflicts); + throw new IllegalArgumentException(message); + } + + } + + /** + * A Builder for a ParametrizedFieldMapper + */ + public abstract static class Builder extends Mapper.Builder { + + protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder(); + protected final CopyTo.Builder copyTo = new CopyTo.Builder(); + + /** + * Creates a new Builder with a field name + */ + protected Builder(String name) { + super(name); + } + + /** + * Initialises all parameters from an existing mapper + */ + public Builder init(FieldMapper initializer) { + for (Parameter param : getParameters()) { + param.init(initializer); + } + for (Mapper subField : initializer.multiFields) { + multiFieldsBuilder.add(subField); + } + return this; + } + + private void merge(FieldMapper in, Conflicts conflicts) { + for (Parameter param : getParameters()) { + param.merge(in, conflicts); + } + for (Mapper newSubField : in.multiFields) { + multiFieldsBuilder.update(newSubField, parentPath(newSubField.name())); + } + this.copyTo.reset(in.copyTo); + validate(); + } + + private void validate() { + for (Parameter param : getParameters()) { + param.validate(); + } + } + + /** + * @return the list of parameters defined for this mapper + */ + protected abstract List> getParameters(); + + @Override + public abstract ParametrizedFieldMapper build(BuilderContext context); + + /** + * Builds the full name of the field, taking into account parent objects + */ + protected String buildFullName(BuilderContext context) { + return context.path().pathAsText(name); + } + + private void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + for (Parameter parameter : getParameters()) { + parameter.toXContent(builder, includeDefaults); + } + } + + /** + * Parse mapping parameters from a map of mappings + * @param name the field mapper name + * @param parserContext the parser context + * @param fieldNode the root node of the map of mappings for this field + */ + public final void parse(String name, ParserContext parserContext, Map fieldNode) { + Map> paramsMap = new HashMap<>(); + Map> deprecatedParamsMap = new HashMap<>(); + for (Parameter param : getParameters()) { + paramsMap.put(param.name, param); + for (String deprecatedName : param.deprecatedNames) { + deprecatedParamsMap.put(deprecatedName, param); + } + } + String type = (String) fieldNode.remove("type"); + for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + final String propName = entry.getKey(); + final Object propNode = entry.getValue(); + if (Objects.equals("fields", propName)) { + TypeParsers.parseMultiField(multiFieldsBuilder::add, name, parserContext, propName, propNode); + iterator.remove(); + continue; + } + if (Objects.equals("copy_to", propName)) { + TypeParsers.parseCopyFields(propNode).forEach(copyTo::add); + iterator.remove(); + continue; + } + Parameter parameter = deprecatedParamsMap.get(propName); + if (parameter != null) { + deprecationLogger.deprecate(propName, "Parameter [{}] on mapper [{}] is deprecated, use [{}]", + propName, name, parameter.name); + } else { + parameter = paramsMap.get(propName); + } + if (parameter == null) { + if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) { + deprecationLogger.deprecate(propName, + "Parameter [{}] has no effect on type [{}] and will be removed in future", propName, type); + iterator.remove(); + continue; + } + throw new MapperParsingException("unknown parameter [" + propName + + "] on mapper [" + name + "] of type [" + type + "]"); + } + if (propNode == null && parameter.acceptsNull == false) { + throw new MapperParsingException("[" + propName + "] on mapper [" + name + + "] of type [" + type + "] must not have a [null] value"); + } + parameter.parse(name, parserContext, propNode); + iterator.remove(); + } + validate(); + } + + // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they + // made no sense; if we've got here, that means that they're not declared on a current mapper, + // and so we emit a deprecation warning rather than failing a previously working mapping. + private static final Set DEPRECATED_PARAMS + = Set.of("store", "meta", "index", "doc_values", "boost", "index_options", "similarity"); + + private static boolean isDeprecatedParameter(String propName, Version indexCreatedVersion) { + if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { + return false; + } + return DEPRECATED_PARAMS.contains(propName); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 8d887a57a1547..28874363b456f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -172,7 +172,7 @@ public Mapper.Builder parse(String name, Map node, } else if (propName.equals("format")) { builder.format(propNode.toString()); iterator.remove(); - } else if (TypeParsers.parseMultiField(builder, name, parserContext, propName, propNode)) { + } else if (TypeParsers.parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { iterator.remove(); } } @@ -210,35 +210,8 @@ public RangeFieldType(String name, DateFormatter formatter) { this(name, true, true, formatter, Collections.emptyMap()); } - RangeFieldType(RangeFieldType other) { - super(other); - this.rangeType = other.rangeType; - this.dateTimeFormatter = other.dateTimeFormatter; - this.dateMathParser = other.dateMathParser; - } - public RangeType rangeType() { return rangeType; } - @Override - public RangeFieldType clone() { - return new RangeFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (!super.equals(o)) return false; - RangeFieldType that = (RangeFieldType) o; - return Objects.equals(rangeType, that.rangeType) && - (rangeType == RangeType.DATE) ? - Objects.equals(dateTimeFormatter, that.dateTimeFormatter) - : dateTimeFormatter == null && that.dateTimeFormatter == null; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), rangeType, dateTimeFormatter); - } - @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 366730060d706..a66287edc6d5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -107,15 +107,6 @@ private RoutingFieldType() { setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - protected RoutingFieldType(RoutingFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new RoutingFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 30e548c3add4e..c98bc5de4a9fc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -124,15 +124,6 @@ static final class SeqNoFieldType extends SimpleMappedFieldType { super(NAME, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected SeqNoFieldType(SeqNoFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new SeqNoFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SimpleMappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/SimpleMappedFieldType.java index 8253bd599bd98..918d5665df62d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SimpleMappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SimpleMappedFieldType.java @@ -37,10 +37,6 @@ protected SimpleMappedFieldType(String name, boolean isSearchable, boolean hasDo super(name, isSearchable, hasDocValues, textSearchInfo, meta); } - protected SimpleMappedFieldType(MappedFieldType ref) { - super(ref); - } - @Override public final Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, ShapeRelation relation, ZoneId timeZone, DateMathParser parser, QueryShardContext context) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 24a9b42a5ead0..021001b174c6b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -152,15 +152,6 @@ private SourceFieldType() { super(NAME, false, false, TextSearchInfo.NONE, Collections.emptyMap()); } - protected SourceFieldType(SourceFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new SourceFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java index 94c329a6aa462..c6a7104973d55 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java @@ -55,10 +55,6 @@ public StringFieldType(String name, boolean isSearchable, boolean hasDocValues, super(name, isSearchable, hasDocValues, textSearchInfo, meta); } - protected StringFieldType(MappedFieldType ref) { - super(ref); - } - @Override public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions, QueryShardContext context) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java index 5d6db99baf972..0101b68fb88c8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java @@ -39,10 +39,6 @@ abstract class TermBasedFieldType extends SimpleMappedFieldType { super(name, isSearchable, hasDocValues, textSearchInfo, meta); } - protected TermBasedFieldType(MappedFieldType ref) { - super(ref); - } - /** Returns the indexed value used to construct search "values". * This method is used for the default implementations of most * query factory methods such as {@link #termQuery}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index e459757dcc5b4..503b2f98865a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -387,11 +387,6 @@ void setAnalyzer(String name, Analyzer delegate) { setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate))); } - @Override - public MappedFieldType clone() { - return new PhraseFieldType(parent); - } - @Override public String typeName() { return "phrase"; @@ -418,6 +413,13 @@ static final class PrefixFieldType extends StringFieldType { this.hasPositions = hasPositions; } + static boolean canMerge(PrefixFieldType first, PrefixFieldType second) { + if (first == null) { + return second == null; + } + return second != null && first.minChars == second.minChars && first.maxChars == second.maxChars; + } + void setAnalyzer(NamedAnalyzer delegate) { setIndexAnalyzer(new NamedAnalyzer(delegate.name(), AnalyzerScope.INDEX, new PrefixWrappedAnalyzer(delegate.analyzer(), minChars, maxChars))); @@ -467,11 +469,6 @@ public IntervalsSource intervals(BytesRef term) { return Intervals.or(Intervals.fixField(name(), Intervals.wildcard(new BytesRef(wildcardTerm))), Intervals.term(term)); } - @Override - public PrefixFieldType clone() { - return new PrefixFieldType(parentField, name(), minChars, maxChars, hasPositions); - } - @Override public String typeName() { return "prefix"; @@ -486,21 +483,6 @@ public String toString() { public Query existsQuery(QueryShardContext context) { throw new UnsupportedOperationException(); } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - PrefixFieldType that = (PrefixFieldType) o; - return minChars == that.minChars && - maxChars == that.maxChars; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), minChars, maxChars); - } } private static final class PhraseFieldMapper extends FieldMapper { @@ -588,44 +570,6 @@ public TextFieldType(String name) { this(name, Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap()); } - protected TextFieldType(TextFieldType ref) { - super(ref); - this.fielddata = ref.fielddata; - this.fielddataMinFrequency = ref.fielddataMinFrequency; - this.fielddataMaxFrequency = ref.fielddataMaxFrequency; - this.fielddataMinSegmentSize = ref.fielddataMinSegmentSize; - this.indexPhrases = ref.indexPhrases; - if (ref.prefixFieldType != null) { - this.prefixFieldType = ref.prefixFieldType.clone(); - } - this.indexedFieldType = ref.indexedFieldType; - } - - @Override - public TextFieldType clone() { - return new TextFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (super.equals(o) == false) { - return false; - } - TextFieldType that = (TextFieldType) o; - return fielddata == that.fielddata - && indexPhrases == that.indexPhrases - && Objects.equals(prefixFieldType, that.prefixFieldType) - && fielddataMinFrequency == that.fielddataMinFrequency - && fielddataMaxFrequency == that.fielddataMaxFrequency - && fielddataMinSegmentSize == that.fielddataMinSegmentSize; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), fielddata, indexPhrases, prefixFieldType, - fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize); - } - public boolean fielddata() { return fielddata; } @@ -909,7 +853,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { if (mw.fieldType().indexPhrases != this.fieldType().indexPhrases) { conflicts.add("mapper [" + name() + "] has different [index_phrases] settings"); } - if (Objects.equals(mw.fieldType().prefixFieldType, this.fieldType().prefixFieldType) == false) { + if (PrefixFieldType.canMerge(mw.fieldType().prefixFieldType, this.fieldType().prefixFieldType) == false) { conflicts.add("mapper [" + name() + "] has different [index_prefixes] settings"); } if (this.prefixFieldMapper != null && mw.prefixFieldMapper != null) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 5f93e7257384b..60e0ddba18c28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -89,15 +89,6 @@ private TypeFieldType() { super(NAME, Collections.emptyMap()); } - protected TypeFieldType(TypeFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new TypeFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index 7fc98ec2f5229..8754bdd4ce821 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -29,10 +29,12 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.similarity.SimilarityProvider; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -181,12 +183,7 @@ public static void checkNull(String propName, Object propNode) { /** * Parse the {@code meta} key of the mapping. */ - public static void parseMeta(FieldMapper.Builder builder, String name, Map fieldNode) { - Object metaObject = fieldNode.remove("meta"); - if (metaObject == null) { - // no meta - return; - } + public static Map parseMeta(String name, Object metaObject) { if (metaObject instanceof Map == false) { throw new MapperParsingException("[meta] must be an object, got " + metaObject.getClass().getSimpleName() + "[" + metaObject + "] for field [" + name +"]"); @@ -219,17 +216,17 @@ public static void parseMeta(FieldMapper.Builder builder, String name, Map, Object> entryValueFunction = Map.Entry::getValue; final Function stringCast = String.class::cast; - Map checkedMeta = meta.entrySet().stream() + return meta.entrySet().stream() .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, entryValueFunction.andThen(stringCast))); - builder.meta(checkedMeta); } + + /** * Parse common field attributes such as {@code doc_values} or {@code store}. */ public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, Mapper.TypeParser.ParserContext parserContext) { - parseMeta(builder, name, fieldNode); for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); final String propName = entry.getKey(); @@ -238,6 +235,9 @@ public static void parseField(FieldMapper.Builder builder, String name, Map builder, String name, Map copyFields = parseCopyFields(propNode); + FieldMapper.CopyTo.Builder cpBuilder = new FieldMapper.CopyTo.Builder(); + copyFields.forEach(cpBuilder::add); + builder.copyTo(cpBuilder.build()); } iterator.remove(); } @@ -269,8 +272,8 @@ public static void parseField(FieldMapper.Builder builder, String name, Map multiFieldsBuilder, String name, + Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) { if (propName.equals("fields")) { if (parserContext.isWithinMultiField()) { // For indices created prior to 8.0, we only emit a deprecation warning and do not fail type parsing. This is to @@ -327,7 +330,7 @@ public static boolean parseMultiField(FieldMapper.Builder builder, String name, if (typeParser == null) { throw new MapperParsingException("no handler for type [" + type + "] declared on field [" + multiFieldName + "]"); } - builder.addMultiField(typeParser.parse(multiFieldName, multiFieldNodes, parserContext)); + multiFieldsBuilder.accept(typeParser.parse(multiFieldName, multiFieldNodes, parserContext)); multiFieldNodes.remove("type"); DocumentMapperParser.checkNoRemainingFields(propName, multiFieldNodes, parserContext.indexVersionCreated()); } @@ -383,17 +386,16 @@ public static void parseTermVector(String fieldName, String termVector, FieldMap } } - @SuppressWarnings({"unchecked", "rawtypes"}) - public static void parseCopyFields(Object propNode, FieldMapper.Builder builder) { - FieldMapper.CopyTo.Builder copyToBuilder = new FieldMapper.CopyTo.Builder(); + public static List parseCopyFields(Object propNode) { + List copyFields = new ArrayList<>(); if (isArray(propNode)) { for (Object node : (List) propNode) { - copyToBuilder.add(nodeStringValue(node, null)); + copyFields.add(nodeStringValue(node, null)); } } else { - copyToBuilder.add(nodeStringValue(propNode, null)); + copyFields.add(nodeStringValue(propNode, null)); } - builder.copyTo(copyToBuilder.build()); + return copyFields; } public static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index dbd70e82bead9..b43ccb0c51ed8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -75,15 +75,6 @@ private VersionFieldType() { super(NAME, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected VersionFieldType(VersionFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new VersionFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java index 54a418fe673c7..dd08f8ff763ad 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/RetentionLeaseSyncAction.java @@ -25,7 +25,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.WriteResponse; @@ -80,7 +80,7 @@ public RetentionLeaseSyncAction( final ThreadPool threadPool, final ShardStateAction shardStateAction, final ActionFilters actionFilters, - final WriteMemoryLimits writeMemoryLimits) { + final IndexingPressure indexingPressure) { super( settings, ACTION_NAME, @@ -92,7 +92,7 @@ public RetentionLeaseSyncAction( actionFilters, RetentionLeaseSyncAction.Request::new, RetentionLeaseSyncAction.Request::new, - ThreadPool.Names.MANAGEMENT, false, writeMemoryLimits); + ThreadPool.Names.MANAGEMENT, false, indexingPressure); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java new file mode 100644 index 0000000000000..309cf863b6324 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/stats/IndexingPressureStats.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.stats; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +public class IndexingPressureStats implements Writeable, ToXContentFragment { + + private final long totalCoordinatingAndPrimaryBytes; + private final long totalReplicaBytes; + private final long currentCoordinatingAndPrimaryBytes; + private final long currentReplicaBytes; + private final long coordinatingAndPrimaryRejections; + private final long replicaRejections; + + public IndexingPressureStats(StreamInput in) throws IOException { + totalCoordinatingAndPrimaryBytes = in.readVLong(); + totalReplicaBytes = in.readVLong(); + currentCoordinatingAndPrimaryBytes = in.readVLong(); + currentReplicaBytes = in.readVLong(); + coordinatingAndPrimaryRejections = in.readVLong(); + replicaRejections = in.readVLong(); + } + + public IndexingPressureStats(long totalCoordinatingAndPrimaryBytes, long totalReplicaBytes, long currentCoordinatingAndPrimaryBytes, + long currentReplicaBytes, long coordinatingAndPrimaryRejections, long replicaRejections) { + this.totalCoordinatingAndPrimaryBytes = totalCoordinatingAndPrimaryBytes; + this.totalReplicaBytes = totalReplicaBytes; + this.currentCoordinatingAndPrimaryBytes = currentCoordinatingAndPrimaryBytes; + this.currentReplicaBytes = currentReplicaBytes; + this.coordinatingAndPrimaryRejections = coordinatingAndPrimaryRejections; + this.replicaRejections = replicaRejections; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalCoordinatingAndPrimaryBytes); + out.writeVLong(totalReplicaBytes); + out.writeVLong(currentCoordinatingAndPrimaryBytes); + out.writeVLong(currentReplicaBytes); + out.writeVLong(coordinatingAndPrimaryRejections); + out.writeVLong(replicaRejections); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("indexing_pressure"); + builder.startObject("total"); + builder.field("coordinating_and_primary_bytes", totalCoordinatingAndPrimaryBytes); + builder.field("replica_bytes", totalReplicaBytes); + builder.field("all_bytes", totalReplicaBytes + totalCoordinatingAndPrimaryBytes); + builder.field("coordinating_and_primary_memory_limit_rejections", coordinatingAndPrimaryRejections); + builder.field("replica_memory_limit_rejections", replicaRejections); + builder.endObject(); + builder.startObject("current"); + builder.field("coordinating_and_primary_bytes", currentCoordinatingAndPrimaryBytes); + builder.field("replica_bytes", currentReplicaBytes); + builder.field("all_bytes", currentCoordinatingAndPrimaryBytes + currentReplicaBytes); + builder.endObject(); + return builder.endObject(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/translog/Translog.java b/server/src/main/java/org/elasticsearch/index/translog/Translog.java index 72982962af40b..99f6435235022 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -1161,7 +1161,10 @@ public Type opType() { @Override public long estimateSize() { - return (id.length() * 2) + source.length() + 12; + return (2 * id.length()) + + source.length() + + (routing != null ? 2 * routing.length() : 0) + + (4 * Long.BYTES); // timestamp, seq_no, primary_term, and version } public String id() { @@ -1328,7 +1331,8 @@ public Type opType() { @Override public long estimateSize() { - return 4 + (id.length() * 2) + 24; + return (2 * id.length()) + + (3 * Long.BYTES); // seq_no, primary_term, and version; } public String id() { @@ -1384,14 +1388,16 @@ public boolean equals(Object o) { Delete delete = (Delete) o; - return version == delete.version && + return id.equals(delete.id) && seqNo == delete.seqNo && - primaryTerm == delete.primaryTerm; + primaryTerm == delete.primaryTerm && + version == delete.version; } @Override public int hashCode() { - int result = Long.hashCode(seqNo); + int result = id.hashCode(); + result += 31 * Long.hashCode(seqNo); result = 31 * result + Long.hashCode(primaryTerm); result = 31 * result + Long.hashCode(version); return result; @@ -1477,7 +1483,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return 31 * 31 * 31 + 31 * 31 * Long.hashCode(seqNo) + 31 * Long.hashCode(primaryTerm) + reason().hashCode(); + return 31 * 31 * Long.hashCode(seqNo) + 31 * Long.hashCode(primaryTerm) + reason().hashCode(); } @Override diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 911c955c22a6d..a04fb809d49b0 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; @@ -156,7 +155,6 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(NestedPathFieldMapper.NAME, new NestedPathFieldMapper.TypeParser()); builtInMetadataMappers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, new SeqNoFieldMapper.TypeParser()); - builtInMetadataMappers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); //_field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); return Collections.unmodifiableMap(builtInMetadataMappers); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 375c3e6f1da9d..827e1428bb2ab 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -1062,13 +1062,13 @@ public ShardDeletionCheckResult canDeleteShardContent(ShardId shardId, IndexSett if (isAllocated) { return ShardDeletionCheckResult.STILL_ALLOCATED; // we are allocated - can't delete the shard } else if (indexSettings.hasCustomDataPath()) { - // lets see if it's on a custom path (return false if the shared doesn't exist) + // lets see if it's on a custom path (return false if the shard doesn't exist) // we don't need to delete anything that is not there return Files.exists(nodeEnv.resolveCustomLocation(indexSettings.customDataPath(), shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : ShardDeletionCheckResult.NO_FOLDER_FOUND; } else { - // lets see if it's path is available (return false if the shared doesn't exist) + // lets see if it's path is available (return false if the shard doesn't exist) // we don't need to delete anything that is not there return FileSystemUtils.exists(nodeEnv.availableShardPaths(shardId)) ? ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE : diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index e0bd86463705b..b8508fca5daaa 100644 --- a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -22,6 +22,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; @@ -30,8 +31,14 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ReleasableLock; +import org.elasticsearch.monitor.jvm.GcNames; +import org.elasticsearch.monitor.jvm.JvmInfo; +import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.util.ArrayList; @@ -40,6 +47,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import static org.elasticsearch.indices.breaker.BreakerSettings.CIRCUIT_BREAKER_LIMIT_SETTING; @@ -104,7 +114,14 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService { // Tripped count for when redistribution was attempted but wasn't successful private final AtomicLong parentTripCount = new AtomicLong(0); + private final OverLimitStrategy overLimitStrategy; + public HierarchyCircuitBreakerService(Settings settings, List customBreakers, ClusterSettings clusterSettings) { + this(settings, customBreakers, clusterSettings, HierarchyCircuitBreakerService::createOverLimitStrategy); + } + + HierarchyCircuitBreakerService(Settings settings, List customBreakers, ClusterSettings clusterSettings, + Function overLimitStrategyFactory) { super(); HashMap childCircuitBreakers = new HashMap<>(); childCircuitBreakers.put(CircuitBreaker.FIELDDATA, validateAndCreateBreaker( @@ -168,6 +185,8 @@ public HierarchyCircuitBreakerService(Settings settings, List c CIRCUIT_BREAKER_OVERHEAD_SETTING, (name, updatedValues) -> updateCircuitBreakerSettings(name, updatedValues.v1(), updatedValues.v2()), (s, t) -> {}); + + this.overLimitStrategy = overLimitStrategyFactory.apply(this.trackRealMemoryUsage); } private void updateCircuitBreakerSettings(String name, ByteSizeValue newLimit, Double newOverhead) { @@ -231,7 +250,7 @@ public CircuitBreakerStats stats(String name) { breaker.getTrippedCount()); } - private static class MemoryUsage { + static class MemoryUsage { final long baseUsage; final long totalUsage; final long transientChildUsage; @@ -268,6 +287,10 @@ private MemoryUsage memoryUsed(long newBytesReserved) { //package private to allow overriding it in tests long currentMemoryUsage() { + return realMemoryUsage(); + } + + static long realMemoryUsage() { try { return MEMORY_MX_BEAN.getHeapMemoryUsage().getUsed(); } catch (IllegalArgumentException ex) { @@ -290,7 +313,7 @@ public long getParentLimit() { public void checkParentLimit(long newBytesReserved, String label) throws CircuitBreakingException { final MemoryUsage memoryUsed = memoryUsed(newBytesReserved); long parentLimit = this.parentSettings.getLimit(); - if (memoryUsed.totalUsage > parentLimit) { + if (memoryUsed.totalUsage > parentLimit && overLimitStrategy.overLimit(memoryUsed).totalUsage > parentLimit) { this.parentTripCount.incrementAndGet(); final StringBuilder message = new StringBuilder("[parent] Data too large, data for [" + label + "]" + " would be [" + memoryUsed.totalUsage + "/" + new ByteSizeValue(memoryUsed.totalUsage) + "]" + @@ -334,4 +357,163 @@ private CircuitBreaker validateAndCreateBreaker(BreakerSettings breakerSettings) this, breakerSettings.getName()); } + + static OverLimitStrategy createOverLimitStrategy(boolean trackRealMemoryUsage) { + JvmInfo jvmInfo = JvmInfo.jvmInfo(); + if (trackRealMemoryUsage && jvmInfo.useG1GC().equals("true") + // messing with GC is "dangerous" so we apply an escape hatch. Not intended to be used. + && Booleans.parseBoolean(System.getProperty("es.real_memory_circuit_breaker.g1_over_limit_strategy.enabled"), true)) { + TimeValue lockTimeout = TimeValue.timeValueMillis( + Integer.parseInt(System.getProperty("es.real_memory_circuit_breaker.g1_over_limit_strategy.lock_timeout_ms", "500")) + ); + // hardcode interval, do not want any tuning of it outside code changes. + return new G1OverLimitStrategy(jvmInfo, HierarchyCircuitBreakerService::realMemoryUsage, createYoungGcCountSupplier(), + System::currentTimeMillis, 5000, lockTimeout); + } else { + return memoryUsed -> memoryUsed; + } + } + + static LongSupplier createYoungGcCountSupplier() { + List youngBeans = + ManagementFactory.getGarbageCollectorMXBeans().stream() + .filter(mxBean -> GcNames.getByGcName(mxBean.getName(), mxBean.getName()).equals(GcNames.YOUNG)) + .collect(Collectors.toList()); + assert youngBeans.size() == 1; + assert youngBeans.get(0).getCollectionCount() != -1 : "G1 must support getting collection count"; + + if (youngBeans.size() == 1) { + return youngBeans.get(0)::getCollectionCount; + } else { + logger.warn("Unable to find young generation collector, G1 over limit strategy might be impacted [{}]", youngBeans); + return () -> -1; + } + } + + interface OverLimitStrategy { + MemoryUsage overLimit(MemoryUsage memoryUsed); + } + + static class G1OverLimitStrategy implements OverLimitStrategy { + private final long g1RegionSize; + private final LongSupplier currentMemoryUsageSupplier; + private final LongSupplier gcCountSupplier; + private final LongSupplier timeSupplier; + private final TimeValue lockTimeout; + private final long maxHeap; + + private long lastCheckTime = Long.MIN_VALUE; + private final long minimumInterval; + + private long blackHole; + private final ReleasableLock lock = new ReleasableLock(new ReentrantLock()); + + G1OverLimitStrategy(JvmInfo jvmInfo, LongSupplier currentMemoryUsageSupplier, + LongSupplier gcCountSupplier, + LongSupplier timeSupplier, long minimumInterval, TimeValue lockTimeout) { + this.lockTimeout = lockTimeout; + assert minimumInterval > 0; + this.currentMemoryUsageSupplier = currentMemoryUsageSupplier; + this.gcCountSupplier = gcCountSupplier; + this.timeSupplier = timeSupplier; + this.minimumInterval = minimumInterval; + this.maxHeap = jvmInfo.getMem().getHeapMax().getBytes(); + long g1RegionSize = jvmInfo.getG1RegionSize(); + if (g1RegionSize <= 0) { + this.g1RegionSize = fallbackRegionSize(jvmInfo); + } else { + this.g1RegionSize = g1RegionSize; + } + } + + static long fallbackRegionSize(JvmInfo jvmInfo) { + // mimick JDK calculation based on JDK 14 source: + // https://hg.openjdk.java.net/jdk/jdk14/file/6c954123ee8d/src/hotspot/share/gc/g1/heapRegion.cpp#l65 + // notice that newer JDKs will have a slight variant only considering max-heap: + // https://hg.openjdk.java.net/jdk/jdk/file/e7d0ec2d06e8/src/hotspot/share/gc/g1/heapRegion.cpp#l67 + // based on this JDK "bug": + // https://bugs.openjdk.java.net/browse/JDK-8241670 + long averageHeapSize = + (jvmInfo.getMem().getHeapMax().getBytes() + JvmInfo.jvmInfo().getMem().getHeapMax().getBytes()) / 2; + long regionSize = Long.highestOneBit(averageHeapSize / 2048); + if (regionSize < ByteSizeUnit.MB.toBytes(1)) { + regionSize = ByteSizeUnit.MB.toBytes(1); + } else if (regionSize > ByteSizeUnit.MB.toBytes(32)) { + regionSize = ByteSizeUnit.MB.toBytes(32); + } + return regionSize; + } + + @Override + public MemoryUsage overLimit(MemoryUsage memoryUsed) { + boolean leader = false; + int allocationIndex = 0; + long allocationDuration = 0; + try (ReleasableLock locked = lock.tryAcquire(lockTimeout)) { + if (locked != null) { + long begin = timeSupplier.getAsLong(); + leader = begin >= lastCheckTime + minimumInterval; + overLimitTriggered(leader); + if (leader) { + long initialCollectionCount = gcCountSupplier.getAsLong(); + logger.info("attempting to trigger G1GC due to high heap usage [{}]", memoryUsed.baseUsage); + long localBlackHole = 0; + // number of allocations, corresponding to (approximately) number of free regions + 1 + int allocationCount = Math.toIntExact((maxHeap - memoryUsed.baseUsage) / g1RegionSize + 1); + // allocations of half-region size becomes single humongous alloc, thus taking up a full region. + int allocationSize = (int) (g1RegionSize >> 1); + long maxUsageObserved = memoryUsed.baseUsage; + for (; allocationIndex < allocationCount; ++allocationIndex) { + long current = currentMemoryUsageSupplier.getAsLong(); + if (current >= maxUsageObserved) { + maxUsageObserved = current; + } else { + // we observed a memory drop, so some GC must have occurred + break; + } + if (initialCollectionCount != gcCountSupplier.getAsLong()) { + break; + } + localBlackHole += new byte[allocationSize].hashCode(); + } + + blackHole += localBlackHole; + logger.trace("black hole [{}]", blackHole); + + long now = timeSupplier.getAsLong(); + this.lastCheckTime = now; + allocationDuration = now - begin; + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // fallthrough + } + + final long current = currentMemoryUsageSupplier.getAsLong(); + if (current < memoryUsed.baseUsage) { + if (leader) { + logger.info("GC did bring memory usage down, before [{}], after [{}], allocations [{}], duration [{}]", + memoryUsed.baseUsage, current, allocationIndex, allocationDuration); + } + return new MemoryUsage(current, memoryUsed.totalUsage - memoryUsed.baseUsage + current, + memoryUsed.transientChildUsage, memoryUsed.permanentChildUsage); + } else { + if (leader) { + logger.info("GC did not bring memory usage down, before [{}], after [{}], allocations [{}], duration [{}]", + memoryUsed.baseUsage, current, allocationIndex, allocationDuration); + } + // prefer original measurement when reporting if heap usage was not brought down. + return memoryUsed; + } + } + + void overLimitTriggered(boolean leader) { + // for tests to override. + } + + TimeValue getLockTimeout() { + return lockTimeout; + } + } } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index e546d05d3f670..828edcf47ef0c 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -40,6 +40,7 @@ import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.StopWatch; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -85,6 +86,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; @@ -223,7 +225,7 @@ && isTargetSameHistory() } else { final Engine.IndexCommitRef safeCommitRef; try { - safeCommitRef = shard.acquireSafeIndexCommit(); + safeCommitRef = acquireSafeCommit(shard); resources.add(safeCommitRef); } catch (final Exception e) { throw new RecoveryEngineException(shard.shardId(), 1, "snapshot failed", e); @@ -395,17 +397,34 @@ public void onFailure(Exception e) { */ private Releasable acquireStore(Store store) { store.incRef(); - return Releasables.releaseOnce(() -> { - final PlainActionFuture future = new PlainActionFuture<>(); - assert threadPool.generic().isShutdown() == false; - // TODO: We shouldn't use the generic thread pool here as we already execute this from the generic pool. - // While practically unlikely at a min pool size of 128 we could technically block the whole pool by waiting on futures - // below and thus make it impossible for the store release to execute which in turn would block the futures forever - threadPool.generic().execute(ActionRunnable.run(future, store::decRef)); - FutureUtils.get(future); + return Releasables.releaseOnce(() -> runWithGenericThreadPool(store::decRef)); + } + + /** + * Releasing a safe commit can access some commit files. It's better not to use {@link CancellableThreads} to interact + * with the file systems due to interrupt (see {@link org.apache.lucene.store.NIOFSDirectory} javadocs for more detail). + * This method acquires a safe commit and wraps it to make sure that it will be released using the generic thread pool. + */ + private Engine.IndexCommitRef acquireSafeCommit(IndexShard shard) { + final Engine.IndexCommitRef commitRef = shard.acquireSafeIndexCommit(); + final AtomicBoolean closed = new AtomicBoolean(false); + return new Engine.IndexCommitRef(commitRef.getIndexCommit(), () -> { + if (closed.compareAndSet(false, true)) { + runWithGenericThreadPool(commitRef::close); + } }); } + private void runWithGenericThreadPool(CheckedRunnable task) { + final PlainActionFuture future = new PlainActionFuture<>(); + assert threadPool.generic().isShutdown() == false; + // TODO: We shouldn't use the generic thread pool here as we already execute this from the generic pool. + // While practically unlikely at a min pool size of 128 we could technically block the whole pool by waiting on futures + // below and thus make it impossible for the store release to execute which in turn would block the futures forever + threadPool.generic().execute(ActionRunnable.run(future, task)); + FutureUtils.get(future); + } + static final class SendFileResult { final List phase1FileNames; final List phase1FileSizes; diff --git a/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java index ecfb71cd44137..1aae961f48974 100644 --- a/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/elasticsearch/monitor/fs/FsHealthService.java @@ -129,7 +129,7 @@ public StatusInfo getHealth() { class FsHealthMonitor implements Runnable { - private static final String TEMP_FILE_NAME = ".es_temp_file"; + static final String TEMP_FILE_NAME = ".es_temp_file"; private byte[] byteToWrite; FsHealthMonitor(){ diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 4d2584202e540..b1d4bce73bec7 100644 --- a/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/server/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -98,6 +98,7 @@ public class JvmInfo implements ReportingService.Info { String onOutOfMemoryError = null; String useCompressedOops = "unknown"; String useG1GC = "unknown"; + long g1RegisionSize = -1; String useSerialGC = "unknown"; long configuredInitialHeapSize = -1; long configuredMaxHeapSize = -1; @@ -130,6 +131,8 @@ public class JvmInfo implements ReportingService.Info { try { Object useG1GCVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC"); useG1GC = (String) valueMethod.invoke(useG1GCVmOptionObject); + Object regionSizeVmOptionObject = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "G1HeapRegionSize"); + g1RegisionSize = Long.parseLong((String) valueMethod.invoke(regionSizeVmOptionObject)); } catch (Exception ignored) { } @@ -180,9 +183,11 @@ public class JvmInfo implements ReportingService.Info { onOutOfMemoryError, useCompressedOops, useG1GC, - useSerialGC); + useSerialGC, + g1RegisionSize); } + @SuppressForbidden(reason = "PathUtils#get") private static boolean usingBundledJdk() { /* @@ -229,12 +234,13 @@ public static JvmInfo jvmInfo() { private final String useCompressedOops; private final String useG1GC; private final String useSerialGC; + private final long g1RegionSize; private JvmInfo(long pid, String version, String vmName, String vmVersion, String vmVendor, boolean bundledJdk, Boolean usingBundledJdk, long startTime, long configuredInitialHeapSize, long configuredMaxHeapSize, Mem mem, String[] inputArguments, String bootClassPath, String classPath, Map systemProperties, String[] gcCollectors, String[] memoryPools, String onError, String onOutOfMemoryError, String useCompressedOops, String useG1GC, - String useSerialGC) { + String useSerialGC, long g1RegionSize) { this.pid = pid; this.version = version; this.vmName = vmName; @@ -257,6 +263,7 @@ private JvmInfo(long pid, String version, String vmName, String vmVersion, Strin this.useCompressedOops = useCompressedOops; this.useG1GC = useG1GC; this.useSerialGC = useSerialGC; + this.g1RegionSize = g1RegionSize; } public JvmInfo(StreamInput in) throws IOException { @@ -291,6 +298,7 @@ public JvmInfo(StreamInput in) throws IOException { this.onOutOfMemoryError = null; this.useG1GC = "unknown"; this.useSerialGC = "unknown"; + this.g1RegionSize = -1; } @Override @@ -486,6 +494,10 @@ public String useSerialGC() { return this.useSerialGC; } + public long getG1RegionSize() { + return g1RegionSize; + } + public String[] getGcCollectors() { return gcCollectors; } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index cecc94d2f3b97..68258a55047a6 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -30,7 +30,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionModule; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.search.SearchExecutionStatsCollector; import org.elasticsearch.action.search.SearchPhaseController; import org.elasticsearch.action.search.SearchTransportService; @@ -538,6 +538,7 @@ protected Node(final Environment initialEnvironment, final SearchTransportService searchTransportService = new SearchTransportService(transportService, SearchExecutionStatsCollector.makeWrapper(responseCollectorService)); final HttpServerTransport httpServerTransport = newHttpTransport(networkModule); + final IndexingPressure indexingLimits = new IndexingPressure(settings); final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); RepositoriesModule repositoriesModule = new RepositoriesModule(this.environment, @@ -566,7 +567,7 @@ protected Node(final Environment initialEnvironment, this.nodeService = new NodeService(settings, threadPool, monitorService, discoveryModule.getDiscovery(), transportService, indicesService, pluginsService, circuitBreakerService, scriptService, httpServerTransport, ingestService, clusterService, settingsModule.getSettingsFilter(), responseCollectorService, - searchTransportService); + searchTransportService, indexingLimits); final SearchService searchService = newSearchService(clusterService, indicesService, threadPool, scriptService, bigArrays, searchModule.getFetchPhase(), @@ -584,7 +585,6 @@ protected Node(final Environment initialEnvironment, new PersistentTasksClusterService(settings, registry, clusterService, threadPool); resourcesToClose.add(persistentTasksClusterService); final PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); - final WriteMemoryLimits bulkIndexingLimits = new WriteMemoryLimits(); modules.add(b -> { b.bind(Node.class).toInstance(this); @@ -603,7 +603,7 @@ protected Node(final Environment initialEnvironment, b.bind(ScriptService.class).toInstance(scriptService); b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry()); b.bind(IngestService.class).toInstance(ingestService); - b.bind(WriteMemoryLimits.class).toInstance(bulkIndexingLimits); + b.bind(IndexingPressure.class).toInstance(indexingLimits); b.bind(UsageService.class).toInstance(usageService); b.bind(AggregationUsageService.class).toInstance(searchModule.getValuesSourceRegistry().getUsageService()); b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry); diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index 98e1f18dff289..3394a796f9836 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -19,6 +19,7 @@ package org.elasticsearch.node; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Build; import org.elasticsearch.Version; @@ -59,6 +60,7 @@ public class NodeService implements Closeable { private final HttpServerTransport httpServerTransport; private final ResponseCollectorService responseCollectorService; private final SearchTransportService searchTransportService; + private final IndexingPressure indexingPressure; private final Discovery discovery; @@ -67,7 +69,7 @@ public class NodeService implements Closeable { CircuitBreakerService circuitBreakerService, ScriptService scriptService, @Nullable HttpServerTransport httpServerTransport, IngestService ingestService, ClusterService clusterService, SettingsFilter settingsFilter, ResponseCollectorService responseCollectorService, - SearchTransportService searchTransportService) { + SearchTransportService searchTransportService, IndexingPressure indexingPressure) { this.settings = settings; this.threadPool = threadPool; this.monitorService = monitorService; @@ -82,6 +84,7 @@ public class NodeService implements Closeable { this.scriptService = scriptService; this.responseCollectorService = responseCollectorService; this.searchTransportService = searchTransportService; + this.indexingPressure = indexingPressure; clusterService.addStateApplier(ingestService); } @@ -103,7 +106,8 @@ public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool, boolean fs, boolean transport, boolean http, boolean circuitBreaker, - boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection, boolean scriptCache) { + boolean script, boolean discoveryStats, boolean ingest, boolean adaptiveSelection, boolean scriptCache, + boolean indexingPressure) { // for indices stats we want to include previous allocated shards stats as well (it will // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats) return new NodeStats(transportService.getLocalNode(), System.currentTimeMillis(), @@ -120,8 +124,7 @@ public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, bo discoveryStats ? discovery.stats() : null, ingest ? ingestService.stats() : null, adaptiveSelection ? responseCollectorService.getAdaptiveStats(searchTransportService.getPendingSearchRequests()) : null, - scriptCache ? scriptService.cacheStats() : null - ); + indexingPressure ? this.indexingPressure.stats() : null); } public IngestService getIngestService() { diff --git a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java index ce4eab29bc978..a27301724ac87 100644 --- a/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/FilterRepository.java @@ -86,7 +86,7 @@ public void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryS @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { in.deleteSnapshots(snapshotIds, repositoryStateId, repositoryMetaVersion, listener); } diff --git a/server/src/main/java/org/elasticsearch/repositories/Repository.java b/server/src/main/java/org/elasticsearch/repositories/Repository.java index b15fa5fa77763..ec0fb5b561ad1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/Repository.java +++ b/server/src/main/java/org/elasticsearch/repositories/Repository.java @@ -140,7 +140,7 @@ void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryStateId, * @param listener completion listener */ void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener); + ActionListener listener); /** * Returns snapshot throttle time in nanoseconds */ diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java index d660088b374fc..6f13ae232422b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryData.java @@ -410,14 +410,18 @@ public List resolveIndices(final List indices) { /** * Resolve the given index names to index ids, creating new index ids for * new indices in the repository. + * + * @param indicesToResolve names of indices to resolve + * @param inFlightIds name to index mapping for currently in-flight snapshots not yet in the repository data to fall back to */ - public List resolveNewIndices(final List indicesToResolve) { + public List resolveNewIndices(List indicesToResolve, Map inFlightIds) { List snapshotIndices = new ArrayList<>(); for (String index : indicesToResolve) { - final IndexId indexId; - if (indices.containsKey(index)) { - indexId = indices.get(index); - } else { + IndexId indexId = indices.get(index); + if (indexId == null) { + indexId = inFlightIds.get(index); + } + if (indexId == null) { indexId = new IndexId(index, UUIDs.randomBase64UUID()); } snapshotIndices.add(indexId); diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 7665068f27d78..c2e7588e575b1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -427,7 +427,6 @@ public void updateState(ClusterState state) { private long bestGeneration(Collection operations) { final String repoName = metadata.name(); - assert operations.size() <= 1 : "Assumed one or no operations but received " + operations; return operations.stream().filter(e -> e.repository().equals(repoName)).mapToLong(RepositoryOperation::repositoryStateId) .max().orElse(RepositoryData.EMPTY_REPO_GEN); } @@ -544,7 +543,7 @@ public RepositoryStats stats() { @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { if (isReadOnly()) { listener.onFailure(new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository")); } else { @@ -618,7 +617,7 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map snapshotIds, long repositoryStateId, Map foundIndices, Map rootBlobs, RepositoryData repositoryData, Version repoMetaVersion, - ActionListener listener) { + ActionListener listener) { if (SnapshotsService.useShardGenerations(repoMetaVersion)) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets @@ -639,13 +638,13 @@ private void doDeleteShardSnapshots(Collection snapshotIds, long rep } final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, builder.build()); writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), - ActionListener.wrap(v -> writeUpdatedRepoDataStep.onResponse(updatedRepoData), listener::onFailure)); + ActionListener.wrap(writeUpdatedRepoDataStep::onResponse, listener::onFailure)); }, listener::onFailure); // Once we have updated the repository, run the clean-ups writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion final ActionListener afterCleanupsListener = - new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); + new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(updatedRepoData)), 2); asyncCleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); asyncCleanupUnlinkedShardLevelBlobs(repositoryData, snapshotIds, writeShardMetaDataAndComputeDeletesStep.result(), afterCleanupsListener); @@ -653,11 +652,11 @@ private void doDeleteShardSnapshots(Collection snapshotIds, long rep } else { // Write the new repository data first (with the removed snapshot), using no shard generations final RepositoryData updatedRepoData = repositoryData.removeSnapshots(snapshotIds, ShardGenerations.EMPTY); - writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), ActionListener.wrap(v -> { + writeIndexGen(updatedRepoData, repositoryStateId, repoMetaVersion, Function.identity(), ActionListener.wrap(newRepoData -> { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion final ActionListener afterCleanupsListener = - new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); - asyncCleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); + new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(newRepoData)), 2); + asyncCleanupUnlinkedRootAndIndicesBlobs(snapshotIds, foundIndices, rootBlobs, newRepoData, afterCleanupsListener); final StepListener> writeMetaAndComputeDeletesStep = new StepListener<>(); writeUpdatedShardMetaDataAndComputeDeletes(snapshotIds, repositoryData, false, writeMetaAndComputeDeletesStep); writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> @@ -1571,10 +1570,10 @@ public ClusterState execute(ClusterState currentState) { "Tried to update from unexpected pending repo generation [" + meta.pendingGeneration() + "] after write to generation [" + newGen + "]"); } - return stateFilter.apply(ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()) - .putCustom(RepositoriesMetadata.TYPE, - currentState.metadata().custom(RepositoriesMetadata.TYPE).withUpdatedGeneration( - metadata.name(), newGen, newGen))).build()); + return updateRepositoryGenerationsIfNecessary(stateFilter.apply(ClusterState.builder(currentState) + .metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, + currentState.metadata().custom(RepositoriesMetadata.TYPE) + .withUpdatedGeneration(metadata.name(), newGen, newGen))).build()), expectedGen, newGen); } @Override @@ -1639,6 +1638,45 @@ public void onFailure(Exception e) { return true; } + /** + * Updates the repository generation that running deletes and snapshot finalizations will be based on for this repository if any such + * operations are found in the cluster state while setting the safe repository generation. + * + * @param state cluster state to update + * @param oldGen previous safe repository generation + * @param newGen new safe repository generation + * @return updated cluster state + */ + private ClusterState updateRepositoryGenerationsIfNecessary(ClusterState state, long oldGen, long newGen) { + final String repoName = metadata.name(); + final SnapshotsInProgress updatedSnapshotsInProgress; + boolean changedSnapshots = false; + final List snapshotEntries = new ArrayList<>(); + for (SnapshotsInProgress.Entry entry : state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries()) { + if (entry.repository().equals(repoName) && entry.repositoryStateId() == oldGen) { + snapshotEntries.add(entry.withRepoGen(newGen)); + changedSnapshots = true; + } else { + snapshotEntries.add(entry); + } + } + updatedSnapshotsInProgress = changedSnapshots ? SnapshotsInProgress.of(snapshotEntries) : null; + final SnapshotDeletionsInProgress updatedDeletionsInProgress; + boolean changedDeletions = false; + final List deletionEntries = new ArrayList<>(); + for (SnapshotDeletionsInProgress.Entry entry : + state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY).getEntries()) { + if (entry.repository().equals(repoName) && entry.repositoryStateId() == oldGen) { + deletionEntries.add(entry.withRepoGen(newGen)); + changedDeletions = true; + } else { + deletionEntries.add(entry); + } + } + updatedDeletionsInProgress = changedDeletions ? SnapshotDeletionsInProgress.of(deletionEntries) : null; + return SnapshotsService.updateWithSnapshots(state, updatedSnapshotsInProgress, updatedDeletionsInProgress); + } + private RepositoryMetadata getRepoMetadata(ClusterState state) { final RepositoryMetadata repositoryMetadata = state.getMetadata().custom(RepositoriesMetadata.TYPE).repository(metadata.name()); @@ -1721,7 +1759,7 @@ public void snapshotShard(Store store, MapperService mapperService, SnapshotId s final ShardId shardId = store.shardId(); final long startTime = threadPool.absoluteTimeInMillis(); try { - final String generation = ShardGenerations.fixShardGeneration(snapshotStatus.generation()); + final String generation = snapshotStatus.generation(); logger.debug("[{}] [{}] snapshot to [{}] [{}] ...", shardId, snapshotId, metadata.name(), generation); final BlobContainer shardContainer = shardContainer(indexId, shardId); final Set blobs; @@ -2200,8 +2238,6 @@ public BlobStoreIndexShardSnapshot loadShardSnapshot(BlobContainer shardContaine private Tuple buildBlobStoreIndexShardSnapshots(Set blobs, BlobContainer shardContainer, @Nullable String generation) throws IOException { - assert ShardGenerations.fixShardGeneration(generation) == generation - : "Generation must not be numeric but received [" + generation + "]"; if (generation != null) { if (generation.equals(ShardGenerations.NEW_SHARD_GEN)) { return new Tuple<>(BlobStoreIndexShardSnapshots.EMPTY, ShardGenerations.NEW_SHARD_GEN); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java index 160c7770f94de..e20a5ab67357e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetDataStreamsAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -44,7 +45,8 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request(request.param("name")); + GetDataStreamAction.Request getDataStreamsRequest = new GetDataStreamAction.Request( + Strings.splitStringByCommaToArray(request.param("name"))); return channel -> client.admin().indices().getDataStreams(getDataStreamsRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 5baf9b8e9ffeb..8d306156ce82e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -55,14 +55,17 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); - Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); } + return sendPutMappingRequest(request, client, sourceAsMap); + } + + protected RestChannelConsumer sendPutMappingRequest(RestRequest request, NodeClient client, Map sourceAsMap) { + PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index"))); putMappingRequest.source(sourceAsMap); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); diff --git a/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java b/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java index ed224204f8933..1174b75fb507a 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestConditionalScript.java @@ -19,7 +19,6 @@ package org.elasticsearch.script; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.TimeValue; import java.util.Map; @@ -33,7 +32,7 @@ public abstract class IngestConditionalScript { /** The context used to compile {@link IngestConditionalScript} factories. */ public static final ScriptContext CONTEXT = new ScriptContext<>("processor_conditional", Factory.class, - 200, TimeValue.timeValueMillis(0), new Tuple<>(375, TimeValue.timeValueMinutes(5))); + 200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple()); /** The generic runtime parameters for the script. */ private final Map params; diff --git a/server/src/main/java/org/elasticsearch/script/IngestScript.java b/server/src/main/java/org/elasticsearch/script/IngestScript.java index ffd3475fc52c3..6af53e19614fa 100644 --- a/server/src/main/java/org/elasticsearch/script/IngestScript.java +++ b/server/src/main/java/org/elasticsearch/script/IngestScript.java @@ -20,7 +20,6 @@ package org.elasticsearch.script; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.TimeValue; import java.util.Map; @@ -34,7 +33,7 @@ public abstract class IngestScript { /** The context used to compile {@link IngestScript} factories. */ public static final ScriptContext CONTEXT = new ScriptContext<>("ingest", Factory.class, - 200, TimeValue.timeValueMillis(0), new Tuple<>(375, TimeValue.timeValueMinutes(5))); + 200, TimeValue.timeValueMillis(0), ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple()); /** The generic runtime parameters for the script. */ private final Map params; diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index 5d59a91b207ce..e156cfa125b19 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -42,7 +42,7 @@ public class ScriptCache { private static final Logger logger = LogManager.getLogger(ScriptService.class); - static final Tuple UNLIMITED_COMPILATION_RATE = new Tuple<>(0, TimeValue.ZERO); + static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO); private final Cache cache; private final ScriptMetrics scriptMetrics; @@ -51,14 +51,14 @@ public class ScriptCache { // Cache settings or derived from settings final int cacheSize; final TimeValue cacheExpire; - final Tuple rate; + final CompilationRate rate; private final double compilesAllowedPerNano; private final String contextRateSetting; ScriptCache( int cacheMaxSize, TimeValue cacheExpire, - Tuple maxCompilationRate, + CompilationRate maxCompilationRate, String contextRateSetting ) { this.cacheSize = cacheMaxSize; @@ -78,9 +78,9 @@ public class ScriptCache { this.cache = cacheBuilder.removalListener(new ScriptCacheRemovalListener()).build(); this.rate = maxCompilationRate; - this.compilesAllowedPerNano = ((double) rate.v1()) / rate.v2().nanos(); + this.compilesAllowedPerNano = ((double) rate.count) / rate.time.nanos(); this.scriptMetrics = new ScriptMetrics(); - this.tokenBucketState = new AtomicReference(new TokenBucketState(this.rate.v1())); + this.tokenBucketState = new AtomicReference(new TokenBucketState(this.rate.count)); } FactoryType compile( @@ -132,8 +132,8 @@ static void rethrow(Throwable t) throws T { throw (T) t; } - public ScriptStats stats() { - return scriptMetrics.stats(); + public ScriptContextStats stats(String context) { + return scriptMetrics.stats(context); } /** @@ -156,8 +156,8 @@ void checkCompilationLimit() { double scriptsPerTimeWindow = current.availableTokens + (timePassed) * compilesAllowedPerNano; // It's been over the time limit anyway, readjust the bucket to be level - if (scriptsPerTimeWindow > rate.v1()) { - scriptsPerTimeWindow = rate.v1(); + if (scriptsPerTimeWindow > rate.count) { + scriptsPerTimeWindow = rate.count; } // If there is enough tokens in the bucket, allow the request and decrease the tokens by 1 @@ -173,7 +173,7 @@ void checkCompilationLimit() { scriptMetrics.onCompilationLimit(); // Otherwise reject the request throw new CircuitBreakingException("[script] Too many dynamic script compilations within, max: [" + - rate.v1() + "/" + rate.v2() +"]; please use indexed, or scripts with parameters instead; " + + rate + "]; please use indexed, or scripts with parameters instead; " + "this limit can be changed by the [" + contextRateSetting + "] setting", CircuitBreaker.Durability.TRANSIENT); } @@ -243,4 +243,76 @@ private TokenBucketState(long lastInlineCompileTime, double availableTokens, boo this.tokenSuccessfullyTaken = tokenSuccessfullyTaken; } } + + public static class CompilationRate { + public final int count; + public final TimeValue time; + private final String source; + + public CompilationRate(Integer count, TimeValue time) { + this.count = count; + this.time = time; + this.source = null; + } + + public CompilationRate(Tuple rate) { + this(rate.v1(), rate.v2()); + } + + /** + * Parses a string as a non-negative int count and a {@code TimeValue} as arguments split by a slash + */ + public CompilationRate(String value) { + if (value.contains("/") == false || value.startsWith("/") || value.endsWith("/")) { + throw new IllegalArgumentException("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [" + + value + "]"); + } + int idx = value.indexOf("/"); + String count = value.substring(0, idx); + String time = value.substring(idx + 1); + try { + int rate = Integer.parseInt(count); + if (rate < 0) { + throw new IllegalArgumentException("rate [" + rate + "] must be positive"); + } + TimeValue timeValue = TimeValue.parseTimeValue(time, "script.max_compilations_rate"); + if (timeValue.nanos() <= 0) { + throw new IllegalArgumentException("time value [" + time + "] must be positive"); + } + // protect against a too hard to check limit, like less than a minute + if (timeValue.seconds() < 60) { + throw new IllegalArgumentException("time value [" + time + "] must be at least on a one minute resolution"); + } + this.count = rate; + this.time = timeValue; + this.source = value; + } catch (NumberFormatException e) { + // the number format exception message is so confusing, that it makes more sense to wrap it with a useful one + throw new IllegalArgumentException("could not parse [" + count + "] as integer in value [" + value + "]", e); + } + } + + public Tuple asTuple() { + return new Tuple<>(this.count, this.time); + } + + @Override + public String toString() { + return source != null ? source : count + "/" + time.toHumanReadableString(0); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CompilationRate that = (CompilationRate) o; + return count == that.count && + Objects.equals(time, that.time); + } + + @Override + public int hashCode() { + return Objects.hash(count, time); + } + } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java deleted file mode 100644 index a5d6fd1cf46b8..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.script; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContentFragment; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -public class ScriptCacheStats implements Writeable, ToXContentFragment { - private final Map context; - private final ScriptStats general; - - public ScriptCacheStats(Map context) { - this.context = Collections.unmodifiableMap(context); - this.general = null; - } - - public ScriptCacheStats(ScriptStats general) { - this.general = Objects.requireNonNull(general); - this.context = null; - } - - public ScriptCacheStats(StreamInput in) throws IOException { - boolean isContext = in.readBoolean(); - if (isContext == false) { - general = new ScriptStats(in); - context = null; - return; - } - - general = null; - int size = in.readInt(); - Map context = new HashMap<>(size); - for (int i=0; i < size; i++) { - String name = in.readString(); - context.put(name, new ScriptStats(in)); - } - this.context = Collections.unmodifiableMap(context); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - if (general != null) { - out.writeBoolean(false); - general.writeTo(out); - return; - } - - out.writeBoolean(true); - out.writeInt(context.size()); - for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) { - out.writeString(name); - context.get(name).writeTo(out); - } - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.SCRIPT_CACHE_STATS); - builder.startObject(Fields.SUM); - if (general != null) { - builder.field(ScriptStats.Fields.COMPILATIONS, general.getCompilations()); - builder.field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions()); - builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered()); - builder.endObject().endObject(); - return builder; - } - - ScriptStats sum = sum(); - builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations()); - builder.field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions()); - builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered()); - builder.endObject(); - - builder.startArray(Fields.CONTEXTS); - for (String name: context.keySet().stream().sorted().collect(Collectors.toList())) { - ScriptStats stats = context.get(name); - builder.startObject(); - builder.field(Fields.CONTEXT, name); - builder.field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations()); - builder.field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions()); - builder.field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered()); - builder.endObject(); - } - builder.endArray(); - builder.endObject(); - - return builder; - } - - /** - * Get the context specific stats, null if using general cache - */ - public Map getContextStats() { - return context; - } - - /** - * Get the general stats, null if using context cache - */ - public ScriptStats getGeneralStats() { - return general; - } - - /** - * The sum of all script stats, either the general stats or the sum of all stats of the context stats. - */ - public ScriptStats sum() { - if (general != null) { - return general; - } - return ScriptStats.sum(context.values()); - } - - static final class Fields { - static final String SCRIPT_CACHE_STATS = "script_cache"; - static final String CONTEXT = "context"; - static final String SUM = "sum"; - static final String CONTEXTS = "contexts"; - } -} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java new file mode 100644 index 0000000000000..1a1719ad440d5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/ScriptContextStats.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class ScriptContextStats implements Writeable, ToXContentFragment, Comparable { + private final String context; + private final long compilations; + private final long cacheEvictions; + private final long compilationLimitTriggered; + + public ScriptContextStats(String context, long compilations, long cacheEvictions, long compilationLimitTriggered) { + this.context = Objects.requireNonNull(context); + this.compilations = compilations; + this.cacheEvictions = cacheEvictions; + this.compilationLimitTriggered = compilationLimitTriggered; + } + + public ScriptContextStats(StreamInput in) throws IOException { + context = in.readString(); + compilations = in.readVLong(); + cacheEvictions = in.readVLong(); + compilationLimitTriggered = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(context); + out.writeVLong(compilations); + out.writeVLong(cacheEvictions); + out.writeVLong(compilationLimitTriggered); + } + + public String getContext() { + return context; + } + + public long getCompilations() { + return compilations; + } + + public long getCacheEvictions() { + return cacheEvictions; + } + + public long getCompilationLimitTriggered() { + return compilationLimitTriggered; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Fields.CONTEXT, getContext()); + builder.field(Fields.COMPILATIONS, getCompilations()); + builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions()); + builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered()); + builder.endObject(); + return builder; + } + + @Override + public int compareTo(ScriptContextStats o) { + return this.context.compareTo(o.context); + } + + static final class Fields { + static final String CONTEXT = "context"; + static final String COMPILATIONS = "compilations"; + static final String CACHE_EVICTIONS = "cache_evictions"; + static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered"; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java index 18c1027074b16..093beeb6ac126 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptMetrics.java @@ -26,10 +26,6 @@ public class ScriptMetrics { final CounterMetric cacheEvictionsMetric = new CounterMetric(); final CounterMetric compilationLimitTriggered = new CounterMetric(); - public ScriptStats stats() { - return new ScriptStats(compilationsMetric.count(), cacheEvictionsMetric.count(), compilationLimitTriggered.count()); - } - public void onCompilation() { compilationsMetric.inc(); } @@ -41,4 +37,12 @@ public void onCacheEviction() { public void onCompilationLimit() { compilationLimitTriggered.inc(); } -} + + public ScriptContextStats stats(String context) { + return new ScriptContextStats( + context, + compilationsMetric.count(), + cacheEvictionsMetric.count(), + compilationLimitTriggered.count() + ); + }} diff --git a/server/src/main/java/org/elasticsearch/script/ScriptService.java b/server/src/main/java/org/elasticsearch/script/ScriptService.java index 23913afbb1799..254512e5b73f2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptService.java @@ -34,7 +34,6 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -64,53 +63,8 @@ public class ScriptService implements Closeable, ClusterStateApplier { static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic"; - // Special setting value for SCRIPT_GENERAL_MAX_COMPILATIONS_RATE to indicate the script service should use context - // specific caches - static final Tuple USE_CONTEXT_RATE_VALUE = new Tuple<>(-1, TimeValue.MINUS_ONE); - static final String USE_CONTEXT_RATE_KEY = "use-context"; - - // a parsing function that requires a non negative int and a timevalue as arguments split by a slash - // this allows you to easily define rates - static final Function> MAX_COMPILATION_RATE_FUNCTION = - (String value) -> { - if (value.contains("/") == false || value.startsWith("/") || value.endsWith("/")) { - throw new IllegalArgumentException("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [" + - value + "]"); - } - int idx = value.indexOf("/"); - String count = value.substring(0, idx); - String time = value.substring(idx + 1); - try { - - int rate = Integer.parseInt(count); - if (rate < 0) { - throw new IllegalArgumentException("rate [" + rate + "] must be positive"); - } - TimeValue timeValue = TimeValue.parseTimeValue(time, "script.max_compilations_rate"); - if (timeValue.nanos() <= 0) { - throw new IllegalArgumentException("time value [" + time + "] must be positive"); - } - // protect against a too hard to check limit, like less than a minute - if (timeValue.seconds() < 60) { - throw new IllegalArgumentException("time value [" + time + "] must be at least on a one minute resolution"); - } - return Tuple.tuple(rate, timeValue); - } catch (NumberFormatException e) { - // the number format exception message is so confusing, that it makes more sense to wrap it with a useful one - throw new IllegalArgumentException("could not parse [" + count + "] as integer in value [" + value + "]", e); - } - }; - - public static final Setting SCRIPT_GENERAL_CACHE_SIZE_SETTING = - Setting.intSetting("script.cache.max_size", 100, 0, Property.NodeScope, Property.Deprecated); - public static final Setting SCRIPT_GENERAL_CACHE_EXPIRE_SETTING = - Setting.positiveTimeSetting("script.cache.expire", TimeValue.timeValueMillis(0), Property.NodeScope, Property.Deprecated); public static final Setting SCRIPT_MAX_SIZE_IN_BYTES = Setting.intSetting("script.max_size_in_bytes", 65535, 0, Property.Dynamic, Property.NodeScope); - public static final Setting> SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING = - new Setting<>("script.max_compilations_rate", USE_CONTEXT_RATE_KEY, - (String value) -> value.equals(USE_CONTEXT_RATE_KEY) ? USE_CONTEXT_RATE_VALUE: MAX_COMPILATION_RATE_FUNCTION.apply(value), - Property.Dynamic, Property.NodeScope, Property.Deprecated); // Per-context settings static final String CONTEXT_PREFIX = "script.context."; @@ -128,15 +82,15 @@ public class ScriptService implements Closeable, ClusterStateApplier { // Unlimited compilation rate for context-specific script caches static final String UNLIMITED_COMPILATION_RATE_KEY = "unlimited"; - public static final Setting.AffixSetting> SCRIPT_MAX_COMPILATIONS_RATE_SETTING = + public static final Setting.AffixSetting SCRIPT_MAX_COMPILATIONS_RATE_SETTING = Setting.affixKeySetting(CONTEXT_PREFIX, "max_compilations_rate", - key -> new Setting<>(key, "75/5m", + key -> new Setting(key, "75/5m", (String value) -> value.equals(UNLIMITED_COMPILATION_RATE_KEY) ? ScriptCache.UNLIMITED_COMPILATION_RATE: - MAX_COMPILATION_RATE_FUNCTION.apply(value), + new ScriptCache.CompilationRate(value), Property.NodeScope, Property.Dynamic)); - private static final Tuple SCRIPT_COMPILATION_RATE_ZERO = new Tuple<>(0, TimeValue.ZERO); + private static final ScriptCache.CompilationRate SCRIPT_COMPILATION_RATE_ZERO = new ScriptCache.CompilationRate(0, TimeValue.ZERO); public static final Setting SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING = Setting.boolSetting("script.disable_max_compilations_rate", false, Property.NodeScope); @@ -242,7 +196,7 @@ public ScriptService(Settings settings, Map engines, Map cacheHolder.get().set(context.name, contextCache(settings, context)), List.of(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name), SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context.name), - SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name), - SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, // general settings used for fallbacks - SCRIPT_GENERAL_CACHE_SIZE_SETTING + SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name) ) ); } - - // Handle all settings for context and general caches, this flips between general and context caches. - clusterSettings.addSettingsUpdateConsumer( - (settings) -> setCacheHolder(settings), - List.of(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING, - SCRIPT_GENERAL_CACHE_EXPIRE_SETTING, - SCRIPT_GENERAL_CACHE_SIZE_SETTING, - SCRIPT_MAX_COMPILATIONS_RATE_SETTING, - SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING, - SCRIPT_CACHE_EXPIRE_SETTING, - SCRIPT_CACHE_SIZE_SETTING), - this::validateCacheSettings - ); } /** @@ -287,7 +226,6 @@ void registerClusterSettingsListeners(ClusterSettings clusterSettings) { * when using the general cache. */ void validateCacheSettings(Settings settings) { - boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE); List> affixes = List.of(SCRIPT_MAX_COMPILATIONS_RATE_SETTING, SCRIPT_CACHE_EXPIRE_SETTING, SCRIPT_CACHE_SIZE_SETTING); List customRates = new ArrayList<>(); @@ -304,11 +242,6 @@ void validateCacheSettings(Settings settings) { } } } - if (useContext == false && keys.isEmpty() == false) { - keys.sort(Comparator.naturalOrder()); - throw new IllegalArgumentException("Context cache settings [" + String.join(", ", keys) + "] requires [" + - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to be [" + USE_CONTEXT_RATE_KEY + "]"); - } if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings)) { if (customRates.size() > 0) { customRates.sort(Comparator.naturalOrder()); @@ -316,12 +249,6 @@ void validateCacheSettings(Settings settings) { String.join(", ", customRates) + "] if compile rates disabled via [" + SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.getKey() + "]"); } - if (useContext == false) { - throw new IllegalArgumentException("Cannot set custom general compilation rates [" + - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey() + "] to [" + - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings) + "] if compile rates disabled via [" + - SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.getKey() + "]"); - } } } @@ -576,48 +503,11 @@ public ScriptStats stats() { return cacheHolder.get().stats(); } - public ScriptCacheStats cacheStats() { - return cacheHolder.get().cacheStats(); - } - @Override public void applyClusterState(ClusterChangedEvent event) { clusterState = event.state(); } - void setCacheHolder(Settings settings) { - CacheHolder current = cacheHolder.get(); - boolean useContext = SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings).equals(USE_CONTEXT_RATE_VALUE); - - if (current == null) { - if (useContext) { - cacheHolder.set(contextCacheHolder(settings)); - } else { - cacheHolder.set(generalCacheHolder(settings)); - } - return; - } - - // Update - if (useContext) { - if (current.general != null) { - // Flipping to context specific - cacheHolder.set(contextCacheHolder(settings)); - } - } else if (current.general == null) { - // Flipping to general - cacheHolder.set(generalCacheHolder(settings)); - } else if (current.general.rate.equals(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings)) == false) { - // General compilation rate changed, that setting is the only dynamically updated general setting - cacheHolder.set(generalCacheHolder(settings)); - } - } - - CacheHolder generalCacheHolder(Settings settings) { - return new CacheHolder(SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(settings), SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(settings), - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(settings), SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey()); - } - CacheHolder contextCacheHolder(Settings settings) { Map contextCache = new HashMap<>(contexts.size()); contexts.forEach((k, v) -> contextCache.put(k, contextCache(settings, v))); @@ -632,14 +522,15 @@ ScriptCache contextCache(Settings settings, ScriptContext context) { TimeValue cacheExpire = cacheExpireSetting.existsOrFallbackExists(settings) ? cacheExpireSetting.get(settings) : context.cacheExpireDefault; - Setting> rateSetting = SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name); - Tuple rate = null; + Setting rateSetting = + SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context.name); + ScriptCache.CompilationRate rate = null; if (SCRIPT_DISABLE_MAX_COMPILATIONS_RATE_SETTING.get(settings) || compilationLimitsEnabled() == false) { rate = SCRIPT_COMPILATION_RATE_ZERO; } else if (rateSetting.existsOrFallbackExists(settings)) { rate = rateSetting.get(settings); } else { - rate = context.maxCompilationRateDefault; + rate = new ScriptCache.CompilationRate(context.maxCompilationRateDefault); } return new ScriptCache(cacheSize, cacheExpire, rate, rateSetting.getKey()); @@ -651,19 +542,12 @@ ScriptCache contextCache(Settings settings, ScriptContext context) { * 2) context mode, if the context script cache is configured. There is no general cache in this case. */ static class CacheHolder { - final ScriptCache general; final Map> contextCache; - CacheHolder(int cacheMaxSize, TimeValue cacheExpire, Tuple maxCompilationRate, String contextRateSetting) { - contextCache = null; - general = new ScriptCache(cacheMaxSize, cacheExpire, maxCompilationRate, contextRateSetting); - } - CacheHolder(Map context) { Map> refs = new HashMap<>(context.size()); context.forEach((k, v) -> refs.put(k, new AtomicReference<>(v))); contextCache = Collections.unmodifiableMap(refs); - general = null; } /** @@ -671,9 +555,6 @@ static class CacheHolder { * the given context. Returns null in context mode if the requested context does not exist. */ ScriptCache get(String context) { - if (general != null) { - return general; - } AtomicReference ref = contextCache.get(context); if (ref == null) { return null; @@ -682,30 +563,17 @@ ScriptCache get(String context) { } ScriptStats stats() { - if (general != null) { - return general.stats(); + List stats = new ArrayList<>(contextCache.size()); + for (Map.Entry> entry : contextCache.entrySet()) { + stats.add(entry.getValue().get().stats(entry.getKey())); } - return ScriptStats.sum(contextCache.values().stream().map(AtomicReference::get).map(ScriptCache::stats)::iterator); - } - - ScriptCacheStats cacheStats() { - if (general != null) { - return new ScriptCacheStats(general.stats()); - } - Map context = new HashMap<>(contextCache.size()); - for (String name: contextCache.keySet()) { - context.put(name, contextCache.get(name).get().stats()); - } - return new ScriptCacheStats(context); + return new ScriptStats(stats); } /** * Update a single context cache if we're in the context cache mode otherwise no-op. */ void set(String name, ScriptCache cache) { - if (general != null) { - return; - } AtomicReference ref = contextCache.get(name); assert ref != null : "expected script cache to exist for context [" + name + "]"; ScriptCache oldCache = ref.get(); diff --git a/server/src/main/java/org/elasticsearch/script/ScriptStats.java b/server/src/main/java/org/elasticsearch/script/ScriptStats.java index 953b3a18ae92c..af4724228b0d2 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptStats.java @@ -26,13 +26,26 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; public class ScriptStats implements Writeable, ToXContentFragment { + private final List contextStats; private final long compilations; private final long cacheEvictions; private final long compilationLimitTriggered; - public ScriptStats(long compilations, long cacheEvictions, long compilationLimitTriggered) { + + public ScriptStats(List contextStats) { + this.contextStats = contextStats.stream().sorted().collect(Collectors.toUnmodifiableList()); + long compilations = 0; + long cacheEvictions = 0; + long compilationLimitTriggered = 0; + for (ScriptContextStats stats: contextStats) { + compilations += stats.getCompilations(); + cacheEvictions += stats.getCacheEvictions(); + compilationLimitTriggered += stats.getCompilationLimitTriggered(); + } this.compilations = compilations; this.cacheEvictions = cacheEvictions; this.compilationLimitTriggered = compilationLimitTriggered; @@ -42,6 +55,7 @@ public ScriptStats(StreamInput in) throws IOException { compilations = in.readVLong(); cacheEvictions = in.readVLong(); compilationLimitTriggered = in.readVLong(); + contextStats = in.readList(ScriptContextStats::new); } @Override @@ -49,6 +63,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(compilations); out.writeVLong(cacheEvictions); out.writeVLong(compilationLimitTriggered); + out.writeList(contextStats); + } + + public List getContextStats() { + return contextStats; } public long getCompilations() { @@ -66,33 +85,23 @@ public long getCompilationLimitTriggered() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.SCRIPT_STATS); - builder.field(Fields.COMPILATIONS, getCompilations()); - builder.field(Fields.CACHE_EVICTIONS, getCacheEvictions()); - builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, getCompilationLimitTriggered()); + builder.field(Fields.COMPILATIONS, compilations); + builder.field(Fields.CACHE_EVICTIONS, cacheEvictions); + builder.field(Fields.COMPILATION_LIMIT_TRIGGERED, compilationLimitTriggered); + builder.startArray(Fields.CONTEXTS); + for (ScriptContextStats contextStats: contextStats) { + contextStats.toXContent(builder, params); + } + builder.endArray(); builder.endObject(); return builder; } static final class Fields { static final String SCRIPT_STATS = "script"; + static final String CONTEXTS = "contexts"; static final String COMPILATIONS = "compilations"; static final String CACHE_EVICTIONS = "cache_evictions"; static final String COMPILATION_LIMIT_TRIGGERED = "compilation_limit_triggered"; } - - public static ScriptStats sum(Iterable stats) { - long compilations = 0; - long cacheEvictions = 0; - long compilationLimitTriggered = 0; - for (ScriptStats stat: stats) { - compilations += stat.compilations; - cacheEvictions += stat.cacheEvictions; - compilationLimitTriggered += stat.compilationLimitTriggered; - } - return new ScriptStats( - compilations, - cacheEvictions, - compilationLimitTriggered - ); - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java index d5f4fc4c0f1f5..0727cb5a24dc4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java @@ -19,11 +19,13 @@ package org.elasticsearch.search.aggregations; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreMode; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.SearchContext.Lifetime; import org.elasticsearch.search.query.QueryPhaseExecutionException; @@ -34,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; /** * Base implementation for concrete aggregators. @@ -106,6 +109,26 @@ public ScoreMode scoreMode() { addRequestCircuitBreakerBytes(DEFAULT_WEIGHT); } + /** + * Returns a converter for point values if it's safe to use the indexed data instead of + * doc values. Generally, this means that the query has no filters or scripts, the aggregation is + * top level, and the underlying field is indexed, and the index is sorted in the right order. + * + * If those conditions aren't met, return null to indicate a point reader cannot + * be used in this case. + * + * @param config The config for the values source metric. + */ + public final Function pointReaderIfAvailable(ValuesSourceConfig config) { + if (context.query() != null && context.query().getClass() != MatchAllDocsQuery.class) { + return null; + } + if (parent != null) { + return null; + } + return config.getPointReaderOrNull(); + } + /** * Increment or decrement the number of bytes that have been allocated to service * this request and potentially trigger a {@link CircuitBreakingException}. The diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java index 22be092fee9af..b539b37c09122 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregator.java @@ -50,6 +50,7 @@ public abstract class AbstractHistogramAggregator extends BucketsAggregator { protected final long minDocCount; protected final double minBound; protected final double maxBound; + protected final DoubleBounds hardBounds; protected final LongKeyedBucketOrds bucketOrds; public AbstractHistogramAggregator( @@ -62,6 +63,7 @@ public AbstractHistogramAggregator( long minDocCount, double minBound, double maxBound, + DoubleBounds hardBounds, DocValueFormat formatter, SearchContext context, Aggregator parent, @@ -80,6 +82,7 @@ public AbstractHistogramAggregator( this.minDocCount = minDocCount; this.minBound = minBound; this.maxBound = maxBound; + this.hardBounds = hardBounds; this.formatter = formatter; bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), cardinalityUpperBound); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java index 9260a14f57a81..70a96bec3ff44 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; +import org.elasticsearch.Version; import org.elasticsearch.common.Rounding; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -92,8 +93,11 @@ public class DateHistogramAggregationBuilder extends ValuesSourceAggregationBuil PARSER.declareLong(DateHistogramAggregationBuilder::minDocCount, Histogram.MIN_DOC_COUNT_FIELD); - PARSER.declareField(DateHistogramAggregationBuilder::extendedBounds, parser -> ExtendedBounds.PARSER.apply(parser, null), - ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(DateHistogramAggregationBuilder::extendedBounds, parser -> LongBounds.PARSER.apply(parser, null), + Histogram.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); + + PARSER.declareField(DateHistogramAggregationBuilder::hardBounds, parser -> LongBounds.PARSER.apply(parser, null), + Histogram.HARD_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); PARSER.declareObjectArray(DateHistogramAggregationBuilder::order, (p, c) -> InternalOrder.Parser.parseOrderParam(p), Histogram.ORDER_FIELD); @@ -105,7 +109,8 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private DateIntervalWrapper dateHistogramInterval = new DateIntervalWrapper(); private long offset = 0; - private ExtendedBounds extendedBounds; + private LongBounds extendedBounds; + private LongBounds hardBounds; private BucketOrder order = BucketOrder.key(true); private boolean keyed = false; private long minDocCount = 0; @@ -121,13 +126,14 @@ protected DateHistogramAggregationBuilder(DateHistogramAggregationBuilder clone, this.dateHistogramInterval = clone.dateHistogramInterval; this.offset = clone.offset; this.extendedBounds = clone.extendedBounds; + this.hardBounds = clone.hardBounds; this.order = clone.order; this.keyed = clone.keyed; this.minDocCount = clone.minDocCount; } @Override -protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { + protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { return new DateHistogramAggregationBuilder(this, factoriesBuilder, metadata); } @@ -139,7 +145,10 @@ public DateHistogramAggregationBuilder(StreamInput in) throws IOException { minDocCount = in.readVLong(); dateHistogramInterval = new DateIntervalWrapper(in); offset = in.readLong(); - extendedBounds = in.readOptionalWriteable(ExtendedBounds::new); + extendedBounds = in.readOptionalWriteable(LongBounds::new); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + hardBounds = in.readOptionalWriteable(LongBounds::new); + } } @Override @@ -156,6 +165,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException { dateHistogramInterval.writeTo(out); out.writeLong(offset); out.writeOptionalWriteable(extendedBounds); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalWriteable(hardBounds); + } } /** Get the current interval in milliseconds that is set on this builder. */ @@ -281,13 +293,13 @@ public static long parseStringOffset(String offset) { } /** Return extended bounds for this histogram, or {@code null} if none are set. */ - public ExtendedBounds extendedBounds() { + public LongBounds extendedBounds() { return extendedBounds; } /** Set extended bounds on this histogram, so that buckets would also be * generated on intervals that did not match any documents. */ - public DateHistogramAggregationBuilder extendedBounds(ExtendedBounds extendedBounds) { + public DateHistogramAggregationBuilder extendedBounds(LongBounds extendedBounds) { if (extendedBounds == null) { throw new IllegalArgumentException("[extendedBounds] must not be null: [" + name + "]"); } @@ -295,6 +307,21 @@ public DateHistogramAggregationBuilder extendedBounds(ExtendedBounds extendedBou return this; } + + /** Return hard bounds for this histogram, or {@code null} if none are set. */ + public LongBounds hardBounds() { + return hardBounds; + } + + /** Set hard bounds on this histogram, specifying boundaries outside which buckets cannot be created. */ + public DateHistogramAggregationBuilder hardBounds(LongBounds hardBounds) { + if (hardBounds == null) { + throw new IllegalArgumentException("[hardBounds] must not be null: [" + name + "]"); + } + this.hardBounds = hardBounds; + return this; + } + /** Return the order to use to sort buckets of this histogram. */ public BucketOrder order() { return order; @@ -378,9 +405,16 @@ protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) builder.field(Histogram.MIN_DOC_COUNT_FIELD.getPreferredName(), minDocCount); if (extendedBounds != null) { + builder.startObject(Histogram.EXTENDED_BOUNDS_FIELD.getPreferredName()); extendedBounds.toXContent(builder, params); + builder.endObject(); } + if (hardBounds != null) { + builder.startObject(Histogram.HARD_BOUNDS_FIELD.getPreferredName()); + hardBounds.toXContent(builder, params); + builder.endObject(); + } return builder; } @@ -397,11 +431,33 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardC final ZoneId tz = timeZone(); final Rounding rounding = dateHistogramInterval.createRounding(tz, offset); - ExtendedBounds roundedBounds = null; + LongBounds roundedBounds = null; if (this.extendedBounds != null) { // parse any string bounds to longs and round - roundedBounds = this.extendedBounds.parseAndValidate(name, queryShardContext, config.format()).round(rounding); + roundedBounds = this.extendedBounds.parseAndValidate(name, "extended_bounds" , queryShardContext, config.format()) + .round(rounding); } + + LongBounds roundedHardBounds = null; + if (this.hardBounds != null) { + // parse any string bounds to longs and round + roundedHardBounds = this.hardBounds.parseAndValidate(name, "hard_bounds" , queryShardContext, config.format()) + .round(rounding); + } + + if (roundedBounds != null && roundedHardBounds != null) { + if (roundedBounds.getMax() != null && + roundedHardBounds.getMax() != null && roundedBounds.getMax() > roundedHardBounds.getMax()) { + throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + + hardBounds + "], extended bounds: [" + extendedBounds + "]"); + } + if (roundedBounds.getMin() != null && + roundedHardBounds.getMin() != null && roundedBounds.getMin() < roundedHardBounds.getMin()) { + throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + + hardBounds + "], extended bounds: [" + extendedBounds + "]"); + } + } + return new DateHistogramAggregatorFactory( name, config, @@ -410,16 +466,16 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardC minDocCount, rounding, roundedBounds, + roundedHardBounds, queryShardContext, parent, subFactoriesBuilder, - metadata - ); + metadata); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), order, keyed, minDocCount, dateHistogramInterval, minDocCount, extendedBounds); + return Objects.hash(super.hashCode(), order, keyed, minDocCount, dateHistogramInterval, minDocCount, extendedBounds, hardBounds); } @Override @@ -433,6 +489,7 @@ public boolean equals(Object obj) { && Objects.equals(minDocCount, other.minDocCount) && Objects.equals(dateHistogramInterval, other.dateHistogramInterval) && Objects.equals(offset, other.offset) - && Objects.equals(extendedBounds, other.extendedBounds); + && Objects.equals(extendedBounds, other.extendedBounds) + && Objects.equals(hardBounds, other.hardBounds); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java index 34366d2798fb6..8ccc379fb5d8a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationSupplier.java @@ -41,7 +41,8 @@ Aggregator build(String name, BucketOrder order, boolean keyed, long minDocCount, - @Nullable ExtendedBounds extendedBounds, + @Nullable LongBounds extendedBounds, + @Nullable LongBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext aggregationContext, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java index d50d0d785b2d2..e1b147f66203d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java @@ -63,7 +63,8 @@ class DateHistogramAggregator extends BucketsAggregator { private final boolean keyed; private final long minDocCount; - private final ExtendedBounds extendedBounds; + private final LongBounds extendedBounds; + private final LongBounds hardBounds; private final LongKeyedBucketOrds bucketOrds; @@ -75,7 +76,8 @@ class DateHistogramAggregator extends BucketsAggregator { BucketOrder order, boolean keyed, long minDocCount, - @Nullable ExtendedBounds extendedBounds, + @Nullable LongBounds extendedBounds, + @Nullable LongBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext aggregationContext, Aggregator parent, @@ -91,6 +93,7 @@ class DateHistogramAggregator extends BucketsAggregator { this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; + this.hardBounds = hardBounds; // TODO: Stop using null here this.valuesSource = valuesSourceConfig.hasValues() ? (ValuesSource.Numeric) valuesSourceConfig.getValuesSource() : null; this.formatter = valuesSourceConfig.format(); @@ -126,12 +129,14 @@ public void collect(int doc, long owningBucketOrd) throws IOException { if (rounded == previousRounded) { continue; } - long bucketOrd = bucketOrds.add(owningBucketOrd, rounded); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (hardBounds == null || hardBounds.contain(rounded)) { + long bucketOrd = bucketOrds.add(owningBucketOrd, rounded); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } previousRounded = rounded; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java index d72df91008441..c883c4df2f5d6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorFactory.java @@ -53,7 +53,8 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private final BucketOrder order; private final boolean keyed; private final long minDocCount; - private final ExtendedBounds extendedBounds; + private final LongBounds extendedBounds; + private final LongBounds hardBounds; private final Rounding rounding; public DateHistogramAggregatorFactory( @@ -63,7 +64,8 @@ public DateHistogramAggregatorFactory( boolean keyed, long minDocCount, Rounding rounding, - ExtendedBounds extendedBounds, + LongBounds extendedBounds, + LongBounds hardBounds, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -74,6 +76,7 @@ public DateHistogramAggregatorFactory( this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; + this.hardBounds = hardBounds; this.rounding = rounding; } @@ -107,6 +110,7 @@ protected Aggregator doCreateInternal( keyed, minDocCount, extendedBounds, + hardBounds, config, searchContext, parent, @@ -119,7 +123,7 @@ protected Aggregator doCreateInternal( protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { - return new DateHistogramAggregator(name, factories, rounding, null, order, keyed, minDocCount, extendedBounds, + return new DateHistogramAggregator(name, factories, rounding, null, order, keyed, minDocCount, extendedBounds, hardBounds, config, searchContext, parent, CardinalityUpperBound.NONE, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java index 6f9c452fb50ef..d2712af4fd4d8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregator.java @@ -48,6 +48,9 @@ import java.util.Map; import java.util.function.BiConsumer; +import static java.lang.Long.max; +import static java.lang.Long.min; + /** * An aggregator for date values. Every date is rounded down using a configured * {@link Rounding}. @@ -67,7 +70,8 @@ class DateRangeHistogramAggregator extends BucketsAggregator { private final boolean keyed; private final long minDocCount; - private final ExtendedBounds extendedBounds; + private final LongBounds extendedBounds; + private final LongBounds hardBounds; private final LongKeyedBucketOrds bucketOrds; @@ -79,7 +83,8 @@ class DateRangeHistogramAggregator extends BucketsAggregator { BucketOrder order, boolean keyed, long minDocCount, - @Nullable ExtendedBounds extendedBounds, + @Nullable LongBounds extendedBounds, + @Nullable LongBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext aggregationContext, Aggregator parent, @@ -95,6 +100,7 @@ class DateRangeHistogramAggregator extends BucketsAggregator { this.keyed = keyed; this.minDocCount = minDocCount; this.extendedBounds = extendedBounds; + this.hardBounds = hardBounds; // TODO: Stop using null here this.valuesSource = valuesSourceConfig.hasValues() ? (ValuesSource.Range) valuesSourceConfig.getValuesSource() : null; this.formatter = valuesSourceConfig.format(); @@ -140,9 +146,13 @@ public void collect(int doc, long owningBucketOrd) throws IOException { // The encoding should ensure that this assert is always true. assert from >= previousFrom : "Start of range not >= previous start"; final Long to = (Long) range.getTo(); - final long startKey = preparedRounding.round(from); - final long endKey = preparedRounding.round(to); - for (long key = startKey > previousKey ? startKey : previousKey; key <= endKey; + final long effectiveFrom = (hardBounds != null && hardBounds.getMin() != null) ? + max(from, hardBounds.getMin()) : from; + final long effectiveTo = (hardBounds != null && hardBounds.getMax() != null) ? + min(to, hardBounds.getMax()) : to; + final long startKey = preparedRounding.round(effectiveFrom); + final long endKey = preparedRounding.round(effectiveTo); + for (long key = max(startKey, previousKey); key <= endKey; key = preparedRounding.nextRoundingValue(key)) { if (key == previousKey) { continue; @@ -167,6 +177,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException { }; } + + @Override public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { return buildAggregationsForVariableBuckets(owningBucketOrds, bucketOrds, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java new file mode 100644 index 0000000000000..7ab5ee3f5fb7e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBounds.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.histogram; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.InstantiatingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Represent hard_bounds and extended_bounds in histogram aggregations. + * + * This class is similar to {@link LongBounds} used in date histograms, but is using longs to store data. LongBounds and DoubleBounds are + * not used interchangeably and therefore don't share any common interfaces except for serialization. + */ + +public class DoubleBounds implements ToXContentFragment, Writeable { + static final ParseField MIN_FIELD = new ParseField("min"); + static final ParseField MAX_FIELD = new ParseField("max"); + static final InstantiatingObjectParser PARSER; + + static { + InstantiatingObjectParser.Builder parser = + InstantiatingObjectParser.builder("double_bounds", false, DoubleBounds.class); + parser.declareDouble(optionalConstructorArg(), MIN_FIELD); + parser.declareDouble(optionalConstructorArg(), MAX_FIELD); + PARSER = parser.build(); + } + + /** + * Min value + */ + private final Double min; + + /** + * Max value + */ + private final Double max; + + /** + * Construct with bounds. + */ + public DoubleBounds(Double min, Double max) { + this.min = min; + this.max = max; + } + + /** + * Read from a stream. + */ + public DoubleBounds(StreamInput in) throws IOException { + min = in.readOptionalDouble(); + max = in.readOptionalDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalDouble(min); + out.writeOptionalDouble(max); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(MIN_FIELD.getPreferredName(), min); + builder.field(MAX_FIELD.getPreferredName(), max); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(min, max); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + DoubleBounds other = (DoubleBounds) obj; + return Objects.equals(min, other.min) + && Objects.equals(max, other.max); + } + + public Double getMin() { + return min; + } + + public Double getMax() { + return max; + } + + public boolean contain(double value) { + if (max != null && value > max) { + return false; + } + if (min != null && value < min) { + return false; + } + return true; + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + if (min != null) { + b.append(min); + } + b.append("--"); + if (max != null) { + b.append(max); + } + return b.toString(); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java index 07e2eb879623c..8ae266891362a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java @@ -34,6 +34,7 @@ public interface Histogram extends MultiBucketsAggregation { ParseField KEYED_FIELD = new ParseField("keyed"); ParseField MIN_DOC_COUNT_FIELD = new ParseField("min_doc_count"); ParseField EXTENDED_BOUNDS_FIELD = new ParseField("extended_bounds"); + ParseField HARD_BOUNDS_FIELD = new ParseField("hard_bounds"); /** * A bucket in the histogram where documents fall in diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java index 43e1750268ba1..bf7e603262396 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.histogram; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -73,7 +74,10 @@ public class HistogramAggregationBuilder extends ValuesSourceAggregationBuilder< PARSER.declareField((histogram, extendedBounds) -> { histogram.extendedBounds(extendedBounds[0], extendedBounds[1]); - }, parser -> EXTENDED_BOUNDS_PARSER.apply(parser, null), ExtendedBounds.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); + }, parser -> EXTENDED_BOUNDS_PARSER.apply(parser, null), Histogram.EXTENDED_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); + + PARSER.declareField(HistogramAggregationBuilder::hardBounds, parser -> DoubleBounds.PARSER.apply(parser, null), + Histogram.HARD_BOUNDS_FIELD, ObjectParser.ValueType.OBJECT); PARSER.declareObjectArray(HistogramAggregationBuilder::order, (p, c) -> InternalOrder.Parser.parseOrderParam(p), Histogram.ORDER_FIELD); @@ -85,8 +89,10 @@ public static void registerAggregators(ValuesSourceRegistry.Builder builder) { private double interval; private double offset = 0; + //TODO: Replace with DoubleBounds private double minBound = Double.POSITIVE_INFINITY; private double maxBound = Double.NEGATIVE_INFINITY; + private DoubleBounds hardBounds; private BucketOrder order = BucketOrder.key(true); private boolean keyed = false; private long minDocCount = 0; @@ -109,6 +115,7 @@ protected HistogramAggregationBuilder(HistogramAggregationBuilder clone, this.offset = clone.offset; this.minBound = clone.minBound; this.maxBound = clone.maxBound; + this.hardBounds = clone.hardBounds; this.order = clone.order; this.keyed = clone.keyed; this.minDocCount = clone.minDocCount; @@ -129,6 +136,9 @@ public HistogramAggregationBuilder(StreamInput in) throws IOException { offset = in.readDouble(); minBound = in.readDouble(); maxBound = in.readDouble(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + hardBounds = in.readOptionalWriteable(DoubleBounds::new); + } } @Override @@ -140,6 +150,9 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeDouble(offset); out.writeDouble(minBound); out.writeDouble(maxBound); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalWriteable(hardBounds); + } } /** Get the current interval that is set on this builder. */ @@ -201,6 +214,17 @@ public HistogramAggregationBuilder extendedBounds(double minBound, double maxBou return this; } + /** + * Set hard bounds on this histogram, specifying boundaries outside which buckets cannot be created. + */ + public HistogramAggregationBuilder hardBounds(DoubleBounds hardBounds) { + if (hardBounds == null) { + throw new IllegalArgumentException("[hardBounds] must not be null: [" + name + "]"); + } + this.hardBounds = hardBounds; + return this; + } + /** Return the order to use to sort buckets of this histogram. */ public BucketOrder order() { return order; @@ -307,13 +331,24 @@ protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardC ValuesSourceConfig config, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException { + + if (hardBounds != null) { + if (hardBounds.getMax() != null && hardBounds.getMax() < maxBound) { + throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + + hardBounds + "], extended bounds: [" + minBound + "--" + maxBound + "]"); + } + if (hardBounds.getMin() != null && hardBounds.getMin() > minBound) { + throw new IllegalArgumentException("Extended bounds have to be inside hard bounds, hard bounds: [" + + hardBounds + "], extended bounds: [" + minBound + "--" + maxBound + "]"); + } + } return new HistogramAggregatorFactory(name, config, interval, offset, order, keyed, minDocCount, minBound, maxBound, - queryShardContext, parent, subFactoriesBuilder, metadata); + hardBounds, queryShardContext, parent, subFactoriesBuilder, metadata); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), order, keyed, minDocCount, interval, offset, minBound, maxBound); + return Objects.hash(super.hashCode(), order, keyed, minDocCount, interval, offset, minBound, maxBound, hardBounds); } @Override @@ -328,6 +363,7 @@ public boolean equals(Object obj) { && Objects.equals(interval, other.interval) && Objects.equals(offset, other.offset) && Objects.equals(minBound, other.minBound) - && Objects.equals(maxBound, other.maxBound); + && Objects.equals(maxBound, other.maxBound) + && Objects.equals(hardBounds, other.hardBounds); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java index 2f776bef867be..b686a0267fe4a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorFactory.java @@ -48,6 +48,7 @@ public final class HistogramAggregatorFactory extends ValuesSourceAggregatorFact private final boolean keyed; private final long minDocCount; private final double minBound, maxBound; + private final DoubleBounds hardBounds; static void registerAggregators(ValuesSourceRegistry.Builder builder) { builder.register(HistogramAggregationBuilder.NAME, CoreValuesSourceType.RANGE, @@ -67,6 +68,7 @@ public HistogramAggregatorFactory(String name, long minDocCount, double minBound, double maxBound, + DoubleBounds hardBounds, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -79,6 +81,7 @@ public HistogramAggregatorFactory(String name, this.minDocCount = minDocCount; this.minBound = minBound; this.maxBound = maxBound; + this.hardBounds = hardBounds; } public long minDocCount() { @@ -98,7 +101,7 @@ protected Aggregator doCreateInternal(SearchContext searchContext, } HistogramAggregatorSupplier histogramAggregatorSupplier = (HistogramAggregatorSupplier) aggregatorSupplier; return histogramAggregatorSupplier.build(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, - config, searchContext, parent, cardinality, metadata); + hardBounds, config, searchContext, parent, cardinality, metadata); } @Override @@ -106,6 +109,6 @@ protected Aggregator createUnmapped(SearchContext searchContext, Aggregator parent, Map metadata) throws IOException { return new NumericHistogramAggregator(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, - config, searchContext, parent, CardinalityUpperBound.NONE, metadata); + hardBounds, config, searchContext, parent, CardinalityUpperBound.NONE, metadata); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java index 89d5d15e1ffd7..f89e609140c1c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorSupplier.java @@ -40,6 +40,7 @@ Aggregator build( long minDocCount, double minBound, double maxBound, + DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, Aggregator parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java index 4b03be7f34729..697e8efb61600 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java @@ -160,13 +160,13 @@ static class EmptyBucketInfo { final Rounding rounding; final InternalAggregations subAggregations; - final ExtendedBounds bounds; + final LongBounds bounds; EmptyBucketInfo(Rounding rounding, InternalAggregations subAggregations) { this(rounding, subAggregations, null); } - EmptyBucketInfo(Rounding rounding, InternalAggregations subAggregations, ExtendedBounds bounds) { + EmptyBucketInfo(Rounding rounding, InternalAggregations subAggregations, LongBounds bounds) { this.rounding = rounding; this.subAggregations = subAggregations; this.bounds = bounds; @@ -175,7 +175,7 @@ static class EmptyBucketInfo { EmptyBucketInfo(StreamInput in) throws IOException { rounding = Rounding.read(in); subAggregations = InternalAggregations.readFrom(in); - bounds = in.readOptionalWriteable(ExtendedBounds::new); + bounds = in.readOptionalWriteable(LongBounds::new); } void writeTo(StreamOutput out) throws IOException { @@ -377,7 +377,7 @@ protected Bucket reduceBucket(List buckets, ReduceContext context) { private void addEmptyBuckets(List list, ReduceContext reduceContext) { Bucket lastBucket = null; - ExtendedBounds bounds = emptyBucketInfo.bounds; + LongBounds bounds = emptyBucketInfo.bounds; ListIterator iter = list.listIterator(); // first adding all the empty buckets *before* the actual data (based on th extended_bounds.min the user requested) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java index 0390f4fe25e4c..a096feeadff68 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java @@ -152,12 +152,12 @@ public boolean getKeyed() { } } - static class EmptyBucketInfo { + public static class EmptyBucketInfo { final double interval, offset, minBound, maxBound; final InternalAggregations subAggregations; - EmptyBucketInfo(double interval, double offset, double minBound, double maxBound, InternalAggregations subAggregations) { + public EmptyBucketInfo(double interval, double offset, double minBound, double maxBound, InternalAggregations subAggregations) { this.interval = interval; this.offset = offset; this.minBound = minBound; @@ -203,8 +203,15 @@ public int hashCode() { private final long minDocCount; final EmptyBucketInfo emptyBucketInfo; - InternalHistogram(String name, List buckets, BucketOrder order, long minDocCount, EmptyBucketInfo emptyBucketInfo, - DocValueFormat formatter, boolean keyed, Map metadata) { + public InternalHistogram( + String name, + List buckets, + BucketOrder order, + long minDocCount, + EmptyBucketInfo emptyBucketInfo, + DocValueFormat formatter, + boolean keyed, + Map metadata) { super(name, metadata); this.buckets = buckets; this.order = order; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBounds.java similarity index 82% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBounds.java index 4a9deb9bdedfc..e6ddbfb495313 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBounds.java @@ -39,13 +39,18 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; -public class ExtendedBounds implements ToXContentFragment, Writeable { - static final ParseField EXTENDED_BOUNDS_FIELD = Histogram.EXTENDED_BOUNDS_FIELD; +/** + * Represent hard_bounds and extended_bounds in date-histogram aggregations. + * + * This class is similar to {@link DoubleBounds} used in histograms, but is using longs to store data. LongBounds and DoubleBounds are + * * not used interchangeably and therefore don't share any common interfaces except for serialization. + */ +public class LongBounds implements ToXContentFragment, Writeable { static final ParseField MIN_FIELD = new ParseField("min"); static final ParseField MAX_FIELD = new ParseField("max"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "extended_bounds", a -> { + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "bounds", a -> { assert a.length == 2; Long min = null; Long max = null; @@ -69,7 +74,7 @@ public class ExtendedBounds implements ToXContentFragment, Writeable { } else { throw new IllegalArgumentException("Unknown field type [" + a[1].getClass() + "]"); } - return new ExtendedBounds(min, max, minAsStr, maxAsStr); + return new LongBounds(min, max, minAsStr, maxAsStr); }); static { CheckedFunction longOrString = p -> { @@ -105,21 +110,21 @@ public class ExtendedBounds implements ToXContentFragment, Writeable { /** * Construct with parsed bounds. */ - public ExtendedBounds(Long min, Long max) { + public LongBounds(Long min, Long max) { this(min, max, null, null); } /** * Construct with unparsed bounds. */ - public ExtendedBounds(String minAsStr, String maxAsStr) { + public LongBounds(String minAsStr, String maxAsStr) { this(null, null, minAsStr, maxAsStr); } /** * Construct with all possible information. */ - private ExtendedBounds(Long min, Long max, String minAsStr, String maxAsStr) { + private LongBounds(Long min, Long max, String minAsStr, String maxAsStr) { this.min = min; this.max = max; this.minAsStr = minAsStr; @@ -129,7 +134,7 @@ private ExtendedBounds(Long min, Long max, String minAsStr, String maxAsStr) { /** * Read from a stream. */ - public ExtendedBounds(StreamInput in) throws IOException { + public LongBounds(StreamInput in) throws IOException { min = in.readOptionalLong(); max = in.readOptionalLong(); minAsStr = in.readOptionalString(); @@ -147,7 +152,7 @@ public void writeTo(StreamOutput out) throws IOException { /** * Parse the bounds and perform any delayed validation. Returns the result of the parsing. */ - ExtendedBounds parseAndValidate(String aggName, QueryShardContext queryShardContext, DocValueFormat format) { + LongBounds parseAndValidate(String aggName, String boundsName, QueryShardContext queryShardContext, DocValueFormat format) { Long min = this.min; Long max = this.max; assert format != null; @@ -159,23 +164,22 @@ ExtendedBounds parseAndValidate(String aggName, QueryShardContext queryShardCont max = format.parseLong(maxAsStr, false, queryShardContext::nowInMillis); } if (min != null && max != null && min.compareTo(max) > 0) { - throw new IllegalArgumentException("[extended_bounds.min][" + min + "] cannot be greater than " + - "[extended_bounds.max][" + max + "] for histogram aggregation [" + aggName + "]"); + throw new IllegalArgumentException("[" + boundsName + ".min][" + min + "] cannot be greater than " + + "[" + boundsName + ".max][" + max + "] for histogram aggregation [" + aggName + "]"); } - return new ExtendedBounds(min, max, minAsStr, maxAsStr); + return new LongBounds(min, max, minAsStr, maxAsStr); } - ExtendedBounds round(Rounding rounding) { + LongBounds round(Rounding rounding) { // Extended bounds shouldn't be effected by the offset Rounding effectiveRounding = rounding.withoutOffset(); - return new ExtendedBounds( + return new LongBounds( min != null ? effectiveRounding.round(min) : null, max != null ? effectiveRounding.round(max) : null); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(EXTENDED_BOUNDS_FIELD.getPreferredName()); if (min != null) { builder.field(MIN_FIELD.getPreferredName(), min); } else { @@ -186,7 +190,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } else { builder.field(MAX_FIELD.getPreferredName(), maxAsStr); } - builder.endObject(); return builder; } @@ -203,7 +206,7 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) { return false; } - ExtendedBounds other = (ExtendedBounds) obj; + LongBounds other = (LongBounds) obj; return Objects.equals(min, other.min) && Objects.equals(max, other.max) && Objects.equals(minAsStr, other.minAsStr) @@ -218,6 +221,16 @@ public Long getMax() { return max; } + public boolean contain(long value) { + if (max != null && value >= max) { + return false; + } + if (min != null && value < min) { + return false; + } + return true; + } + @Override public String toString() { StringBuilder b = new StringBuilder(); @@ -233,7 +246,7 @@ public String toString() { } b.append("--"); if (max != null) { - b.append(min); + b.append(max); if (maxAsStr != null) { b.append('(').append(maxAsStr).append(')'); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java index 83ed41ae2cdef..a22cc5495bff7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/NumericHistogramAggregator.java @@ -54,6 +54,7 @@ public NumericHistogramAggregator( long minDocCount, double minBound, double maxBound, + DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, Aggregator parent, @@ -70,6 +71,7 @@ public NumericHistogramAggregator( minDocCount, minBound, maxBound, + hardBounds, valuesSourceConfig.format(), context, parent, @@ -110,12 +112,14 @@ public void collect(int doc, long owningBucketOrd) throws IOException { if (key == previousKey) { continue; } - long bucketOrd = bucketOrds.add(owningBucketOrd, Double.doubleToLongBits(key)); - if (bucketOrd < 0) { // already seen - bucketOrd = -1 - bucketOrd; - collectExistingBucket(sub, doc, bucketOrd); - } else { - collectBucket(sub, doc, bucketOrd); + if (hardBounds == null || hardBounds.contain(key)) { + long bucketOrd = bucketOrds.add(owningBucketOrd, Double.doubleToLongBits(key)); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } } previousKey = key; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java index c89ce917b4c16..801def759ea6e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/RangeHistogramAggregator.java @@ -51,6 +51,7 @@ public RangeHistogramAggregator( long minDocCount, double minBound, double maxBound, + DoubleBounds hardBounds, ValuesSourceConfig valuesSourceConfig, SearchContext context, Aggregator parent, @@ -67,6 +68,7 @@ public RangeHistogramAggregator( minDocCount, minBound, maxBound, + hardBounds, valuesSourceConfig.format(), context, parent, @@ -108,9 +110,13 @@ public void collect(int doc, long owningBucketOrd) throws IOException { // The encoding should ensure that this assert is always true. assert from >= previousFrom : "Start of range not >= previous start"; final Double to = rangeType.doubleValue(range.getTo()); - final double startKey = Math.floor((from - offset) / interval); - final double endKey = Math.floor((to - offset) / interval); - for (double key = startKey > previousKey ? startKey : previousKey; key <= endKey; key++) { + final double effectiveFrom = (hardBounds != null && hardBounds.getMin() != null) ? + Double.max(from, hardBounds.getMin()) : from; + final double effectiveTo = (hardBounds != null && hardBounds.getMax() != null) ? + Double.min(to, hardBounds.getMax()) : to; + final double startKey = Math.floor((effectiveFrom - offset) / interval); + final double endKey = Math.floor((effectiveTo - offset) / interval); + for (double key = Math.max(startKey, previousKey); key <= endKey; key++) { if (key == previousKey) { continue; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java index 3e32da0709eb5..63247d70b65f4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregator.java @@ -166,7 +166,7 @@ private class MergeBucketsPhase extends CollectionPhase{ public DoubleArray clusterSizes; // clusterSizes != bucketDocCounts when clusters are in the middle of a merge public int numClusters; - private int avgBucketDistance; + private double avgBucketDistance; MergeBucketsPhase(DoubleArray buffer, int bufferSize) { // Cluster the documents to reduce the number of buckets @@ -174,15 +174,7 @@ private class MergeBucketsPhase extends CollectionPhase{ bucketBufferedDocs(buffer, bufferSize, shardSize * 3 / 4); if(bufferSize > 1) { - // Calculate the average distance between buckets - // Subsequent documents will be compared with this value to determine if they should be collected into - // an existing bucket or into a new bucket - // This can be done in a single linear scan because buckets are sorted by centroid - int sum = 0; - for (int i = 0; i < numClusters - 1; i++) { - sum += clusterCentroids.get(i + 1) - clusterCentroids.get(i); - } - avgBucketDistance = (sum / (numClusters - 1)); + updateAvgBucketDistance(); } } @@ -194,11 +186,9 @@ private class ClusterSorter extends InPlaceMergeSorter { final DoubleArray values; final long[] indexes; - int length; ClusterSorter(DoubleArray values, int length){ this.values = values; - this.length = length; this.indexes = new long[length]; for(int i = 0; i < indexes.length; i++){ @@ -284,7 +274,7 @@ private void bucketBufferedDocs(final DoubleArray buffer, final int bufferSize, @Override public CollectionPhase collectValue(LeafBucketCollector sub, int doc, double val) throws IOException{ int bucketOrd = getNearestBucket(val); - double distance = Math.abs(clusterCentroids.get(bucketOrd)- val); + double distance = Math.abs(clusterCentroids.get(bucketOrd) - val); if(bucketOrd == -1 || distance > (2 * avgBucketDistance) && numClusters < shardSize) { // Make a new bucket since the document is distant from all existing buckets // TODO: (maybe) Create a new bucket for all distant docs and merge down to shardSize buckets at end @@ -293,17 +283,31 @@ public CollectionPhase collectValue(LeafBucketCollector sub, int doc, double val collectBucket(sub, doc, numClusters - 1); if(val > clusterCentroids.get(bucketOrd)){ - // Insert just ahead of bucketOrd so that the array remains sorted + /* + * If the new value is bigger than the nearest bucket then insert + * just ahead of bucketOrd so that the array remains sorted. + */ bucketOrd += 1; } moveLastCluster(bucketOrd); + // We've added a new bucket so update the average distance between the buckets + updateAvgBucketDistance(); } else { addToCluster(bucketOrd, val); collectExistingBucket(sub, doc, bucketOrd); + if (bucketOrd == 0 || bucketOrd == numClusters - 1) { + // Only update average distance if the centroid of one of the end buckets is modifed. + updateAvgBucketDistance(); + } } return this; } + private void updateAvgBucketDistance() { + // Centroids are sorted so the average distance is the difference between the first and last. + avgBucketDistance = (clusterCentroids.get(numClusters - 1) - clusterCentroids.get(0)) / (numClusters - 1); + } + /** * Creates a new cluster with value and appends it to the cluster arrays */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java index 20e354d01575e..0e454db0d6566 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MaxAggregator.java @@ -44,8 +44,6 @@ import java.util.Map; import java.util.function.Function; -import static org.elasticsearch.search.aggregations.metrics.MinAggregator.getPointReaderOrNull; - class MaxAggregator extends NumericMetricsAggregator.SingleValue { final ValuesSource.Numeric valuesSource; @@ -68,7 +66,7 @@ class MaxAggregator extends NumericMetricsAggregator.SingleValue { maxes.fill(0, maxes.size(), Double.NEGATIVE_INFINITY); } this.formatter = config.format(); - this.pointConverter = getPointReaderOrNull(context, parent, config); + this.pointConverter = pointReaderIfAvailable(config); if (pointConverter != null) { pointField = config.fieldContext().field(); } else { @@ -96,7 +94,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, Number segMax = findLeafMaxValue(ctx.reader(), pointField, pointConverter); if (segMax != null) { /* - * There is no parent aggregator (see {@link MinAggregator#getPointReaderOrNull} + * There is no parent aggregator (see {@link AggregatorBase#getPointReaderOrNull} * so the ordinal for the bucket is always 0. */ assert maxes.size() == 1; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java index 34b905ec5fc57..1fa12e9719882 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MinAggregator.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.util.Bits; import org.elasticsearch.common.lease.Releasables; @@ -30,9 +29,6 @@ import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.Aggregator; @@ -71,7 +67,7 @@ class MinAggregator extends NumericMetricsAggregator.SingleValue { mins.fill(0, mins.size(), Double.POSITIVE_INFINITY); } this.format = config.format(); - this.pointConverter = getPointReaderOrNull(context, parent, config); + this.pointConverter = pointReaderIfAvailable(config); if (pointConverter != null) { pointField = config.fieldContext().field(); } else { @@ -159,40 +155,6 @@ public void doClose() { } - /** - * Returns a converter for point values if early termination is applicable to - * the context or null otherwise. - * - * @param context The {@link SearchContext} of the aggregation. - * @param parent The parent aggregator. - * @param config The config for the values source metric. - */ - static Function getPointReaderOrNull(SearchContext context, Aggregator parent, - ValuesSourceConfig config) { - if (context.query() != null && - context.query().getClass() != MatchAllDocsQuery.class) { - return null; - } - if (parent != null) { - return null; - } - if (config.fieldContext() != null && config.script() == null && config.missing() == null) { - MappedFieldType fieldType = config.fieldContext().fieldType(); - if (fieldType == null || fieldType.isSearchable() == false) { - return null; - } - Function converter = null; - if (fieldType instanceof NumberFieldMapper.NumberFieldType) { - converter = ((NumberFieldMapper.NumberFieldType) fieldType)::parsePoint; - } else if (fieldType.getClass() == DateFieldMapper.DateFieldType.class) { - DateFieldMapper.DateFieldType dft = (DateFieldMapper.DateFieldType) fieldType; - converter = dft.resolution()::parsePointAsMillis; - } - return converter; - } - return null; - } - /** * Returns the minimum value indexed in the fieldName field or null * if the value cannot be inferred from the indexed {@link PointValues}. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index a4ae01f372d4b..37a1d53213311 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.DocValueFormat; import java.time.ZoneId; +import java.util.function.Function; import java.util.function.LongSupplier; /** @@ -371,4 +372,36 @@ public ValuesSource getValuesSource() { public boolean hasGlobalOrdinals() { return valuesSource.hasGlobalOrdinals(); } + + /** + * This method is used when an aggregation can optimize by using the indexed data instead of the doc values. We check to see if the + * indexed data will match the values source output (meaning there isn't a script or a missing value, since both could modify the + * value at read time). If the settings allow for it, we then ask the {@link ValuesSourceType} to build the actual point reader + * based on the field type. This allows for a point of extensibility in plugins. + * + * @return null if we cannot apply the optimization, otherwise the point reader function. + */ + @Nullable + public Function getPointReaderOrNull() { + MappedFieldType fieldType = fieldType(); + if (fieldType != null && script() == null && missing() == null) { + return fieldType.pointReaderIfPossible(); + } + return null; + } + + /** + * Returns a human readable description of this values source, for use in error messages and similar. + */ + public String getDescription() { + if (script != null) { + return "Script yielding [" + (scriptValueType != null ? scriptValueType.getPreferredName() : "unknown type") + "]"; + } + + MappedFieldType fieldType = fieldType(); + if (fieldType != null) { + return "Field [" + fieldType.name() + "] of type [" + fieldType.typeName() + "]"; + } + return "unmapped field"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java index ea2a3d6bf0d92..b28d6e4f6147d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistry.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.search.aggregations.support; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.AggregationExecutionException; @@ -131,11 +130,10 @@ public AggregatorSupplier getAggregator(ValuesSourceConfig valuesSourceConfig, S aggregatorRegistry.get(aggregationName) ); if (supplier == null) { - // TODO: push building the description into ValuesSourceConfig - MappedFieldType fieldType = valuesSourceConfig.fieldContext().fieldType(); - String fieldDescription = fieldType.typeName(); - throw new IllegalArgumentException("Field [" + fieldType.name() + "] of type [" + fieldDescription + - "] is not supported for aggregation [" + aggregationName + "]"); } + throw new IllegalArgumentException( + valuesSourceConfig.getDescription() + " is not supported for aggregation [" + aggregationName + "]" + ); + } return supplier; } throw new AggregationExecutionException("Unregistered Aggregation [" + aggregationName + "]"); diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index bc65e17779581..5079498725bf7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -53,6 +53,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequestDeduplicator; @@ -250,8 +251,10 @@ private void startNewShards(SnapshotsInProgress.Entry entry, Map() { @Override diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index f288b3c4b7f5e..ee713b6668253 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -69,6 +69,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.Index; @@ -89,12 +90,16 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Deque; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -103,6 +108,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Collections.emptySet; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.cluster.SnapshotsInProgress.completed; @@ -113,6 +119,8 @@ */ public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateApplier { + public static final Version FULL_CONCURRENCY_VERSION = Version.V_8_0_0; + public static final Version SHARD_GEN_IN_REPO_DATA_VERSION = Version.V_7_6_0; public static final Version INDEX_GEN_IN_REPO_DATA_VERSION = Version.V_8_0_0; @@ -134,6 +142,14 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final Map>>> snapshotCompletionListeners = new ConcurrentHashMap<>(); + /** + * Listeners for snapshot deletion keyed by delete uuid as returned from {@link SnapshotDeletionsInProgress.Entry#uuid()} + */ + private final Map>> snapshotDeletionListeners = new HashMap<>(); + + //Set of repositories currently running either a snapshot finalization or a snapshot delete. + private final Set currentlyFinalizing = Collections.synchronizedSet(new HashSet<>()); + // Set of snapshots that are currently being ended by this node private final Set endingSnapshots = Collections.synchronizedSet(new HashSet<>()); @@ -142,6 +158,18 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus private final TransportService transportService; + private final OngoingRepositoryOperations repositoryOperations = new OngoingRepositoryOperations(); + + /** + * Setting that specifies the maximum number of allowed concurrent snapshot create and delete operations in the + * cluster state. The number of concurrent operations in a cluster state is defined as the sum of the sizes of + * {@link SnapshotsInProgress#entries()} and {@link SnapshotDeletionsInProgress#getEntries()}. + */ + public static final Setting MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING = + Setting.intSetting("snapshot.max_concurrent_operations", 1000, 1, Setting.Property.NodeScope, Setting.Property.Dynamic); + + private volatile int maxConcurrentOperations; + public SnapshotsService(Settings settings, ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, RepositoriesService repositoriesService, TransportService transportService, ActionFilters actionFilters) { this.clusterService = clusterService; @@ -156,6 +184,9 @@ public SnapshotsService(Settings settings, ClusterService clusterService, IndexN if (DiscoveryNode.isMasterNode(settings)) { // addLowPriorityApplier to make sure that Repository will be created before snapshot clusterService.addLowPriorityApplier(this); + maxConcurrentOperations = MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING.get(settings); + clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_CONCURRENT_SNAPSHOT_OPERATIONS_SETTING, + i -> maxConcurrentOperations = i); } } @@ -204,26 +235,36 @@ public ClusterState execute(ClusterState currentState) { throw new InvalidSnapshotNameException( repository.getMetadata().name(), snapshotName, "snapshot with the same name already exists"); } + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final List runningSnapshots = snapshots.entries(); + if (runningSnapshots.stream().anyMatch(s -> { + final Snapshot running = s.snapshot(); + return running.getRepository().equals(repositoryName) && running.getSnapshotId().getName().equals(snapshotName); + })) { + throw new InvalidSnapshotNameException( + repository.getMetadata().name(), snapshotName, "snapshot with the same name is already in-progress"); + } validate(repositoryName, snapshotName, currentState); + final boolean concurrentOperationsAllowed = currentState.nodes().getMinNodeVersion().onOrAfter(FULL_CONCURRENCY_VERSION); final SnapshotDeletionsInProgress deletionsInProgress = - currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); - if (deletionsInProgress.hasDeletionsInProgress()) { + currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + if (deletionsInProgress.hasDeletionsInProgress() && concurrentOperationsAllowed == false) { throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a snapshot deletion is in-progress in [" + deletionsInProgress + "]"); } final RepositoryCleanupInProgress repositoryCleanupInProgress = - currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY); + currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY); if (repositoryCleanupInProgress.hasCleanupInProgress()) { throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]"); } - final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); // Fail if there are any concurrently running snapshots. The only exception to this being a snapshot in INIT state from a // previous master that we can simply ignore and remove from the cluster state because we would clean it up from the // cluster state anyway in #applyClusterState. - if (snapshots.entries().stream().anyMatch(entry -> entry.state() != State.INIT)) { + if (concurrentOperationsAllowed == false && runningSnapshots.stream().anyMatch(entry -> entry.state() != State.INIT)) { throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, " a snapshot is already running"); } + ensureBelowConcurrencyLimit(repositoryName, snapshotName, snapshots, deletionsInProgress); // Store newSnapshot here to be processed in clusterStateProcessed List indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request)); @@ -232,10 +273,13 @@ public ClusterState execute(ClusterState currentState) { logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices); - final List indexIds = repositoryData.resolveNewIndices(indices); + final List indexIds = repositoryData.resolveNewIndices( + indices, runningSnapshots.stream().filter(entry -> entry.repository().equals(repositoryName)) + .flatMap(entry -> entry.indices().stream()).distinct() + .collect(Collectors.toMap(IndexId::getName, Function.identity()))); final Version version = minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null); - ImmutableOpenMap shards = - shards(currentState, indexIds, useShardGenerations(version), repositoryData); + ImmutableOpenMap shards = shards(snapshots, deletionsInProgress, currentState.metadata(), + currentState.routingTable(), indexIds, useShardGenerations(version), repositoryData, repositoryName); if (request.partial() == false) { Set missing = new HashSet<>(); for (ObjectObjectCursor entry : shards) { @@ -249,11 +293,13 @@ public ClusterState execute(ClusterState currentState) { } } newEntry = new SnapshotsInProgress.Entry( - new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), - State.STARTED, indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, - null, userMeta, version); + new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), + completed(shards.values()) ? State.SUCCESS : State.STARTED, indexIds, dataStreams, + threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, null, userMeta, version); + final List newEntries = new ArrayList<>(runningSnapshots); + newEntries.add(newEntry); return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, - SnapshotsInProgress.of(List.of(newEntry))).build(); + SnapshotsInProgress.of(List.copyOf(newEntries))).build(); } @Override @@ -268,8 +314,8 @@ public void clusterStateProcessed(String source, ClusterState oldState, final Cl logger.info("snapshot [{}] started", snapshot); listener.onResponse(snapshot); } finally { - if (newEntry.state().completed() || newEntry.shards().isEmpty()) { - endSnapshot(newEntry, newState.metadata()); + if (newEntry.state().completed()) { + endSnapshot(newEntry, newState.metadata(), repositoryData); } } } @@ -281,6 +327,17 @@ public TimeValue timeout() { }, "create_snapshot [" + snapshotName + ']', listener::onFailure); } + private void ensureBelowConcurrencyLimit(String repository, String name, SnapshotsInProgress snapshotsInProgress, + SnapshotDeletionsInProgress deletionsInProgress) { + final int inProgressOperations = snapshotsInProgress.entries().size() + deletionsInProgress.getEntries().size(); + final int maxOps = maxConcurrentOperations; + if (inProgressOperations >= maxOps) { + throw new ConcurrentSnapshotExecutionException(repository, name, + "Cannot start another operation, already running [" + inProgressOperations + "] operations and the current" + + " limit for concurrent snapshot operations is set to [" + maxOps + "]"); + } + } + /** * Validates snapshot request * @@ -381,9 +438,9 @@ private static Metadata metadataForSnapshot(SnapshotsInProgress.Entry snapshot, * @param snapshots list of snapshots that will be used as a filter, empty list means no snapshots are filtered * @return list of metadata for currently running snapshots */ - public static List currentSnapshots(SnapshotsInProgress snapshotsInProgress, String repository, + public static List currentSnapshots(@Nullable SnapshotsInProgress snapshotsInProgress, String repository, List snapshots) { - if (snapshotsInProgress.entries().isEmpty()) { + if (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) { return Collections.emptyList(); } if ("_all".equals(repository)) { @@ -431,16 +488,10 @@ public void applyClusterState(ClusterChangedEvent event) { try { if (event.localNodeMaster()) { // We don't remove old master when master flips anymore. So, we need to check for change in master - final SnapshotsInProgress snapshotsInProgress = - event.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); final boolean newMaster = event.previousState().nodes().isLocalNodeElectedMaster() == false; - if (snapshotsInProgress.entries().isEmpty() == false) { - processExternalChanges(newMaster || removedNodesCleanupNeeded(snapshotsInProgress, event.nodesDelta().removedNodes()), + processExternalChanges(newMaster || removedNodesCleanupNeeded(snapshotsInProgress, event.nodesDelta().removedNodes()), event.routingTableChanged() && waitingShardsStartedOrUnassigned(snapshotsInProgress, event)); - } - if (newMaster) { - finalizeSnapshotDeletionFromPreviousMaster(event.state()); - } } else if (snapshotCompletionListeners.isEmpty() == false) { // We have snapshot listeners but are not the master any more. Fail all waiting listeners except for those that already // have their snapshots finalizing (those that are already finalizing will fail on their own from to update the cluster @@ -456,11 +507,12 @@ public void applyClusterState(ClusterChangedEvent event) { logger.warn("Failed to update snapshot state ", e); } assert assertConsistentWithClusterState(event.state()); + assert assertNoDanglingSnapshots(event.state()); } private boolean assertConsistentWithClusterState(ClusterState state) { - final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE); - if (snapshotsInProgress != null && snapshotsInProgress.entries().isEmpty() == false) { + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + if (snapshotsInProgress.entries().isEmpty() == false) { synchronized (endingSnapshots) { final Set runningSnapshots = Stream.concat( snapshotsInProgress.entries().stream().map(SnapshotsInProgress.Entry::snapshot), @@ -471,29 +523,44 @@ private boolean assertConsistentWithClusterState(ClusterState state) { + snapshotListenerKeys + " but running snapshots are " + runningSnapshots; } } + final SnapshotDeletionsInProgress snapshotDeletionsInProgress = + state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + if (snapshotDeletionsInProgress.hasDeletionsInProgress()) { + synchronized (repositoryOperations.runningDeletions) { + final Set runningDeletes = Stream.concat( + snapshotDeletionsInProgress.getEntries().stream().map(SnapshotDeletionsInProgress.Entry::uuid), + repositoryOperations.runningDeletions.stream()) + .collect(Collectors.toSet()); + final Set deleteListenerKeys = snapshotDeletionListeners.keySet(); + assert runningDeletes.containsAll(deleteListenerKeys) : "Saw deletions listeners for unknown uuids in " + + deleteListenerKeys + " but running deletes are " + runningDeletes; + } + } return true; } - /** - * Finalizes a snapshot deletion in progress if the current node is the master but it - * was not master in the previous cluster state and there is still a lingering snapshot - * deletion in progress in the cluster state. This means that the old master failed - * before it could clean up an in-progress snapshot deletion. We attempt to delete the - * snapshot files and remove the deletion from the cluster state. It is possible that the - * old master was in a state of long GC and then it resumes and tries to delete the snapshot - * that has already been deleted by the current master. This is acceptable however, since - * the old master's snapshot deletion will just respond with an error but in actuality, the - * snapshot was deleted and a call to GET snapshots would reveal that the snapshot no longer exists. - */ - private void finalizeSnapshotDeletionFromPreviousMaster(ClusterState state) { - SnapshotDeletionsInProgress deletionsInProgress = - state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); - if (deletionsInProgress.hasDeletionsInProgress()) { - assert deletionsInProgress.getEntries().size() == 1 : "only one in-progress deletion allowed per cluster"; - SnapshotDeletionsInProgress.Entry entry = deletionsInProgress.getEntries().get(0); - deleteSnapshotsFromRepository(entry.repository(), entry.getSnapshots(), null, entry.repositoryStateId(), - state.nodes().getMinNodeVersion()); + // Assert that there are no snapshots that have a shard that is waiting to be assigned even though the cluster state would allow for it + // to be assigned + private static boolean assertNoDanglingSnapshots(ClusterState state) { + final SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final SnapshotDeletionsInProgress snapshotDeletionsInProgress = + state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + final Set reposWithRunningDelete = snapshotDeletionsInProgress.getEntries().stream() + .filter(entry -> entry.state() == SnapshotDeletionsInProgress.State.STARTED) + .map(SnapshotDeletionsInProgress.Entry::repository).collect(Collectors.toSet()); + final Set reposSeen = new HashSet<>(); + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { + if (reposSeen.add(entry.repository())) { + for (ObjectCursor value : entry.shards().values()) { + if (value.value.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED)) { + assert reposWithRunningDelete.contains(entry.repository()) + : "Found shard snapshot waiting to be assigned in [" + entry + + "] but it is not blocked by any running delete"; + } + } + } } + return true; } /** @@ -515,11 +582,12 @@ private void processExternalChanges(boolean changedNodes, boolean startShards) { private final Collection finishedSnapshots = new ArrayList<>(); + private final Collection deletionsToExecute = new ArrayList<>(); + @Override public ClusterState execute(ClusterState currentState) { RoutingTable routingTable = currentState.routingTable(); - final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); - assert snapshots != null : "We only submit this kind of update if there have been snapshots before"; + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); DiscoveryNodes nodes = currentState.nodes(); boolean changed = false; final EnumSet statesToUpdate; @@ -531,11 +599,17 @@ public ClusterState execute(ClusterState currentState) { // We are reacting to shards that started only so which only affects the individual shard states of started snapshots statesToUpdate = EnumSet.of(State.STARTED); } - ArrayList entries = new ArrayList<>(); + ArrayList updatedSnapshotEntries = new ArrayList<>(); + + // We keep a cache of shards that failed in this map. If we fail a shardId for a given repository because of + // a node leaving or shard becoming unassigned for one snapshot, we will also fail it for all subsequent enqueued snapshots + // for the same repository + final Map> knownFailures = new HashMap<>(); + for (final SnapshotsInProgress.Entry snapshot : snapshots.entries()) { if (statesToUpdate.contains(snapshot.state())) { - ImmutableOpenMap shards = - processWaitingShardsAndRemovedNodes(snapshot.shards(), routingTable, nodes); + ImmutableOpenMap shards = processWaitingShardsAndRemovedNodes(snapshot.shards(), + routingTable, nodes, knownFailures.computeIfAbsent(snapshot.repository(), k -> new HashMap<>())); if (shards != null) { final SnapshotsInProgress.Entry updatedSnapshot; changed = true; @@ -545,9 +619,9 @@ public ClusterState execute(ClusterState currentState) { } else { updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, shards); } - entries.add(updatedSnapshot); + updatedSnapshotEntries.add(updatedSnapshot); } else { - entries.add(snapshot); + updatedSnapshotEntries.add(snapshot); } } else if (snapshot.repositoryStateId() == RepositoryData.UNKNOWN_REPO_GEN) { // BwC path, older versions could create entries with unknown repo GEN in INIT or ABORTED state that did not yet @@ -559,14 +633,20 @@ public ClusterState execute(ClusterState currentState) { if (snapshot.state().completed() || completed(snapshot.shards().values())) { finishedSnapshots.add(snapshot); } - entries.add(snapshot); + updatedSnapshotEntries.add(snapshot); } } - if (changed) { - return ClusterState.builder(currentState) - .putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(entries)).build(); + final ClusterState res = readyDeletions( + changed ? ClusterState.builder(currentState).putCustom( + SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(updatedSnapshotEntries))).build() : + currentState).v1(); + for (SnapshotDeletionsInProgress.Entry delete + : res.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY).getEntries()) { + if (delete.state() == SnapshotDeletionsInProgress.State.STARTED) { + deletionsToExecute.add(delete); + } } - return currentState; + return res; } @Override @@ -577,19 +657,53 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - finishedSnapshots.forEach(entry -> endSnapshot(entry, newState.metadata())); + final SnapshotDeletionsInProgress snapshotDeletionsInProgress = + newState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + if (finishedSnapshots.isEmpty() == false) { + // If we found snapshots that should be finalized as a result of the CS update we try to initiate finalization for them + // unless there is an executing snapshot delete already. If there is an executing snapshot delete we don't have to + // enqueue the snapshot finalizations here because the ongoing delete will take care of that when removing the delete + // from the cluster state + final Set reposWithRunningDeletes = snapshotDeletionsInProgress.getEntries().stream() + .filter(entry -> entry.state() == SnapshotDeletionsInProgress.State.STARTED) + .map(SnapshotDeletionsInProgress.Entry::repository).collect(Collectors.toSet()); + for (SnapshotsInProgress.Entry entry : finishedSnapshots) { + if (reposWithRunningDeletes.contains(entry.repository()) == false) { + endSnapshot(entry, newState.metadata(), null); + } + } + } + // run newly ready deletes + for (SnapshotDeletionsInProgress.Entry entry : deletionsToExecute) { + if (tryEnterRepoLoop(entry.repository())) { + deleteSnapshotsFromRepository(entry, newState.nodes().getMinNodeVersion()); + } + } } }); } private static ImmutableOpenMap processWaitingShardsAndRemovedNodes( - ImmutableOpenMap snapshotShards, RoutingTable routingTable, DiscoveryNodes nodes) { + ImmutableOpenMap snapshotShards, RoutingTable routingTable, DiscoveryNodes nodes, + Map knownFailures) { boolean snapshotChanged = false; ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder(); for (ObjectObjectCursor shardEntry : snapshotShards) { ShardSnapshotStatus shardStatus = shardEntry.value; ShardId shardId = shardEntry.key; - if (shardStatus.state() == ShardState.WAITING) { + if (shardStatus.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED)) { + // this shard snapshot is waiting for a previous snapshot to finish execution for this shard + final ShardSnapshotStatus knownFailure = knownFailures.get(shardId); + if (knownFailure == null) { + // if no failure is known for the shard we keep waiting + shards.put(shardId, shardStatus); + } else { + // If a failure is known for an execution we waited on for this shard then we fail with the same exception here + // as well + snapshotChanged = true; + shards.put(shardId, knownFailure); + } + } else if (shardStatus.state() == ShardState.WAITING) { IndexRoutingTable indexShardRoutingTable = routingTable.index(shardId.getIndex()); if (indexShardRoutingTable != null) { IndexShardRoutingTable shardRouting = indexShardRoutingTable.shard(shardId.id()); @@ -599,7 +713,7 @@ private static ImmutableOpenMap processWaitingShar snapshotChanged = true; logger.trace("starting shard that we were waiting for [{}] on node [{}]", shardId, shardStatus.nodeId()); shards.put(shardId, - new ShardSnapshotStatus(shardRouting.primaryShard().currentNodeId(), shardStatus.generation())); + new ShardSnapshotStatus(shardRouting.primaryShard().currentNodeId(), shardStatus.generation())); continue; } else if (shardRouting.primaryShard().initializing() || shardRouting.primaryShard().relocating()) { // Shard that we were waiting for hasn't started yet or still relocating - will continue to wait @@ -611,8 +725,10 @@ private static ImmutableOpenMap processWaitingShar // Shard that we were waiting for went into unassigned state or disappeared - giving up snapshotChanged = true; logger.warn("failing snapshot of shard [{}] on unassigned shard [{}]", shardId, shardStatus.nodeId()); - shards.put(shardId, new ShardSnapshotStatus( - shardStatus.nodeId(), ShardState.FAILED, "shard is unassigned", shardStatus.generation())); + final ShardSnapshotStatus failedState = new ShardSnapshotStatus(shardStatus.nodeId(), ShardState.FAILED, + "shard is unassigned", shardStatus.generation()); + shards.put(shardId, failedState); + knownFailures.put(shardId, failedState); } else if (shardStatus.state().completed() == false && shardStatus.nodeId() != null) { if (nodes.nodeExists(shardStatus.nodeId())) { shards.put(shardId, shardStatus); @@ -621,8 +737,10 @@ private static ImmutableOpenMap processWaitingShar snapshotChanged = true; logger.warn("failing snapshot of shard [{}] on closed node [{}]", shardId, shardStatus.nodeId()); - shards.put(shardId, - new ShardSnapshotStatus(shardStatus.nodeId(), ShardState.FAILED, "node shutdown", shardStatus.generation())); + final ShardSnapshotStatus failedState = new ShardSnapshotStatus(shardStatus.nodeId(), ShardState.FAILED, + "node shutdown", shardStatus.generation()); + shards.put(shardId, failedState); + knownFailures.put(shardId, failedState); } } else { shards.put(shardId, shardStatus); @@ -685,19 +803,63 @@ private static boolean removedNodesCleanupNeeded(SnapshotsInProgress snapshotsIn } /** - * Finalizes the shard in repository and then removes it from cluster state - *

- * This is non-blocking method that runs on a thread from SNAPSHOT thread pool + * Finalizes the snapshot in the repository. * * @param entry snapshot */ - private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata) { - if (endingSnapshots.add(entry.snapshot()) == false) { - return; + private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata, @Nullable RepositoryData repositoryData) { + final boolean newFinalization = endingSnapshots.add(entry.snapshot()); + final String repoName = entry.repository(); + if (tryEnterRepoLoop(repoName)) { + if (repositoryData == null) { + repositoriesService.repository(repoName).getRepositoryData(new ActionListener<>() { + @Override + public void onResponse(RepositoryData repositoryData) { + finalizeSnapshotEntry(entry, metadata, repositoryData); + } + + @Override + public void onFailure(Exception e) { + clusterService.submitStateUpdateTask("fail repo tasks for [" + repoName + "]", + new FailPendingRepoTasksTask(repoName, e)); + } + }); + } else { + finalizeSnapshotEntry(entry, metadata, repositoryData); + } + } else { + if (newFinalization) { + repositoryOperations.addFinalization(entry, metadata); + } } - final Snapshot snapshot = entry.snapshot(); + } + + /** + * Try starting to run a snapshot finalization or snapshot delete for the given repository. If this method returns + * {@code true} then snapshot finalizations and deletions for the repo may be executed. Once no more operations are + * ready for the repository {@link #leaveRepoLoop(String)} should be invoked so that a subsequent state change that + * causes another operation to become ready can execute. + * + * @return true if a finalization or snapshot delete may be started at this point + */ + private boolean tryEnterRepoLoop(String repository) { + return currentlyFinalizing.add(repository); + } + + /** + * Stop polling for ready snapshot finalizations or deletes in state {@link SnapshotDeletionsInProgress.State#STARTED} to execute + * for the given repository. + */ + private void leaveRepoLoop(String repository) { + final boolean removed = currentlyFinalizing.remove(repository); + assert removed; + } + + private void finalizeSnapshotEntry(SnapshotsInProgress.Entry entry, Metadata metadata, RepositoryData repositoryData) { + assert currentlyFinalizing.contains(entry.repository()); try { final String failure = entry.failure(); + final Snapshot snapshot = entry.snapshot(); logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure); ArrayList shardFailures = new ArrayList<>(); for (ObjectObjectCursor shardStatus : entry.shards()) { @@ -713,6 +875,7 @@ private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata) { } } final ShardGenerations shardGenerations = buildGenerations(entry, metadata); + final String repository = snapshot.getRepository(); final SnapshotInfo snapshotInfo = new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), entry.dataStreams(), @@ -721,31 +884,36 @@ private void endSnapshot(SnapshotsInProgress.Entry entry, Metadata metadata) { entry.includeGlobalState(), entry.userMetadata()); repositoriesService.repository(snapshot.getRepository()).finalizeSnapshot( shardGenerations, - entry.repositoryStateId(), + repositoryData.getGenId(), metadataForSnapshot(entry, metadata), snapshotInfo, entry.version(), state -> stateWithoutSnapshot(state, snapshot), ActionListener.wrap(newRepoData -> { - final List>> completionListeners = - snapshotCompletionListeners.remove(snapshot); - if (completionListeners != null) { - final Tuple result = Tuple.tuple(newRepoData, snapshotInfo); - try { - ActionListener.onResponse(completionListeners, result); - } catch (Exception e) { - logger.warn("Failed to notify listeners", e); - } - } endingSnapshots.remove(snapshot); + completeListenersIgnoringException( + snapshotCompletionListeners.remove(snapshot), Tuple.tuple(newRepoData, snapshotInfo)); logger.info("snapshot [{}] completed with state [{}]", snapshot, snapshotInfo.state()); - }, e -> handleFinalizationFailure(e, entry))); + runNextQueuedOperation(newRepoData, repository, true); + }, e -> handleFinalizationFailure(e, entry, repositoryData))); } catch (Exception e) { - handleFinalizationFailure(e, entry); + assert false : new AssertionError(e); + handleFinalizationFailure(e, entry, repositoryData); } } - private void handleFinalizationFailure(Exception e, SnapshotsInProgress.Entry entry) { + /** + * Handles failure to finalize a snapshot. If the exception indicates that this node was unable to publish a cluster state and stopped + * being the master node, then fail all snapshot create and delete listeners executing on this node by delegating to + * {@link #failAllListenersOnMasterFailOver}. Otherwise, i.e. as a result of failing to write to the snapshot repository for some + * reason, remove the snapshot's {@link SnapshotsInProgress.Entry} from the cluster state and move on with other queued snapshot + * operations if there are any. + * + * @param e exception encountered + * @param entry snapshot entry that failed to finalize + * @param repositoryData current repository data for the snapshot's repository + */ + private void handleFinalizationFailure(Exception e, SnapshotsInProgress.Entry entry, RepositoryData repositoryData) { Snapshot snapshot = entry.snapshot(); if (ExceptionsHelper.unwrap(e, NotMasterException.class, FailedToCommitClusterStateException.class) != null) { // Failure due to not being master any more, don't try to remove snapshot from cluster state the next master @@ -753,15 +921,127 @@ private void handleFinalizationFailure(Exception e, SnapshotsInProgress.Entry en logger.debug(() -> new ParameterizedMessage( "[{}] failed to update cluster state during snapshot finalization", snapshot), e); failSnapshotCompletionListeners(snapshot, - new SnapshotException(snapshot, "Failed to update cluster state during snapshot finalization", e)); + new SnapshotException(snapshot, "Failed to update cluster state during snapshot finalization", e)); + failAllListenersOnMasterFailOver(e); } else { logger.warn(() -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e); - removeSnapshotFromClusterState(snapshot, e); + removeFailedSnapshotFromClusterState(snapshot, e, repositoryData); + } + } + + /** + * Run the next queued up repository operation for the given repository name. + * + * @param repositoryData current repository data + * @param repository repository name + * @param attemptDelete whether to try and run delete operations that are ready in the cluster state if no + * snapshot create operations remain to execute + */ + private void runNextQueuedOperation(RepositoryData repositoryData, String repository, boolean attemptDelete) { + assert currentlyFinalizing.contains(repository); + final Tuple nextFinalization = repositoryOperations.pollFinalization(repository); + if (nextFinalization == null) { + if (attemptDelete) { + runReadyDeletions(repositoryData, repository); + } else { + leaveRepoLoop(repository); + } + } else { + logger.trace("Moving on to finalizing next snapshot [{}]", nextFinalization); + finalizeSnapshotEntry(nextFinalization.v1(), nextFinalization.v2(), repositoryData); + } + } + + /** + * Runs a cluster state update that checks whether we have outstanding snapshot deletions that can be executed and executes them. + * + * TODO: optimize this to execute in a single CS update together with finalizing the latest snapshot + */ + private void runReadyDeletions(RepositoryData repositoryData, String repository) { + clusterService.submitStateUpdateTask("Run ready deletions", new ClusterStateUpdateTask() { + + private SnapshotDeletionsInProgress.Entry deletionToRun; + + @Override + public ClusterState execute(ClusterState currentState) { + assert readyDeletions(currentState).v1() == currentState : + "Deletes should have been set to ready by finished snapshot deletes and finalizations"; + for (SnapshotDeletionsInProgress.Entry entry : + currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY).getEntries()) { + if (entry.repository().equals(repository) && entry.state() == SnapshotDeletionsInProgress.State.STARTED) { + deletionToRun = entry; + break; + } + } + return currentState; + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn("Failed to run ready delete operations", e); + failAllListenersOnMasterFailOver(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + if (deletionToRun == null) { + runNextQueuedOperation(repositoryData, repository, false); + } else { + deleteSnapshotsFromRepository(deletionToRun, repositoryData, newState.nodes().getMinNodeVersion()); + } + } + }); + } + + /** + * Finds snapshot delete operations that are ready to execute in the given {@link ClusterState} and computes a new cluster state that + * has all executable deletes marked as executing. Returns a {@link Tuple} of the updated cluster state and all executable deletes. + * This can either be {@link SnapshotDeletionsInProgress.Entry} that were already in state + * {@link SnapshotDeletionsInProgress.State#STARTED} or waiting entries in state {@link SnapshotDeletionsInProgress.State#WAITING} + * that were moved to {@link SnapshotDeletionsInProgress.State#STARTED} in the returned updated cluster state. + * + * @param currentState current cluster state + * @return tuple of an updated cluster state and currently executable snapshot delete operations + */ + private static Tuple> readyDeletions(ClusterState currentState) { + final SnapshotDeletionsInProgress deletions = + currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + if (deletions.hasDeletionsInProgress() == false) { + return Tuple.tuple(currentState, List.of()); } + final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE); + assert snapshotsInProgress != null; + final Set repositoriesSeen = new HashSet<>(); + boolean changed = false; + final ArrayList readyDeletions = new ArrayList<>(); + final List newDeletes = new ArrayList<>(); + for (SnapshotDeletionsInProgress.Entry entry : deletions.getEntries()) { + final String repo = entry.repository(); + if (repositoriesSeen.add(entry.repository()) && entry.state() == SnapshotDeletionsInProgress.State.WAITING + && snapshotsInProgress.entries().stream() + .filter(se -> se.repository().equals(repo)).noneMatch(SnapshotsService::isWritingToRepository)) { + changed = true; + final SnapshotDeletionsInProgress.Entry newEntry = entry.started(); + readyDeletions.add(newEntry); + newDeletes.add(newEntry); + } else { + newDeletes.add(entry); + } + } + return Tuple.tuple(changed ? ClusterState.builder(currentState).putCustom( + SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.of(newDeletes)).build() : currentState, readyDeletions); } + /** + * Computes the cluster state resulting from removing a given snapshot create operation from the given state. + * + * @param state current cluster state + * @param snapshot snapshot for which to remove the snapshot operation + * @return updated cluster state + */ private static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot snapshot) { SnapshotsInProgress snapshots = state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + ClusterState result = state; boolean changed = false; ArrayList entries = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshots.entries()) { @@ -772,17 +1052,21 @@ private static ClusterState stateWithoutSnapshot(ClusterState state, Snapshot sn } } if (changed) { - return ClusterState.builder(state).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(entries)).build(); + result = ClusterState.builder(state).putCustom( + SnapshotsInProgress.TYPE, SnapshotsInProgress.of(unmodifiableList(entries))).build(); } - return state; + return readyDeletions(result).v1(); } /** - * Removes record of running snapshot from cluster state and notifies the listener when this action is complete - * @param snapshot snapshot - * @param failure exception if snapshot failed + * Removes record of running snapshot from cluster state and notifies the listener when this action is complete. This method is only + * used when the snapshot fails for some reason. During normal operation the snapshot repository will remove the + * {@link SnapshotsInProgress.Entry} from the cluster state once it's done finalizing the snapshot. + * + * @param snapshot snapshot that failed + * @param failure exception that failed the snapshot */ - private void removeSnapshotFromClusterState(final Snapshot snapshot, Exception failure) { + private void removeFailedSnapshotFromClusterState(Snapshot snapshot, Exception failure, RepositoryData repositoryData) { assert failure != null : "Failure must be supplied"; clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() { @@ -795,32 +1079,29 @@ public ClusterState execute(ClusterState currentState) { public void onFailure(String source, Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to remove snapshot metadata", snapshot), e); failSnapshotCompletionListeners( - snapshot, new SnapshotException(snapshot, "Failed to remove snapshot from cluster state", e)); + snapshot, new SnapshotException(snapshot, "Failed to remove snapshot from cluster state", e)); + failAllListenersOnMasterFailOver(e); } @Override public void onNoLongerMaster(String source) { - failSnapshotCompletionListeners( - snapshot, ExceptionsHelper.useOrSuppress(failure, new SnapshotException(snapshot, "no longer master"))); + failure.addSuppressed(new SnapshotException(snapshot, "no longer master")); + failSnapshotCompletionListeners(snapshot, failure); + failAllListenersOnMasterFailOver(new NotMasterException(source)); } @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { failSnapshotCompletionListeners(snapshot, failure); + runNextQueuedOperation(repositoryData, snapshot.getRepository(), true); } }); } private void failSnapshotCompletionListeners(Snapshot snapshot, Exception e) { - final List>> completionListeners = snapshotCompletionListeners.remove(snapshot); - if (completionListeners != null) { - try { - ActionListener.onFailure(completionListeners, e); - } catch (Exception ex) { - logger.warn("Failed to notify listeners", ex); - } - } endingSnapshots.remove(snapshot); + failListenersIgnoringException(snapshotCompletionListeners.remove(snapshot), e); + assert repositoryOperations.assertNotQueued(snapshot); } /** @@ -849,17 +1130,18 @@ public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionLis @Override public ClusterState execute(ClusterState currentState) throws Exception { + final Version minNodeVersion = currentState.nodes().getMinNodeVersion(); final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); - final SnapshotsInProgress.Entry snapshotEntry = findInProgressSnapshot(snapshots, snapshotNames, repositoryName); + final List snapshotEntries = findInProgressSnapshots(snapshots, snapshotNames, repositoryName); final List snapshotIds = matchingSnapshotIds( - snapshotEntry == null ? null : snapshotEntry.snapshot().getSnapshotId(), - repositoryData, snapshotNames, repositoryName); - if (snapshotEntry == null) { - deleteFromRepoTask = - createDeleteStateUpdate(snapshotIds, repositoryName, repositoryData.getGenId(), Priority.NORMAL, listener); + snapshotEntries.stream().map(e -> e.snapshot().getSnapshotId()).collect(Collectors.toList()), repositoryData, + snapshotNames, repositoryName); + if (snapshotEntries.isEmpty() || minNodeVersion.onOrAfter(SnapshotsService.FULL_CONCURRENCY_VERSION)) { + deleteFromRepoTask = createDeleteStateUpdate(snapshotIds, repositoryName, repositoryData, Priority.NORMAL, listener); return deleteFromRepoTask.execute(currentState); } - + assert snapshotEntries.size() == 1 : "Expected just a single running snapshot but saw " + snapshotEntries; + final SnapshotsInProgress.Entry snapshotEntry = snapshotEntries.get(0); runningSnapshot = snapshotEntry.snapshot(); final ImmutableOpenMap shards; @@ -879,16 +1161,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { abortedDuringInit = true; } else if (state == State.STARTED) { // snapshot is started - mark every non completed shard as aborted - final ImmutableOpenMap.Builder shardsBuilder = ImmutableOpenMap.builder(); - for (ObjectObjectCursor shardEntry : snapshotEntry.shards()) { - ShardSnapshotStatus status = shardEntry.value; - if (status.state().completed() == false) { - status = new ShardSnapshotStatus( - status.nodeId(), ShardState.ABORTED, "aborted by snapshot deletion", status.generation()); - } - shardsBuilder.put(shardEntry.key, status); - } - shards = shardsBuilder.build(); + shards = abortEntry(snapshotEntry); failure = "Snapshot was aborted by deletion"; } else { boolean hasUncompletedShards = false; @@ -915,14 +1188,14 @@ public ClusterState execute(ClusterState currentState) throws Exception { } return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(snapshots.entries().stream() - // remove init state snapshot we found from a previous master if there was one - .filter(existing -> abortedDuringInit == false || existing.equals(snapshotEntry) == false) - .map(existing -> { - if (existing.equals(snapshotEntry)) { - return new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards, failure); - } - return existing; - }).collect(Collectors.toUnmodifiableList()))).build(); + // remove init state snapshot we found from a previous master if there was one + .filter(existing -> abortedDuringInit == false || existing.equals(snapshotEntry) == false) + .map(existing -> { + if (existing.equals(snapshotEntry)) { + return new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards, failure); + } + return existing; + }).collect(Collectors.toUnmodifiableList()))).build(); } @Override @@ -944,8 +1217,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS listener.onResponse(null); } else { clusterService.submitStateUpdateTask("delete snapshot", - createDeleteStateUpdate(outstandingDeletes, repositoryName, repositoryData.getGenId(), - Priority.IMMEDIATE, listener)); + createDeleteStateUpdate(outstandingDeletes, repositoryName, repositoryData, Priority.IMMEDIATE, listener)); } return; } @@ -954,8 +1226,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS result -> { logger.debug("deleted snapshot completed - deleting files"); clusterService.submitStateUpdateTask("delete snapshot", - createDeleteStateUpdate(outstandingDeletes, repositoryName, - result.v1().getGenId(), Priority.IMMEDIATE, listener)); + createDeleteStateUpdate(outstandingDeletes, repositoryName, result.v1(), Priority.IMMEDIATE, listener)); }, e -> { if (ExceptionsHelper.unwrap(e, NotMasterException.class, FailedToCommitClusterStateException.class) != null) { @@ -978,11 +1249,11 @@ public TimeValue timeout() { }, "delete snapshot", listener::onFailure); } - private static List matchingSnapshotIds(@Nullable SnapshotId inProgress, RepositoryData repositoryData, + private static List matchingSnapshotIds(List inProgress, RepositoryData repositoryData, String[] snapshotsOrPatterns, String repositoryName) { final Map allSnapshotIds = repositoryData.getSnapshotIds().stream().collect( Collectors.toMap(SnapshotId::getName, Function.identity())); - final Set foundSnapshots = new HashSet<>(); + final Set foundSnapshots = new HashSet<>(inProgress); for (String snapshotOrPattern : snapshotsOrPatterns) { if (Regex.isSimpleMatchPattern(snapshotOrPattern)) { for (Map.Entry entry : allSnapshotIds.entrySet()) { @@ -993,7 +1264,7 @@ private static List matchingSnapshotIds(@Nullable SnapshotId inProgr } else { final SnapshotId foundId = allSnapshotIds.get(snapshotOrPattern); if (foundId == null) { - if (inProgress == null || inProgress.getName().equals(snapshotOrPattern) == false) { + if (inProgress.stream().noneMatch(snapshotId -> snapshotId.getName().equals(snapshotOrPattern))) { throw new SnapshotMissingException(repositoryName, snapshotOrPattern); } } else { @@ -1004,22 +1275,20 @@ private static List matchingSnapshotIds(@Nullable SnapshotId inProgr return List.copyOf(foundSnapshots); } - // Return in-progress snapshot entry by name and repository in the given cluster state or null if none is found - @Nullable - private static SnapshotsInProgress.Entry findInProgressSnapshot(SnapshotsInProgress snapshots, String[] snapshotNames, - String repositoryName) { - SnapshotsInProgress.Entry snapshotEntry = null; + // Return in-progress snapshot entries by name and repository in the given cluster state or null if none is found + private static List findInProgressSnapshots(SnapshotsInProgress snapshots, String[] snapshotNames, + String repositoryName) { + List entries = new ArrayList<>(); for (SnapshotsInProgress.Entry entry : snapshots.entries()) { if (entry.repository().equals(repositoryName) - && Regex.simpleMatch(snapshotNames, entry.snapshot().getSnapshotId().getName())) { - snapshotEntry = entry; - break; + && Regex.simpleMatch(snapshotNames, entry.snapshot().getSnapshotId().getName())) { + entries.add(entry); } } - return snapshotEntry; + return entries; } - private ClusterStateUpdateTask createDeleteStateUpdate(List snapshotIds, String repoName, long repositoryStateId, + private ClusterStateUpdateTask createDeleteStateUpdate(List snapshotIds, String repoName, RepositoryData repositoryData, Priority priority, ActionListener listener) { // Short circuit to noop state update if there isn't anything to delete if (snapshotIds.isEmpty()) { @@ -1041,45 +1310,99 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }; } return new ClusterStateUpdateTask(priority) { + + private SnapshotDeletionsInProgress.Entry newDelete; + + private boolean reusedExistingDelete = false; + + private final Collection completedSnapshots = new ArrayList<>(); + @Override public ClusterState execute(ClusterState currentState) { final SnapshotDeletionsInProgress deletionsInProgress = - currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); - if (deletionsInProgress.hasDeletionsInProgress()) { - throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), - "cannot delete - another snapshot is currently being deleted in [" + deletionsInProgress + "]"); + currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + final Version minNodeVersion = currentState.nodes().getMinNodeVersion(); + if (minNodeVersion.before(FULL_CONCURRENCY_VERSION)) { + if (deletionsInProgress.hasDeletionsInProgress()) { + throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), + "cannot delete - another snapshot is currently being deleted in [" + deletionsInProgress + "]"); + } } final RepositoryCleanupInProgress repositoryCleanupInProgress = - currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY); + currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY); if (repositoryCleanupInProgress.hasCleanupInProgress()) { throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), - "cannot delete snapshots while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]"); + "cannot delete snapshots while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]"); } final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE, RestoreInProgress.EMPTY); // don't allow snapshot deletions while a restore is taking place, // otherwise we could end up deleting a snapshot that is being restored // and the files the restore depends on would all be gone + for (RestoreInProgress.Entry entry : restoreInProgress) { if (repoName.equals(entry.snapshot().getRepository()) && snapshotIds.contains(entry.snapshot().getSnapshotId())) { throw new ConcurrentSnapshotExecutionException(new Snapshot(repoName, snapshotIds.get(0)), - "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]"); + "cannot delete snapshot during a restore in progress in [" + restoreInProgress + "]"); } } - SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); - if (snapshots.entries().isEmpty() == false) { - // However other snapshots are running - cannot continue - throw new ConcurrentSnapshotExecutionException( - repoName, snapshotIds.toString(), "another snapshot is currently running cannot delete"); + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final SnapshotsInProgress updatedSnapshots; + if (minNodeVersion.onOrAfter(FULL_CONCURRENCY_VERSION)) { + updatedSnapshots = SnapshotsInProgress.of(snapshots.entries().stream() + .map(existing -> { + // snapshot is started - mark every non completed shard as aborted + if (existing.state() == State.STARTED && snapshotIds.contains(existing.snapshot().getSnapshotId())) { + final ImmutableOpenMap abortedShards = abortEntry(existing); + final boolean isCompleted = completed(abortedShards.values()); + final SnapshotsInProgress.Entry abortedEntry = new SnapshotsInProgress.Entry( + existing, isCompleted ? State.SUCCESS : State.ABORTED, abortedShards, + "Snapshot was aborted by deletion"); + if (isCompleted) { + completedSnapshots.add(abortedEntry); + } + return abortedEntry; + } + return existing; + }).collect(Collectors.toUnmodifiableList())); + } else { + if (snapshots.entries().isEmpty() == false) { + // However other snapshots are running - cannot continue + throw new ConcurrentSnapshotExecutionException( + repoName, snapshotIds.toString(), "another snapshot is currently running cannot delete"); + } + updatedSnapshots = snapshots; } // add the snapshot deletion to the cluster state - SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry( + final SnapshotDeletionsInProgress.Entry replacedEntry = deletionsInProgress.getEntries().stream().filter(entry -> + entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.WAITING) + .findFirst().orElse(null); + if (replacedEntry == null) { + final Optional foundDuplicate = + deletionsInProgress.getEntries().stream().filter(entry -> + entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED + && entry.getSnapshots().containsAll(snapshotIds)).findFirst(); + if (foundDuplicate.isPresent()) { + newDelete = foundDuplicate.get(); + reusedExistingDelete = true; + return currentState; + } + ensureBelowConcurrencyLimit(repoName, snapshotIds.get(0).getName(), snapshots, deletionsInProgress); + newDelete = new SnapshotDeletionsInProgress.Entry( snapshotIds, repoName, threadPool.absoluteTimeInMillis(), - repositoryStateId - ); - return ClusterState.builder(currentState).putCustom( - SnapshotDeletionsInProgress.TYPE, deletionsInProgress.withAddedEntry(entry)).build(); + repositoryData.getGenId(), + updatedSnapshots.entries().stream().filter(entry -> repoName.equals(entry.repository())).noneMatch( + SnapshotsService::isWritingToRepository) + && deletionsInProgress.getEntries().stream().noneMatch(entry -> + repoName.equals(entry.repository()) && entry.state() == SnapshotDeletionsInProgress.State.STARTED) + ? SnapshotDeletionsInProgress.State.STARTED : SnapshotDeletionsInProgress.State.WAITING); + } else { + newDelete = replacedEntry.withAddedSnapshots(snapshotIds); + } + return updateWithSnapshots(currentState, updatedSnapshots, + (replacedEntry == null ? deletionsInProgress : deletionsInProgress.withRemovedEntry(replacedEntry.uuid())) + .withAddedEntry(newDelete)); } @Override @@ -1089,11 +1412,64 @@ public void onFailure(String source, Exception e) { @Override public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - deleteSnapshotsFromRepository(repoName, snapshotIds, listener, repositoryStateId, newState.nodes().getMinNodeVersion()); + addDeleteListener(newDelete.uuid(), listener); + if (reusedExistingDelete) { + return; + } + if (newDelete.state() == SnapshotDeletionsInProgress.State.STARTED) { + if (tryEnterRepoLoop(repoName)) { + deleteSnapshotsFromRepository(newDelete, repositoryData, newState.nodes().getMinNodeVersion()); + } else { + logger.trace("Delete [{}] could not execute directly and was queued", newDelete); + } + } else { + for (SnapshotsInProgress.Entry completedSnapshot : completedSnapshots) { + endSnapshot(completedSnapshot, newState.metadata(), repositoryData); + } + } } }; } + /** + * Checks if the given {@link SnapshotsInProgress.Entry} is currently writing to the repository. + * + * @param entry snapshot entry + * @return true if entry is currently writing to the repository + */ + private static boolean isWritingToRepository(SnapshotsInProgress.Entry entry) { + if (entry.state().completed()) { + // Entry is writing to the repo because it's finalizing on master + return true; + } + for (ObjectCursor value : entry.shards().values()) { + if (value.value.isActive()) { + // Entry is writing to the repo because it's writing to a shard on a data node or waiting to do so for a concrete shard + return true; + } + } + return false; + } + + private ImmutableOpenMap abortEntry(SnapshotsInProgress.Entry existing) { + final ImmutableOpenMap.Builder shardsBuilder = + ImmutableOpenMap.builder(); + for (ObjectObjectCursor shardEntry : existing.shards()) { + ShardSnapshotStatus status = shardEntry.value; + if (status.state().completed() == false) { + final String nodeId = status.nodeId(); + status = new ShardSnapshotStatus(nodeId, nodeId == null ? ShardState.FAILED : ShardState.ABORTED, + "aborted by snapshot deletion", status.generation()); + } + shardsBuilder.put(shardEntry.key, status); + } + return shardsBuilder.build(); + } + + private void addDeleteListener(String deleteUUID, ActionListener listener) { + snapshotDeletionListeners.computeIfAbsent(deleteUUID, k -> new CopyOnWriteArrayList<>()).add(listener); + } + /** * Determines the minimum {@link Version} that the snapshot repository must be compatible with from the current nodes in the cluster * and the contents of the repository. The minimum version is determined as the lowest version found across all snapshots in the @@ -1146,82 +1522,361 @@ public static boolean useIndexGenerations(Version repositoryMetaVersion) { /** Deletes snapshot from repository * - * @param repoName repository name - * @param snapshotIds snapshot ids - * @param listener listener - * @param repositoryStateId the unique id representing the state of the repository at the time the deletion began + * @param deleteEntry delete entry in cluster state + * @param minNodeVersion minimum node version in the cluster + */ + private void deleteSnapshotsFromRepository(SnapshotDeletionsInProgress.Entry deleteEntry, Version minNodeVersion) { + final long expectedRepoGen = deleteEntry.repositoryStateId(); + repositoriesService.getRepositoryData(deleteEntry.repository(), new ActionListener<>() { + @Override + public void onResponse(RepositoryData repositoryData) { + assert repositoryData.getGenId() == expectedRepoGen : + "Repository generation should not change as long as a ready delete is found in the cluster state but found [" + + expectedRepoGen + "] in cluster state and [" + repositoryData.getGenId() + "] in the repository"; + deleteSnapshotsFromRepository(deleteEntry, repositoryData, minNodeVersion); + } + + @Override + public void onFailure(Exception e) { + clusterService.submitStateUpdateTask("fail repo tasks for [" + deleteEntry.repository() + "]", + new FailPendingRepoTasksTask(deleteEntry.repository(), e)); + } + }); + } + + /** Deletes snapshot from repository + * + * @param deleteEntry delete entry in cluster state + * @param repositoryData the {@link RepositoryData} of the repository to delete from * @param minNodeVersion minimum node version in the cluster */ - private void deleteSnapshotsFromRepository(String repoName, Collection snapshotIds, @Nullable ActionListener listener, - long repositoryStateId, Version minNodeVersion) { - Repository repository = repositoriesService.repository(repoName); - repository.getRepositoryData(ActionListener.wrap(repositoryData -> repository.deleteSnapshots( + private void deleteSnapshotsFromRepository(SnapshotDeletionsInProgress.Entry deleteEntry, + RepositoryData repositoryData, Version minNodeVersion) { + if (repositoryOperations.startDeletion(deleteEntry.uuid())) { + assert currentlyFinalizing.contains(deleteEntry.repository()); + final List snapshotIds = deleteEntry.getSnapshots(); + assert deleteEntry.state() == SnapshotDeletionsInProgress.State.STARTED : + "incorrect state for entry [" + deleteEntry + "]"; + repositoriesService.repository(deleteEntry.repository()).deleteSnapshots( snapshotIds, - repositoryStateId, + repositoryData.getGenId(), minCompatibleVersion(minNodeVersion, repositoryData, snapshotIds), - ActionListener.wrap(v -> { - logger.info("snapshots {} deleted", snapshotIds); - removeSnapshotDeletionFromClusterState(snapshotIds, null, listener); - }, ex -> removeSnapshotDeletionFromClusterState(snapshotIds, ex, listener) - )), ex -> removeSnapshotDeletionFromClusterState(snapshotIds, ex, listener))); + ActionListener.wrap(updatedRepoData -> { + logger.info("snapshots {} deleted", snapshotIds); + removeSnapshotDeletionFromClusterState(deleteEntry, null, updatedRepoData); + }, ex -> removeSnapshotDeletionFromClusterState(deleteEntry, ex, repositoryData) + )); + } } /** - * Removes the snapshot deletion from {@link SnapshotDeletionsInProgress} in the cluster state. + * Removes a {@link SnapshotDeletionsInProgress.Entry} from {@link SnapshotDeletionsInProgress} in the cluster state after it executed + * on the repository. + * + * @param deleteEntry delete entry to remove from the cluster state + * @param failure failure encountered while executing the delete on the repository or {@code null} if the delete executed + * successfully + * @param repositoryData current {@link RepositoryData} for the repository we just ran the delete on. */ - private void removeSnapshotDeletionFromClusterState(final Collection snapshotIds, @Nullable final Exception failure, - @Nullable final ActionListener listener) { - clusterService.submitStateUpdateTask("remove snapshot deletion metadata", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) { - final SnapshotDeletionsInProgress deletions = - currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); - if (deletions.hasDeletionsInProgress()) { - assert deletions.getEntries().size() == 1 : "should have exactly one deletion in progress"; - SnapshotDeletionsInProgress.Entry entry = deletions.getEntries().get(0); - return ClusterState.builder(currentState).putCustom( - SnapshotDeletionsInProgress.TYPE, deletions.withRemovedEntry(entry)).build(); + private void removeSnapshotDeletionFromClusterState(final SnapshotDeletionsInProgress.Entry deleteEntry, + @Nullable final Exception failure, final RepositoryData repositoryData) { + final ClusterStateUpdateTask clusterStateUpdateTask; + if (failure == null) { + // If we didn't have a failure during the snapshot delete we will remove all snapshot ids that the delete successfully removed + // from the repository from enqueued snapshot delete entries during the cluster state update. After the cluster state update we + // resolve the delete listeners with the latest repository data from after the delete. + clusterStateUpdateTask = new RemoveSnapshotDeletionAndContinueTask(deleteEntry, repositoryData) { + @Override + protected SnapshotDeletionsInProgress filterDeletions(SnapshotDeletionsInProgress deletions) { + boolean changed = false; + List updatedEntries = new ArrayList<>(deletions.getEntries().size()); + for (SnapshotDeletionsInProgress.Entry entry : deletions.getEntries()) { + if (entry.repository().equals(deleteEntry.repository())) { + final List updatedSnapshotIds = new ArrayList<>(entry.getSnapshots()); + if (updatedSnapshotIds.removeAll(deleteEntry.getSnapshots())) { + changed = true; + updatedEntries.add(entry.withSnapshots(updatedSnapshotIds)); + } else { + updatedEntries.add(entry); + } + } else { + updatedEntries.add(entry); + } + } + return changed ? SnapshotDeletionsInProgress.of(updatedEntries) : deletions; + } + + @Override + protected void handleListeners(List> deleteListeners) { + assert repositoryData.getSnapshotIds().stream().noneMatch(deleteEntry.getSnapshots()::contains) + : "Repository data contained snapshot ids " + repositoryData.getSnapshotIds() + + " that should should been deleted by [" + deleteEntry + "]"; + completeListenersIgnoringException(deleteListeners, null); + } + }; + } else { + // The delete failed to execute on the repository. We remove it from the cluster state and then fail all listeners associated + // with it. + clusterStateUpdateTask = new RemoveSnapshotDeletionAndContinueTask(deleteEntry, repositoryData) { + @Override + protected void handleListeners(List> deleteListeners) { + failListenersIgnoringException(deleteListeners, failure); + } + }; + } + clusterService.submitStateUpdateTask("remove snapshot deletion metadata", clusterStateUpdateTask); + } + + /** + * Handle snapshot or delete failure due to not being master any more so we don't try to do run additional cluster state updates. + * The next master will try handling the missing operations. All we can do is fail all the listeners on this master node so that + * transport requests return and we don't leak listeners. + * + * @param e exception that caused us to realize we are not master any longer + */ + private void failAllListenersOnMasterFailOver(Exception e) { + logger.debug("Failing all snapshot operation listeners because this node is not master any longer", e); + synchronized (currentlyFinalizing) { + if (ExceptionsHelper.unwrap(e, NotMasterException.class, FailedToCommitClusterStateException.class) != null) { + repositoryOperations.clear(); + for (Snapshot snapshot : Set.copyOf(snapshotCompletionListeners.keySet())) { + failSnapshotCompletionListeners(snapshot, new SnapshotException(snapshot, "no longer master")); } + final Exception wrapped = + new RepositoryException("_all", "Failed to update cluster state during repository operation", e); + for (Iterator>> iterator = snapshotDeletionListeners.values().iterator(); + iterator.hasNext(); ) { + final List> listeners = iterator.next(); + iterator.remove(); + failListenersIgnoringException(listeners, wrapped); + } + assert snapshotDeletionListeners.isEmpty() : + "No new listeners should have been added but saw " + snapshotDeletionListeners; + } else { + assert false : + new AssertionError("Modifying snapshot state should only ever fail because we failed to publish new state", e); + logger.error("Unexpected failure during cluster state update", e); + } + currentlyFinalizing.clear(); + } + } + + /** + * A cluster state update that will remove a given {@link SnapshotDeletionsInProgress.Entry} from the cluster state + * and trigger running the next snapshot-delete or -finalization operation available to execute if there is one + * ready in the cluster state as a result of this state update. + */ + private abstract class RemoveSnapshotDeletionAndContinueTask extends ClusterStateUpdateTask { + + // Snapshots that can be finalized after the delete operation has been removed from the cluster state + protected final List newFinalizations = new ArrayList<>(); + + private List readyDeletions = Collections.emptyList(); + + protected final SnapshotDeletionsInProgress.Entry deleteEntry; + + private final RepositoryData repositoryData; + + RemoveSnapshotDeletionAndContinueTask(SnapshotDeletionsInProgress.Entry deleteEntry, RepositoryData repositoryData) { + this.deleteEntry = deleteEntry; + this.repositoryData = repositoryData; + } + + @Override + public ClusterState execute(ClusterState currentState) { + final SnapshotDeletionsInProgress deletions = currentState.custom(SnapshotDeletionsInProgress.TYPE); + assert deletions != null : "We only run this if there were deletions in the cluster state before"; + final SnapshotDeletionsInProgress updatedDeletions = deletions.withRemovedEntry(deleteEntry.uuid()); + if (updatedDeletions == deletions) { return currentState; } + final SnapshotDeletionsInProgress newDeletions = filterDeletions(updatedDeletions); + final Tuple> res = readyDeletions( + updateWithSnapshots(currentState, updatedSnapshotsInProgress(currentState, newDeletions), newDeletions)); + readyDeletions = res.v2(); + return res.v1(); + } - @Override - public void onFailure(String source, Exception e) { - logger.warn(() -> new ParameterizedMessage("{} failed to remove snapshot deletion metadata", snapshotIds), e); - if (listener != null) { - listener.onFailure(e); + @Override + public void onFailure(String source, Exception e) { + logger.warn(() -> new ParameterizedMessage("{} failed to remove snapshot deletion metadata", deleteEntry), e); + repositoryOperations.finishDeletion(deleteEntry.uuid()); + failAllListenersOnMasterFailOver(e); + } + + protected SnapshotDeletionsInProgress filterDeletions(SnapshotDeletionsInProgress deletions) { + return deletions; + } + + @Override + public final void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + final List> deleteListeners; + repositoryOperations.finishDeletion(deleteEntry.uuid()); + deleteListeners = snapshotDeletionListeners.remove(deleteEntry.uuid()); + handleListeners(deleteListeners); + if (newFinalizations.isEmpty()) { + if (readyDeletions.isEmpty()) { + leaveRepoLoop(deleteEntry.repository()); + } else { + for (SnapshotDeletionsInProgress.Entry readyDeletion : readyDeletions) { + deleteSnapshotsFromRepository(readyDeletion, repositoryData, newState.nodes().getMinNodeVersion()); + } + } + } else { + leaveRepoLoop(deleteEntry.repository()); + assert readyDeletions.stream().noneMatch(entry -> entry.repository().equals(deleteEntry.repository())) + : "New finalizations " + newFinalizations + " added even though deletes " + readyDeletions + " are ready"; + for (SnapshotsInProgress.Entry entry : newFinalizations) { + endSnapshot(entry, newState.metadata(), repositoryData); } } + } - @Override - public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { - if (listener != null) { - if (failure != null) { - listener.onFailure(failure); + /** + * Invoke snapshot delete listeners for {@link #deleteEntry}. + * + * @param deleteListeners delete snapshot listeners or {@code null} if there weren't any for {@link #deleteEntry}. + */ + protected abstract void handleListeners(@Nullable List> deleteListeners); + + /** + * Computes an updated {@link SnapshotsInProgress} that takes into account an updated version of + * {@link SnapshotDeletionsInProgress} that has a {@link SnapshotDeletionsInProgress.Entry} removed from it + * relative to the {@link SnapshotDeletionsInProgress} found in {@code currentState}. + * The removal of a delete from the cluster state can trigger two possible actions on in-progress snapshots: + *

    + *
  • Snapshots that had unfinished shard snapshots in state {@link ShardSnapshotStatus#UNASSIGNED_QUEUED} that + * could not be started because the delete was running can have those started.
    • + *
  • Snapshots that had all their shards reach a completed state while a delete was running (e.g. as a result of + * nodes dropping out of the cluster or another incoming delete aborting them) need not be updated in the cluster + * state but need to have their finalization triggered now that it's possible with the removal of the delete + * from the state.
    • + *
+ * + * @param currentState current cluster state + * @param updatedDeletions deletions with removed entry + * @return updated snapshot in progress instance or {@code null} if there are no changes to it + */ + @Nullable + private SnapshotsInProgress updatedSnapshotsInProgress(ClusterState currentState, + SnapshotDeletionsInProgress updatedDeletions) { + final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final List snapshotEntries = new ArrayList<>(); + + // Keep track of shardIds that we started snapshots for as a result of removing this delete so we don't assign + // them to multiple snapshots by accident + final Set reassignedShardIds = new HashSet<>(); + + boolean changed = false; + + final String repoName = deleteEntry.repository(); + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { + if (entry.repository().equals(repoName)) { + if (entry.state().completed() == false) { + // Collect waiting shards that in entry that we can assign now that we are done with the deletion + final List canBeUpdated = new ArrayList<>(); + for (ObjectObjectCursor value : entry.shards()) { + if (value.value.equals(ShardSnapshotStatus.UNASSIGNED_QUEUED) + && reassignedShardIds.contains(value.key) == false) { + canBeUpdated.add(value.key); + } + } + if (canBeUpdated.isEmpty()) { + // No shards can be updated in this snapshot so we just add it as is again + snapshotEntries.add(entry); + } else { + final ImmutableOpenMap shardAssignments = shards(snapshotsInProgress, + updatedDeletions, currentState.metadata(), currentState.routingTable(), entry.indices(), + entry.version().onOrAfter(SHARD_GEN_IN_REPO_DATA_VERSION), repositoryData, repoName); + final ImmutableOpenMap.Builder updatedAssignmentsBuilder = + ImmutableOpenMap.builder(entry.shards()); + for (ShardId shardId : canBeUpdated) { + final boolean added = reassignedShardIds.add(shardId); + assert added; + updatedAssignmentsBuilder.put(shardId, shardAssignments.get(shardId)); + } + snapshotEntries.add(entry.withShards(updatedAssignmentsBuilder.build())); + changed = true; + } } else { - logger.info("Successfully deleted snapshots {}", snapshotIds); - listener.onResponse(null); + // Entry is already completed so we will finalize it now that the delete doesn't block us after + // this CS update finishes + newFinalizations.add(entry); + snapshotEntries.add(entry); } + } else { + // Entry is for another repository we just keep it as is + snapshotEntries.add(entry); } } - }); + return changed ? SnapshotsInProgress.of(snapshotEntries) : null; + } + } + + /** + * Shortcut to build new {@link ClusterState} from the current state and updated values of {@link SnapshotsInProgress} and + * {@link SnapshotDeletionsInProgress}. + * + * @param state current cluster state + * @param snapshotsInProgress new value for {@link SnapshotsInProgress} or {@code null} if it's unchanged + * @param snapshotDeletionsInProgress new value for {@link SnapshotDeletionsInProgress} or {@code null} if it's unchanged + * @return updated cluster state + */ + public static ClusterState updateWithSnapshots(ClusterState state, + @Nullable SnapshotsInProgress snapshotsInProgress, + @Nullable SnapshotDeletionsInProgress snapshotDeletionsInProgress) { + if (snapshotsInProgress == null && snapshotDeletionsInProgress == null) { + return state; + } + ClusterState.Builder builder = ClusterState.builder(state); + if (snapshotsInProgress != null) { + builder.putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress); + } + if (snapshotDeletionsInProgress != null) { + builder.putCustom(SnapshotDeletionsInProgress.TYPE, snapshotDeletionsInProgress); + } + return builder.build(); + } + + private static void failListenersIgnoringException(@Nullable List> listeners, Exception failure) { + if (listeners != null) { + try { + ActionListener.onFailure(listeners, failure); + } catch (Exception ex) { + assert false : new AssertionError(ex); + logger.warn("Failed to notify listeners", ex); + } + } + } + + private static void completeListenersIgnoringException(@Nullable List> listeners, T result) { + if (listeners != null) { + try { + ActionListener.onResponse(listeners, result); + } catch (Exception ex) { + assert false : new AssertionError(ex); + logger.warn("Failed to notify listeners", ex); + } + } } /** - * Calculates the list of shards that should be included into the current snapshot + * Calculates the assignment of shards to data nodes for a new snapshot based on the given cluster state and the + * indices that should be included in the snapshot. * - * @param clusterState cluster state * @param indices Indices to snapshot * @param useShardGenerations whether to write {@link ShardGenerations} during the snapshot * @return list of shard to be included into current snapshot */ - private static ImmutableOpenMap shards(ClusterState clusterState, - List indices, - boolean useShardGenerations, - RepositoryData repositoryData) { + private static ImmutableOpenMap shards( + @Nullable SnapshotsInProgress snapshotsInProgress, @Nullable SnapshotDeletionsInProgress deletionsInProgress, + Metadata metadata, RoutingTable routingTable, List indices, boolean useShardGenerations, + RepositoryData repositoryData, String repoName) { ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); - Metadata metadata = clusterState.metadata(); final ShardGenerations shardGenerations = repositoryData.shardGenerations(); + final Set inProgressShards = busyShardsForRepo(repoName, snapshotsInProgress); + final boolean readyToExecute = deletionsInProgress == null || deletionsInProgress.getEntries().stream() + .noneMatch(entry -> entry.repository().equals(repoName) && entry.state() == SnapshotDeletionsInProgress.State.STARTED); for (IndexId index : indices) { final String indexName = index.getName(); final boolean isNewIndex = repositoryData.getIndices().containsKey(indexName) == false; @@ -1231,7 +1886,7 @@ private static ImmutableOpenMap busyShardsForRepo(String repoName, @Nullable SnapshotsInProgress snapshots) { + final List runningSnapshots = snapshots == null ? List.of() : snapshots.entries(); + final Set inProgressShards = new HashSet<>(); + for (SnapshotsInProgress.Entry runningSnapshot : runningSnapshots) { + if (runningSnapshot.repository().equals(repoName) == false) { + continue; + } + for (ObjectObjectCursor shard : runningSnapshot.shards()) { + if (shard.value.isActive()) { + inProgressShards.add(shard.key); + } + } + } + return inProgressShards; + } + /** * Returns the data streams that are currently being snapshotted (with partial == false) and that are contained in the * indices-to-check set. */ - public static Set snapshottingDataStreams(final ClusterState currentState, final Set dataStreamsToCheck) { + public static Set snapshottingDataStreams(final ClusterState currentState, + final Set dataStreamsToCheck) { + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); + if (snapshots == null) { + return emptySet(); + } + Map dataStreams = currentState.metadata().dataStreams(); - return currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries().stream() + return snapshots.entries().stream() .filter(e -> e.partial() == false) .flatMap(e -> e.dataStreams().stream()) .filter(ds -> dataStreams.containsKey(ds) && dataStreamsToCheck.contains(ds)) @@ -1285,8 +1974,13 @@ public static Set snapshottingDataStreams(final ClusterState currentStat * Returns the indices that are currently being snapshotted (with partial == false) and that are contained in the indices-to-check set. */ public static Set snapshottingIndices(final ClusterState currentState, final Set indicesToCheck) { + final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE); + if (snapshots == null) { + return emptySet(); + } + final Set indices = new HashSet<>(); - for (final SnapshotsInProgress.Entry entry : currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries()) { + for (final SnapshotsInProgress.Entry entry : snapshots.entries()) { if (entry.partial() == false) { for (IndexId index : entry.indices()) { IndexMetadata indexMetadata = currentState.metadata().index(index.getName()); @@ -1326,16 +2020,20 @@ protected void doClose() { } /** - * Assert that no in-memory state for any running snapshot operation exists in this instance. + * Assert that no in-memory state for any running snapshot-create or -delete operation exists in this instance. */ public boolean assertAllListenersResolved() { - synchronized (endingSnapshots) { - final DiscoveryNode localNode = clusterService.localNode(); - assert endingSnapshots.isEmpty() : "Found leaked ending snapshots " + endingSnapshots - + " on [" + localNode + "]"; - assert snapshotCompletionListeners.isEmpty() : "Found leaked snapshot completion listeners " + snapshotCompletionListeners - + " on [" + localNode + "]"; - } + final DiscoveryNode localNode = clusterService.localNode(); + assert endingSnapshots.isEmpty() : "Found leaked ending snapshots " + endingSnapshots + + " on [" + localNode + "]"; + assert snapshotCompletionListeners.isEmpty() : "Found leaked snapshot completion listeners " + snapshotCompletionListeners + + " on [" + localNode + "]"; + assert currentlyFinalizing.isEmpty() : "Found leaked finalizations " + currentlyFinalizing + + " on [" + localNode + "]"; + assert snapshotDeletionListeners.isEmpty() : "Found leaked snapshot delete listeners " + snapshotDeletionListeners + + " on [" + localNode + "]"; + assert repositoryOperations.isEmpty() : "Found leaked snapshots to finalize " + repositoryOperations + + " on [" + localNode + "]"; return true; } @@ -1344,23 +2042,44 @@ private static class SnapshotStateExecutor implements ClusterStateTaskExecutor execute(ClusterState currentState, List tasks) { - final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); int changedCount = 0; final List entries = new ArrayList<>(); - for (SnapshotsInProgress.Entry entry : snapshots.entries()) { + final Map> reusedShardIdsByRepo = new HashMap<>(); + for (SnapshotsInProgress.Entry entry : currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries()) { ImmutableOpenMap.Builder shards = ImmutableOpenMap.builder(); boolean updated = false; for (UpdateIndexShardSnapshotStatusRequest updateSnapshotState : tasks) { + final ShardId finishedShardId = updateSnapshotState.shardId(); if (entry.snapshot().equals(updateSnapshotState.snapshot())) { logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), - updateSnapshotState.shardId(), updateSnapshotState.status().state()); + finishedShardId, updateSnapshotState.status().state()); if (updated == false) { shards.putAll(entry.shards()); updated = true; } - shards.put(updateSnapshotState.shardId(), updateSnapshotState.status()); + shards.put(finishedShardId, updateSnapshotState.status()); changedCount++; + } else { + final String updatedRepository = updateSnapshotState.snapshot().getRepository(); + final Set reusedShardIds = reusedShardIdsByRepo.computeIfAbsent(updatedRepository, k -> new HashSet<>()); + if (entry.repository().equals(updatedRepository) && + entry.state().completed() == false && reusedShardIds.contains(finishedShardId) == false + && entry.shards().keys().contains(finishedShardId)) { + final ShardSnapshotStatus existingStatus = entry.shards().get(finishedShardId); + if (existingStatus.state() != ShardState.QUEUED) { + continue; + } + if (updated == false) { + shards.putAll(entry.shards()); + updated = true; + } + final ShardSnapshotStatus finishedStatus = updateSnapshotState.status(); + logger.trace("Starting [{}] on [{}] with generation [{}]", finishedShardId, + finishedStatus.nodeId(), finishedStatus.generation()); + shards.put(finishedShardId, new ShardSnapshotStatus(finishedStatus.nodeId(), finishedStatus.generation())); + reusedShardIds.add(finishedShardId); + } } } @@ -1380,8 +2099,8 @@ private static class SnapshotStateExecutor implements ClusterStateTaskExecutor 0) { logger.trace("changed cluster state triggered by {} snapshot state updates", changedCount); return ClusterTasksResult.builder().successes(tasks) - .build(ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, - SnapshotsInProgress.of(entries)).build()); + .build(ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, + SnapshotsInProgress.of(unmodifiableList(entries))).build()); } return ClusterTasksResult.builder().successes(tasks).build(currentState); } @@ -1418,7 +2137,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS final SnapshotsInProgress.Entry updatedEntry = snapshotsInProgress.snapshot(request.snapshot()); // If the entry is still in the cluster state and is completed, try finalizing the snapshot in the repo if (updatedEntry != null && updatedEntry.state().completed()) { - endSnapshot(updatedEntry, newState.metadata()); + endSnapshot(updatedEntry, newState.metadata(), null); } } } @@ -1457,4 +2176,173 @@ protected ClusterBlockException checkBlock(UpdateIndexShardSnapshotStatusRequest return null; } } + + /** + * Cluster state update task that removes all {@link SnapshotsInProgress.Entry} and {@link SnapshotDeletionsInProgress.Entry} for a + * given repository from the cluster state and afterwards fails all relevant listeners in {@link #snapshotCompletionListeners} and + * {@link #snapshotDeletionListeners}. + */ + private final class FailPendingRepoTasksTask extends ClusterStateUpdateTask { + + // Snapshots to fail after the state update + private final List snapshotsToFail = new ArrayList<>(); + + // Delete uuids to fail because after the state update + private final List deletionsToFail = new ArrayList<>(); + + // Failure that caused the decision to fail all snapshots and deletes for a repo + private final Exception failure; + + private final String repository; + + FailPendingRepoTasksTask(String repository, Exception failure) { + this.repository = repository; + this.failure = failure; + } + + @Override + public ClusterState execute(ClusterState currentState) { + final SnapshotDeletionsInProgress deletionsInProgress = + currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY); + boolean changed = false; + final List remainingEntries = deletionsInProgress.getEntries(); + List updatedEntries = new ArrayList<>(remainingEntries.size()); + for (SnapshotDeletionsInProgress.Entry entry : remainingEntries) { + if (entry.repository().equals(repository)) { + changed = true; + deletionsToFail.add(entry.uuid()); + } else { + updatedEntries.add(entry); + } + } + final SnapshotDeletionsInProgress updatedDeletions = changed ? SnapshotDeletionsInProgress.of(updatedEntries) : null; + final SnapshotsInProgress snapshotsInProgress = + currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY); + final List snapshotEntries = new ArrayList<>(); + boolean changedSnapshots = false; + for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) { + if (entry.repository().equals(repository)) { + // We failed to read repository data for this delete, it is not the job of SnapshotsService to + // retry these kinds of issues so we fail all the pending snapshots + snapshotsToFail.add(entry.snapshot()); + changedSnapshots = true; + } else { + // Entry is for another repository we just keep it as is + snapshotEntries.add(entry); + } + } + final SnapshotsInProgress updatedSnapshotsInProgress = changedSnapshots ? SnapshotsInProgress.of(snapshotEntries) : null; + return updateWithSnapshots(currentState, updatedSnapshotsInProgress, updatedDeletions); + } + + @Override + public void onFailure(String source, Exception e) { + logger.info( + () -> new ParameterizedMessage("Failed to remove all snapshot tasks for repo [{}] from cluster state", repository), e); + failAllListenersOnMasterFailOver(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + logger.warn(() -> + new ParameterizedMessage("Removed all snapshot tasks for repository [{}] from cluster state, now failing listeners", + repository), failure); + synchronized (currentlyFinalizing) { + Tuple finalization; + while ((finalization = repositoryOperations.pollFinalization(repository)) != null) { + assert snapshotsToFail.contains(finalization.v1().snapshot()) : + "[" + finalization.v1() + "] not found in snapshots to fail " + snapshotsToFail; + } + leaveRepoLoop(repository); + for (Snapshot snapshot : snapshotsToFail) { + failSnapshotCompletionListeners(snapshot, failure); + } + for (String delete : deletionsToFail) { + failListenersIgnoringException(snapshotDeletionListeners.remove(delete), failure); + repositoryOperations.finishDeletion(delete); + } + } + } + } + + private static final class OngoingRepositoryOperations { + + /** + * Map of repository name to a deque of {@link SnapshotsInProgress.Entry} that need to be finalized for the repository and the + * {@link Metadata to use when finalizing}. + */ + private final Map> snapshotsToFinalize = new HashMap<>(); + + /** + * Set of delete operations currently being executed against the repository. The values in this set are the delete UUIDs returned + * by {@link SnapshotDeletionsInProgress.Entry#uuid()}. + */ + private final Set runningDeletions = Collections.synchronizedSet(new HashSet<>()); + + @Nullable + private Metadata latestKnownMetaData; + + @Nullable + synchronized Tuple pollFinalization(String repository) { + assertConsistent(); + final SnapshotsInProgress.Entry nextEntry; + final Deque queued = snapshotsToFinalize.get(repository); + if (queued == null) { + return null; + } + nextEntry = queued.pollFirst(); + assert nextEntry != null; + final Tuple res = Tuple.tuple(nextEntry, latestKnownMetaData); + if (queued.isEmpty()) { + snapshotsToFinalize.remove(repository); + } + if (snapshotsToFinalize.isEmpty()) { + latestKnownMetaData = null; + } + assert assertConsistent(); + return res; + } + + boolean startDeletion(String deleteUUID) { + return runningDeletions.add(deleteUUID); + } + + void finishDeletion(String deleteUUID) { + runningDeletions.remove(deleteUUID); + } + + synchronized void addFinalization(SnapshotsInProgress.Entry entry, Metadata metadata) { + snapshotsToFinalize.computeIfAbsent(entry.repository(), k -> new LinkedList<>()).add(entry); + this.latestKnownMetaData = metadata; + assertConsistent(); + } + + /** + * Clear all state associated with running snapshots. To be used on master-failover if the current node stops + * being master. + */ + synchronized void clear() { + snapshotsToFinalize.clear(); + runningDeletions.clear(); + latestKnownMetaData = null; + } + + synchronized boolean isEmpty() { + return snapshotsToFinalize.isEmpty(); + } + + synchronized boolean assertNotQueued(Snapshot snapshot) { + assert snapshotsToFinalize.getOrDefault(snapshot.getRepository(), new LinkedList<>()).stream() + .noneMatch(entry -> entry.snapshot().equals(snapshot)) : "Snapshot [" + snapshot + "] is still in finalization queue"; + return true; + } + + synchronized boolean assertConsistent() { + assert (latestKnownMetaData == null && snapshotsToFinalize.isEmpty()) + || (latestKnownMetaData != null && snapshotsToFinalize.isEmpty() == false) : + "Should not hold on to metadata if there are no more queued snapshots"; + assert snapshotsToFinalize.values().stream().noneMatch(Collection::isEmpty) : "Found empty queue in " + snapshotsToFinalize; + return true; + } + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/package-info.java b/server/src/main/java/org/elasticsearch/snapshots/package-info.java index 55a0c93433b07..5c08aadece0b7 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/package-info.java +++ b/server/src/main/java/org/elasticsearch/snapshots/package-info.java @@ -98,5 +98,42 @@ *
  • After the deletion of the snapshot's data from the repository finishes, the {@code SnapshotsService} will submit a cluster state * update to remove the deletion's entry in {@code SnapshotDeletionsInProgress} which concludes the process of deleting a snapshot.
    • * + * + *

    Concurrent Snapshot Operations

    + * + * Snapshot create and delete operations may be started concurrently. Operations targeting different repositories run independently of + * each other. Multiple operations targeting the same repository are executed according to the following rules: + * + *

    Concurrent Snapshot Creation

    + * + * If multiple snapshot creation jobs are started at the same time, the data-node operations of multiple snapshots may run in parallel + * across different shards. If multiple snapshots want to snapshot a certain shard, then the shard snapshots for that shard will be + * executed one by one. This is enforced by the master node setting the shard's snapshot state to + * {@link org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus#UNASSIGNED_QUEUED} for all but one snapshot. The order of + * operations on a single shard is given by the order in which the snapshots were started. + * As soon as all shards for a given snapshot have finished, it will be finalized as explained above. Finalization will happen one snapshot + * at a time, working in the order in which snapshots had their shards completed. + * + *

    Concurrent Snapshot Deletes

    + * + * A snapshot delete will be executed as soon as there are no more shard snapshots or snapshot finalizations executing running for a given + * repository. Before a delete is executed on the repository it will be set to state + * {@link org.elasticsearch.cluster.SnapshotDeletionsInProgress.State#STARTED}. If it cannot be executed when it is received it will be + * set to state {@link org.elasticsearch.cluster.SnapshotDeletionsInProgress.State#WAITING} initially. + * If a delete is received for a given repository while there is already an ongoing delete for the same repository, there are two possible + * scenarios: + * 1. If the delete is in state {@code META_DATA} (i.e. already running on the repository) then the new delete will be added in state + * {@code WAITING} and will be executed after the current delete. The only exception here would be the case where the new delete covers + * the exact same snapshots as the already running delete. In this case no new delete operation is added and second delete request will + * simply wait for the existing delete to return. + * 2. If the existing delete is in state {@code WAITING} then the existing + * {@link org.elasticsearch.cluster.SnapshotDeletionsInProgress.Entry} in the cluster state will be updated to cover both the snapshots + * in the existing delete as well as additional snapshots that may be found in the second delete request. + * + * In either of the above scenarios, in-progress snapshots will be aborted in the same cluster state update that adds a delete to the + * cluster state, if a delete applies to them. + * + * If a snapshot request is received while there already is a delete in the cluster state for the same repository, that snapshot will not + * start doing any shard snapshots until the delete has been executed. */ package org.elasticsearch.snapshots; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 05f9e764502f7..936c13b8a8fb9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.monitor.process.ProcessStats; import org.elasticsearch.node.AdaptiveSelectionStats; import org.elasticsearch.node.ResponseCollectorService; -import org.elasticsearch.script.ScriptCacheStats; +import org.elasticsearch.script.ScriptContextStats; import org.elasticsearch.script.ScriptStats; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -46,9 +46,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; @@ -241,11 +243,35 @@ public void testSerialization() throws IOException { } } ScriptStats scriptStats = nodeStats.getScriptStats(); + ScriptStats deserializedScriptStats = deserializedNodeStats.getScriptStats(); if (scriptStats == null) { - assertNull(deserializedNodeStats.getScriptStats()); + assertNull(deserializedScriptStats); } else { - assertEquals(scriptStats.getCacheEvictions(), deserializedNodeStats.getScriptStats().getCacheEvictions()); - assertEquals(scriptStats.getCompilations(), deserializedNodeStats.getScriptStats().getCompilations()); + List deserialized = deserializedScriptStats.getContextStats(); + long evictions = 0; + long limited = 0; + long compilations = 0; + List stats = scriptStats.getContextStats(); + for (ScriptContextStats generatedStats: stats) { + List maybeDeserStats = deserialized.stream().filter( + s -> s.getContext().equals(generatedStats.getContext()) + ).collect(Collectors.toList()); + + assertEquals(1, maybeDeserStats.size()); + ScriptContextStats deserStats = maybeDeserStats.get(0); + + evictions += generatedStats.getCacheEvictions(); + assertEquals(generatedStats.getCacheEvictions(), deserStats.getCacheEvictions()); + + limited += generatedStats.getCompilationLimitTriggered(); + assertEquals(generatedStats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); + + compilations += generatedStats.getCompilations(); + assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); + } + assertEquals(evictions, scriptStats.getCacheEvictions()); + assertEquals(limited, scriptStats.getCompilationLimitTriggered()); + assertEquals(compilations, scriptStats.getCompilations()); } DiscoveryStats discoveryStats = nodeStats.getDiscoveryStats(); DiscoveryStats deserializedDiscoveryStats = deserializedNodeStats.getDiscoveryStats(); @@ -312,34 +338,6 @@ public void testSerialization() throws IOException { assertEquals(aStats.responseTime, bStats.responseTime, 0.01); }); } - ScriptCacheStats scriptCacheStats = nodeStats.getScriptCacheStats(); - ScriptCacheStats deserializedScriptCacheStats = deserializedNodeStats.getScriptCacheStats(); - if (scriptCacheStats == null) { - assertNull(deserializedScriptCacheStats); - } else { - Map deserialized = deserializedScriptCacheStats.getContextStats(); - long evictions = 0; - long limited = 0; - long compilations = 0; - Map stats = scriptCacheStats.getContextStats(); - for (String context: stats.keySet()) { - ScriptStats deserStats = deserialized.get(context); - ScriptStats generatedStats = stats.get(context); - - evictions += generatedStats.getCacheEvictions(); - assertEquals(generatedStats.getCacheEvictions(), deserStats.getCacheEvictions()); - - limited += generatedStats.getCompilationLimitTriggered(); - assertEquals(generatedStats.getCompilationLimitTriggered(), deserStats.getCompilationLimitTriggered()); - - compilations += generatedStats.getCompilations(); - assertEquals(generatedStats.getCompilations(), deserStats.getCompilations()); - } - ScriptStats sum = deserializedScriptCacheStats.sum(); - assertEquals(evictions, sum.getCacheEvictions()); - assertEquals(limited, sum.getCompilationLimitTriggered()); - assertEquals(compilations, sum.getCompilations()); - } } } } @@ -454,8 +452,21 @@ public static NodeStats createNodeStats() { } allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray); } - ScriptStats scriptStats = frequently() ? - new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null; + ScriptStats scriptStats = null; + if (frequently()) { + int numContents = randomIntBetween(0, 20); + List stats = new ArrayList<>(numContents); + HashSet contexts = new HashSet<>(); + for (int i = 0; i < numContents; i++) { + stats.add(new ScriptContextStats( + randomValueOtherThanMany(contexts::contains, () -> randomAlphaOfLength(12)), + randomLongBetween(0, 1024), + randomLongBetween(0, 1024), + randomLongBetween(0, 1024)) + ); + } + scriptStats = new ScriptStats(stats); + } DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats( randomBoolean() @@ -514,20 +525,10 @@ public static NodeStats createNodeStats() { } adaptiveSelectionStats = new AdaptiveSelectionStats(nodeConnections, nodeStats); } - ScriptCacheStats scriptCacheStats = null; - if (frequently()) { - int numContents = randomIntBetween(0, 20); - Map stats = new HashMap<>(numContents); - for (int i = 0; i < numContents; i++) { - String context = randomValueOtherThanMany(stats::containsKey, () -> randomAlphaOfLength(12)); - stats.put(context, new ScriptStats(randomLongBetween(0, 1024), randomLongBetween(0, 1024), randomLongBetween(0, 1024))); - } - scriptCacheStats = new ScriptCacheStats(stats); - } //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo, transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, - ingestStats, adaptiveSelectionStats, scriptCacheStats); + ingestStats, adaptiveSelectionStats, null); } private IngestStats.Stats getPipelineStats(List pipelineStats, String id) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java index a96879e043757..71ba0e0d2d17e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.action.admin.indices.datastream; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction.Request; @@ -29,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataDeleteIndexService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; @@ -46,6 +46,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; @@ -140,11 +141,18 @@ private SnapshotsInProgress.Entry createEntry(String dataStreamName, String repo public void testDeleteNonexistentDataStream() { final String dataStreamName = "my-data-stream"; - ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); + String[] dataStreamNames = {"foo", "bar", "baz", "eggplant"}; + ClusterState cs = getClusterStateWithDataStreams(List.of( + new Tuple<>(dataStreamNames[0], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[1], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[2], randomIntBetween(1, 3)), + new Tuple<>(dataStreamNames[3], randomIntBetween(1, 3)) + ), List.of()); DeleteDataStreamAction.Request req = new DeleteDataStreamAction.Request(new String[]{dataStreamName}); - ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, - () -> DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req)); - assertThat(e.getMessage(), containsString("data_streams matching [" + dataStreamName + "] not found")); + ClusterState newState = DeleteDataStreamAction.TransportAction.removeDataStream(getMetadataDeleteIndexService(), cs, req); + assertThat(newState.metadata().dataStreams().size(), equalTo(cs.metadata().dataStreams().size())); + assertThat(newState.metadata().dataStreams().keySet(), + containsInAnyOrder(cs.metadata().dataStreams().keySet().toArray(Strings.EMPTY_ARRAY))); } @SuppressWarnings("unchecked") diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java index dc449f85908d4..bc80ea02ab318 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/datastream/GetDataStreamsRequestTests.java @@ -18,21 +18,19 @@ */ package org.elasticsearch.action.admin.indices.datastream; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction.Request; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.util.List; -import java.util.Map; -import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamRequestTests.getClusterStateWithDataStreams; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -45,16 +43,20 @@ protected Writeable.Reader instanceReader() { @Override protected Request createTestInstance() { - final String searchParameter; + final String[] searchParameter; switch (randomIntBetween(1, 4)) { case 1: - searchParameter = randomAlphaOfLength(8); + searchParameter = generateRandomStringArray(3, 8, false, false); break; case 2: - searchParameter = randomAlphaOfLength(8) + "*"; + String[] parameters = generateRandomStringArray(3, 8, false, false); + for (int k = 0; k < parameters.length; k++) { + parameters[k] = parameters[k] + "*"; + } + searchParameter = parameters; break; case 3: - searchParameter = "*"; + searchParameter = new String[]{"*"}; break; default: searchParameter = null; @@ -65,58 +67,75 @@ protected Request createTestInstance() { public void testGetDataStream() { final String dataStreamName = "my-data-stream"; - IndexMetadata idx = DataStreamTestHelper.createFirstBackingIndex(dataStreamName).build(); - DataStream existingDataStream = - new DataStream(dataStreamName, createTimestampField("@timestamp"), List.of(idx.getIndex())); - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metadata(Metadata.builder().dataStreams(Map.of(dataStreamName, existingDataStream)).build()).build(); - GetDataStreamAction.Request req = new GetDataStreamAction.Request(dataStreamName); - List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, req); + ClusterState cs = getClusterStateWithDataStreams( + List.of(new Tuple<>(dataStreamName, 1)), List.of()); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamName}); + List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); assertThat(dataStreams.size(), equalTo(1)); assertThat(dataStreams.get(0).getName(), equalTo(dataStreamName)); } public void testGetDataStreamsWithWildcards() { final String[] dataStreamNames = {"my-data-stream", "another-data-stream"}; - IndexMetadata idx1 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[0]).build(); - IndexMetadata idx2 = DataStreamTestHelper.createFirstBackingIndex(dataStreamNames[1]).build(); - - DataStream ds1 = new DataStream(dataStreamNames[0], createTimestampField("@timestamp"), List.of(idx1.getIndex())); - DataStream ds2 = new DataStream(dataStreamNames[1], createTimestampField("@timestamp"), List.of(idx2.getIndex())); - ClusterState cs = ClusterState.builder(new ClusterName("_name")) - .metadata(Metadata.builder().dataStreams( - Map.of(dataStreamNames[0], ds1, dataStreamNames[1], ds2)).build()) - .build(); - - GetDataStreamAction.Request req = new GetDataStreamAction.Request(dataStreamNames[1].substring(0, 5) + "*"); - List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, req); + ClusterState cs = getClusterStateWithDataStreams( + List.of(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), List.of()); + + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamNames[1].substring(0, 5) + "*"}); + List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); assertThat(dataStreams.size(), equalTo(1)); assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); - req = new GetDataStreamAction.Request("*"); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, req); + req = new GetDataStreamAction.Request(new String[]{"*"}); + dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); assertThat(dataStreams.size(), equalTo(2)); assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); - req = new GetDataStreamAction.Request((String) null); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, req); + req = new GetDataStreamAction.Request((String[]) null); + dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); assertThat(dataStreams.size(), equalTo(2)); assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); - req = new GetDataStreamAction.Request("matches-none*"); - dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, req); + req = new GetDataStreamAction.Request(new String[]{"matches-none*"}); + dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); assertThat(dataStreams.size(), equalTo(0)); } + public void testGetDataStreamsWithoutWildcards() { + final String[] dataStreamNames = {"my-data-stream", "another-data-stream"}; + ClusterState cs = getClusterStateWithDataStreams( + List.of(new Tuple<>(dataStreamNames[0], 1), new Tuple<>(dataStreamNames[1], 1)), List.of()); + + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamNames[0], dataStreamNames[1]}); + List dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(2)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + assertThat(dataStreams.get(1).getName(), equalTo(dataStreamNames[0])); + + req = new GetDataStreamAction.Request(new String[]{dataStreamNames[1]}); + dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[1])); + + req = new GetDataStreamAction.Request(new String[]{dataStreamNames[0]}); + dataStreams = GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req); + assertThat(dataStreams.size(), equalTo(1)); + assertThat(dataStreams.get(0).getName(), equalTo(dataStreamNames[0])); + + GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(new String[]{"foo"}); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, + () -> GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req2)); + assertThat(e.getMessage(), containsString("no such index [foo]")); + } + public void testGetNonexistentDataStream() { final String dataStreamName = "my-data-stream"; ClusterState cs = ClusterState.builder(new ClusterName("_name")).build(); - GetDataStreamAction.Request req = new GetDataStreamAction.Request(dataStreamName); - ResourceNotFoundException e = expectThrows(ResourceNotFoundException.class, - () -> GetDataStreamAction.TransportAction.getDataStreams(cs, req)); - assertThat(e.getMessage(), containsString("data_stream matching [" + dataStreamName + "] not found")); + GetDataStreamAction.Request req = new GetDataStreamAction.Request(new String[]{dataStreamName}); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, + () -> GetDataStreamAction.TransportAction.getDataStreams(cs, new IndexNameExpressionResolver(), req)); + assertThat(e.getMessage(), containsString("no such index [" + dataStreamName + "]")); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 59624ec7e8b5b..2cde6755c02cb 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -60,10 +60,11 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentFieldMappers; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; @@ -81,7 +82,7 @@ import java.util.Locale; import java.util.Map; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests.generateMapping; +import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -550,11 +551,14 @@ public void testRolloverClusterStateForDataStream() throws Exception { ThreadPool testThreadPool = new TestThreadPool(getTestName()); try { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0)); - TimestampFieldMapper.Builder fieldBuilder = new TimestampFieldMapper.Builder(); - fieldBuilder.setPath("@timestamp"); DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder("@timestamp").build(builderContext); + MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class); + when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp"); + MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); + when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); + when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType); DocumentFieldMappers documentFieldMappers = - new DocumentFieldMappers(List.of(fieldBuilder.build(builderContext), dateFieldMapper), List.of(), new StandardAnalyzer()); + new DocumentFieldMappers(List.of(mockedTimestampField, dateFieldMapper), List.of(), new StandardAnalyzer()); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); @@ -564,7 +568,8 @@ public void testRolloverClusterStateForDataStream() throws Exception { DocumentMapper documentMapper = mock(DocumentMapper.class); when(documentMapper.mappers()).thenReturn(documentFieldMappers); when(documentMapper.type()).thenReturn("_doc"); - CompressedXContent mapping = new CompressedXContent(generateMapping(dataStream.getTimeStampField().getName())); + CompressedXContent mapping = + new CompressedXContent("{\"_doc\":" + generateMapping(dataStream.getTimeStampField().getName(), "date") + "}"); when(documentMapper.mappingSource()).thenReturn(mapping); RoutingFieldMapper routingFieldMapper = mock(RoutingFieldMapper.class); when(routingFieldMapper.required()).thenReturn(false); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 3939cd98f5f88..e182833c4902f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -31,11 +31,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -120,7 +122,8 @@ private void indicesThatCannotBeCreatedTestCase(Set expected, final ExecutorService direct = EsExecutors.newDirectExecutorService(); when(threadPool.executor(anyString())).thenReturn(direct); TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, - null, null, mock(ActionFilters.class), null, null, new WriteMemoryLimits()) { + null, null, mock(ActionFilters.class), null, null, + new IndexingPressure(Settings.EMPTY)) { @Override void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, AtomicArray responses, Map indicesThatCannotBeCreated) { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 129a98459fdec..8e7af9de16adb 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -52,6 +52,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -143,7 +144,7 @@ null, new ActionFilters(Collections.emptySet()), null, new AutoCreateIndex( SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), new IndexNameExpressionResolver() - ), new WriteMemoryLimits() + ), new IndexingPressure(SETTINGS) ); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 586545ac26363..a22db5e9d30b1 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -82,7 +83,7 @@ class TestTransportBulkAction extends TransportBulkAction { super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null, new ActionFilters(Collections.emptySet()), new Resolver(), new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver()), - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 89408f3d17f91..44007488433db 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -233,7 +234,7 @@ static class TestTransportBulkAction extends TransportBulkAction { actionFilters, indexNameExpressionResolver, autoCreateIndex, - new WriteMemoryLimits(), + new IndexingPressure(Settings.EMPTY), relativeTimeProvider); } diff --git a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java index 6861fb4aa89a2..04845547c29ec 100644 --- a/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/resync/TransportResyncReplicationActionTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; @@ -145,7 +145,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { final TransportResyncReplicationAction action = new TransportResyncReplicationAction(Settings.EMPTY, transportService, clusterService, indexServices, threadPool, shardStateAction, new ActionFilters(new HashSet<>()), - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); assertThat(action.globalBlockLevel(), nullValue()); assertThat(action.indexBlockLevel(), nullValue()); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java index c1ef04b10d43d..68ac15cba23ba 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -367,7 +367,7 @@ protected TestAction(boolean withDocumentFailureOnPrimary, boolean withDocumentF new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()), TransportWriteActionTests.this.clusterService, null, null, null, new ActionFilters(new HashSet<>()), TestRequest::new, TestRequest::new, ThreadPool.Names.SAME, false, - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); this.withDocumentFailureOnPrimary = withDocumentFailureOnPrimary; this.withDocumentFailureOnReplica = withDocumentFailureOnReplica; } @@ -377,7 +377,7 @@ protected TestAction(Settings settings, String actionName, TransportService tran super(settings, actionName, transportService, clusterService, mockIndicesService(clusterService), threadPool, shardStateAction, new ActionFilters(new HashSet<>()), TestRequest::new, TestRequest::new, ThreadPool.Names.SAME, false, - new WriteMemoryLimits()); + new IndexingPressure(settings)); this.withDocumentFailureOnPrimary = false; this.withDocumentFailureOnReplica = false; } diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index a5674994b61b0..5ecf765f82028 100644 --- a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -152,14 +152,11 @@ public void testFillDiskUsage() { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null, - null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null, null), new NodeStats(new DiscoveryNode("node_2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null, - null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null, null), new NodeStats(new DiscoveryNode("node_3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null, - null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1"); @@ -196,14 +193,11 @@ public void testFillDiskUsageSomeInvalidValues() { }; List nodeStats = Arrays.asList( new NodeStats(new DiscoveryNode("node_1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null, - null), + null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null, null, null), new NodeStats(new DiscoveryNode("node_2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null, - null), + null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null, null, null), new NodeStats(new DiscoveryNode("node_3", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null, - null) + null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null, null, null) ); InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages); DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1"); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java index c50b209a3ff15..f2400e1fc2f33 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java @@ -169,18 +169,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req } public void testFailsNodeThatDoesNotRespond() { - final Builder settingsBuilder = Settings.builder(); - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10)); - } - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms"); - } - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 100000) + "ms"); - } - final Settings settings = settingsBuilder.build(); - + final Settings settings = randomSettings(); testBehaviourOfFailingNode(settings, () -> null, "followers check retry count exceeded", (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis() @@ -189,15 +178,7 @@ public void testFailsNodeThatDoesNotRespond() { } public void testFailsNodeThatRejectsCheck() { - final Builder settingsBuilder = Settings.builder(); - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10)); - } - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms"); - } - final Settings settings = settingsBuilder.build(); - + final Settings settings = randomSettings(); testBehaviourOfFailingNode(settings, () -> { throw new ElasticsearchException("simulated exception"); }, @@ -207,15 +188,7 @@ public void testFailsNodeThatRejectsCheck() { } public void testFailureCounterResetsOnSuccess() { - final Builder settingsBuilder = Settings.builder(); - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(2, 10)); - } - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms"); - } - final Settings settings = settingsBuilder.build(); - + final Settings settings = randomSettings(); final int retryCount = FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings); final int maxRecoveries = randomIntBetween(3, 10); @@ -298,16 +271,7 @@ public String toString() { } public void testFailsNodeThatIsUnhealthy() { - final Builder settingsBuilder = Settings.builder(); - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10)); - } - if (randomBoolean()) { - settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms"); - } - final Settings settings = settingsBuilder.build(); - - testBehaviourOfFailingNode(settings, () -> { + testBehaviourOfFailingNode(randomSettings(), () -> { throw new NodeHealthCheckFailureException("non writable exception"); }, "health check failed", 0, () -> new StatusInfo(HEALTHY, "healthy-info")); } @@ -322,7 +286,7 @@ private void testBehaviourOfFailingNode(Settings testSettings, Supplier attributes, Version.CURRENT); } + private static Settings randomSettings() { + final Builder settingsBuilder = Settings.builder(); + if (randomBoolean()) { + settingsBuilder.put(FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), randomIntBetween(1, 10)); + } + if (randomBoolean()) { + settingsBuilder.put(FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), randomIntBetween(100, 100000) + "ms"); + } + if (randomBoolean()) { + settingsBuilder.put(FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), randomIntBetween(1, 100000) + "ms"); + } + return settingsBuilder.build(); + } + private static class ExpectsSuccess implements TransportResponseHandler { private final AtomicBoolean responseReceived = new AtomicBoolean(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index a3128a6903319..ccca14b148e51 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -25,17 +25,14 @@ import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; -import java.io.IOException; import java.util.List; import java.util.Map; import static org.elasticsearch.cluster.DataStreamTestHelper.createFirstBackingIndex; import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; +import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -152,34 +149,6 @@ public static ClusterState createDataStream(final String dataStreamName) throws return MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req); } - public void testValidateTimestampFieldMapping() throws Exception { - String mapping = generateMapping("@timestamp", "date"); - validateTimestampFieldMapping("@timestamp", createMapperService(mapping)); - mapping = generateMapping("@timestamp", "date_nanos"); - validateTimestampFieldMapping("@timestamp", createMapperService(mapping)); - } - - public void testValidateTimestampFieldMappingNoFieldMapping() { - Exception e = expectThrows(IllegalArgumentException.class, - () -> validateTimestampFieldMapping("@timestamp", createMapperService("{}"))); - assertThat(e.getMessage(), - equalTo("[_timestamp] meta field doesn't point to data stream timestamp field [@timestamp]")); - - String mapping = generateMapping("@timestamp2", "date"); - e = expectThrows(IllegalArgumentException.class, - () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping))); - assertThat(e.getMessage(), - equalTo("[_timestamp] meta field doesn't point to data stream timestamp field [@timestamp]")); - } - - public void testValidateTimestampFieldMappingInvalidFieldType() { - String mapping = generateMapping("@timestamp", "keyword"); - Exception e = expectThrows(IllegalArgumentException.class, - () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping))); - assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] is of type [keyword], " + - "but [date,date_nanos] is expected")); - } - private static MetadataCreateIndexService getMetadataCreateIndexService() throws Exception { MetadataCreateIndexService s = mock(MetadataCreateIndexService.class); when(s.applyCreateIndexRequest(any(ClusterState.class), any(CreateIndexClusterStateUpdateRequest.class), anyBoolean())) @@ -203,35 +172,4 @@ private static MetadataCreateIndexService getMetadataCreateIndexService() throws return s; } - public static String generateMapping(String timestampFieldName) { - return generateMapping(timestampFieldName, "date"); - } - - static String generateMapping(String timestampFieldName, String type) { - return "{\n" + - " \"_timestamp\": {\n" + - " \"path\": \"" + timestampFieldName + "\"\n" + - " }," + - " \"properties\": {\n" + - " \"" + timestampFieldName + "\": {\n" + - " \"type\": \"" + type + "\"\n" + - " }\n" + - " }\n" + - " }"; - } - - MapperService createMapperService(String mapping) throws IOException { - String indexName = "test"; - IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - .settings(Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) - .putMapping(mapping) - .build(); - MapperService mapperService = - MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), Settings.EMPTY, indexName); - mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE); - return mapperService; - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 5636ceb1010ca..065bdfc2b445a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.cluster.metadata; import com.fasterxml.jackson.core.JsonParseException; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -34,21 +36,31 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -76,6 +88,12 @@ import static org.hamcrest.Matchers.matchesRegex; public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(DummyPlugin.class); + } + public void testIndexTemplateInvalidNumberOfShards() { PutRequest request = new PutRequest("test", "test_shards"); request.patterns(singletonList("test_shards*")); @@ -1281,4 +1299,83 @@ public static void assertTemplatesEqual(ComposableIndexTemplate actual, Composab } } } + + // Composable index template with data_stream definition need _timestamp meta field mapper, + // this is a dummy impl, so that tests don't fail with the fact that the _timestamp field can't be found. + // (tests using this dummy impl doesn't test the _timestamp validation, but need it to tests other functionality) + public static class DummyPlugin extends Plugin implements MapperPlugin { + + @Override + public Map getMetadataMappers() { + return Map.of("_data_stream_timestamp", new MetadataFieldMapper.TypeParser() { + + @Override + public MetadataFieldMapper.Builder parse(String name, + Map node, + ParserContext parserContext) throws MapperParsingException { + String path = (String) node.remove("path"); + return new MetadataFieldMapper.Builder(name, new FieldType()) { + @Override + public MetadataFieldMapper build(Mapper.BuilderContext context) { + return newInstance(path); + } + }; + } + + @Override + public MetadataFieldMapper getDefault(ParserContext parserContext) { + return newInstance(null); + } + + MetadataFieldMapper newInstance(String path) { + FieldType fieldType = new FieldType(); + fieldType.freeze(); + MappedFieldType mappedFieldType = + new MappedFieldType("_data_stream_timestamp", false, false, TextSearchInfo.NONE, Map.of()) { + @Override + public String typeName() { + return "_data_stream_timestamp"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return null; + } + }; + return new MetadataFieldMapper(fieldType, mappedFieldType) { + @Override + public void preParse(ParseContext context) throws IOException { + + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (path == null) { + return builder; + } + + builder.startObject(simpleName()); + builder.field("path", path); + return builder.endObject(); + } + + @Override + protected String contentType() { + return "_data_stream_timestamp"; + } + }; + } + }); + } + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 060e34a1297e6..c3527d5b6432c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -117,10 +117,10 @@ public void testSnapshotDeletionsInProgressSerialization() throws Exception { ClusterState.Builder builder = ClusterState.builder(ClusterState.EMPTY_STATE) .putCustom(SnapshotDeletionsInProgress.TYPE, - SnapshotDeletionsInProgress.of(List.of( - new SnapshotDeletionsInProgress.Entry( - Collections.singletonList(new SnapshotId("snap1", UUIDs.randomBase64UUID())), "repo1", - randomNonNegativeLong(), randomNonNegativeLong())))); + SnapshotDeletionsInProgress.of(List.of( + new SnapshotDeletionsInProgress.Entry( + Collections.singletonList(new SnapshotId("snap1", UUIDs.randomBase64UUID())), "repo1", + randomNonNegativeLong(), randomNonNegativeLong(), SnapshotDeletionsInProgress.State.STARTED)))); if (includeRestore) { builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress.Builder().add( diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java index 4c53d40fb75db..f9c7da2167088 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryParserTests.java @@ -26,12 +26,17 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParseException; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.test.ESTestCase; +import java.util.List; +import java.util.Map; + /** * Tests for {@link GeometryParser} */ @@ -173,4 +178,45 @@ public void testUnsupportedValueParsing() throws Exception { assertEquals("shape must be an object consisting of type and coordinates", ex.getMessage()); } } + + public void testBasics() { + GeometryParser parser = new GeometryParser(true, randomBoolean(), randomBoolean()); + // point + Point expectedPoint = new Point(-122.084110, 37.386637); + testBasics(parser, Map.of("lat", 37.386637, "lon", -122.084110), expectedPoint); + testBasics(parser, "37.386637, -122.084110", expectedPoint); + testBasics(parser, "POINT (-122.084110 37.386637)", expectedPoint); + testBasics(parser, Map.of("type", "Point", "coordinates", List.of(-122.084110, 37.386637)), expectedPoint); + testBasics(parser, List.of(-122.084110, 37.386637), expectedPoint); + // line + Line expectedLine = new Line(new double[] {0, 1}, new double[] {0, 1}); + testBasics(parser, "LINESTRING(0 0, 1 1)", expectedLine); + testBasics(parser, Map.of("type", "LineString", "coordinates", List.of(List.of(0, 0), List.of(1, 1))), expectedLine); + // polygon + Polygon expectedPolygon = new Polygon(new LinearRing(new double[] {0, 1, 1, 0, 0}, new double[] {0, 0, 1, 1, 0})); + testBasics(parser, "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon); + testBasics(parser, Map.of("type", "Polygon", "coordinates", + List.of( + List.of(List.of(0, 0), List.of(1, 0), List.of(1, 1), List.of(0, 1), List.of(0, 0))) + ), + expectedPolygon); + // geometry collection + testBasics(parser, + List.of( + List.of(-122.084110, 37.386637), + "37.386637, -122.084110", + "POINT (-122.084110 37.386637)", + Map.of("type", "Point", "coordinates", List.of(-122.084110, 37.386637)), + Map.of("type", "LineString", "coordinates", List.of(List.of(0, 0), List.of(1, 1))), + "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" + ), + new GeometryCollection<>(List.of(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon)) + ); + expectThrows(ElasticsearchParseException.class, () -> testBasics(parser, "not a geometry", null)); + } + + private void testBasics(GeometryParser parser, Object value, Geometry expected) { + Geometry geometry = parser.parseGeometry(value); + assertEquals(expected, geometry); + } } diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java index 6a303449ce1c1..e7ebbbf102e1c 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/ReleasableLockTests.java @@ -20,13 +20,25 @@ package org.elasticsearch.common.util.concurrent; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; public class ReleasableLockTests extends ESTestCase { @@ -79,10 +91,10 @@ public void testIsHeldByCurrentThread() throws BrokenBarrierException, Interrupt } private void acquire(final ReleasableLock lockToAcquire, final ReleasableLock otherLock) { - try (@SuppressWarnings("unused") Releasable outer = lockToAcquire.acquire()) { + try (@SuppressWarnings("unused") Releasable outer = randomAcquireMethod(lockToAcquire)) { assertTrue(lockToAcquire.isHeldByCurrentThread()); assertFalse(otherLock.isHeldByCurrentThread()); - try (@SuppressWarnings("unused") Releasable inner = lockToAcquire.acquire()) { + try (@SuppressWarnings("unused") Releasable inner = randomAcquireMethod(lockToAcquire)) { assertTrue(lockToAcquire.isHeldByCurrentThread()); assertFalse(otherLock.isHeldByCurrentThread()); } @@ -94,4 +106,56 @@ private void acquire(final ReleasableLock lockToAcquire, final ReleasableLock ot assertFalse(otherLock.isHeldByCurrentThread()); } + private ReleasableLock randomAcquireMethod(ReleasableLock lock) { + if (randomBoolean()) { + return lock.acquire(); + } else { + try { + ReleasableLock releasableLock = lock.tryAcquire(TimeValue.timeValueSeconds(30)); + assertThat(releasableLock, notNullValue()); + return releasableLock; + } catch (InterruptedException e) { + throw new AssertionError(e); + } + } + } + + public void testTryAcquire() throws Exception { + ReleasableLock lock = new ReleasableLock(new ReentrantLock()); + int numberOfThreads = randomIntBetween(1, 10); + CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads); + AtomicInteger lockedCounter = new AtomicInteger(); + int timeout = randomFrom(0, 5, 10); + List threads = + IntStream.range(0, numberOfThreads).mapToObj(i -> new Thread(() -> { + try { + barrier.await(10, TimeUnit.SECONDS); + try (ReleasableLock locked = lock.tryAcquire(TimeValue.timeValueMillis(timeout))) { + if (locked != null) { + lockedCounter.incrementAndGet(); + } + } + } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { + throw new AssertionError(e); + } + })).collect(Collectors.toList()); + threads.forEach(Thread::start); + try (ReleasableLock locked = randomBoolean() ? lock.acquire() : null) { + barrier.await(10, TimeUnit.SECONDS); + for (Thread thread : threads) { + thread.join(10000); + } + threads.forEach(t -> assertThat(t.isAlive(), is(false))); + + if (locked != null) { + assertThat(lockedCounter.get(), equalTo(0)); + } else { + assertThat(lockedCounter.get(), greaterThanOrEqualTo(1)); + } + } + + try (ReleasableLock locked = lock.tryAcquire(TimeValue.ZERO)) { + assertThat(locked, notNullValue()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java b/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java index 02aaaf8b0202d..4782baf987300 100644 --- a/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java +++ b/server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java @@ -121,7 +121,7 @@ List startCluster(int numberOfNodes) { return nodes; } - static final Settings DEFAULT_SETTINGS = Settings.builder() + public static final Settings DEFAULT_SETTINGS = Settings.builder() .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "5s") // for hitting simulated network failures quickly .put(LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) // for hitting simulated network failures quickly .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "5s") // for hitting simulated network failures quickly diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index c7883c8cd7603..a6bdf6cfacef9 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -122,7 +122,7 @@ public > IFD getForField(String type, String field } else if (type.equals("geo_point")) { fieldType = new GeoPointFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType(); } else if (type.equals("binary")) { - fieldType = new BinaryFieldMapper.Builder(fieldName).docValues(docValues).build(context).fieldType(); + fieldType = new BinaryFieldMapper.Builder(fieldName, docValues).build(context).fieldType(); } else { throw new UnsupportedOperationException(type); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index dad6ac927a667..2de5ab8f75d79 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -32,28 +32,18 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.Set; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class BinaryFieldMapperTests extends FieldMapperTestCase { - - @Override - protected Set unsupportedProperties() { - return Set.of("analyzer", "eager_global_ordinals", "norms", "similarity", "index"); - } - - @Override - protected BinaryFieldMapper.Builder newBuilder() { - return new BinaryFieldMapper.Builder("binary"); - } +public class BinaryFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java deleted file mode 100644 index 7f84a3d36f8ae..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldTypeTests.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import java.util.Map; - -public class BinaryFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new BinaryFieldMapper.BinaryFieldType(name, true, meta); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 564be1efdca85..0d5ed6d24803a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -27,6 +27,9 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -41,25 +44,21 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.Set; import static org.hamcrest.Matchers.containsString; -public class BooleanFieldMapperTests extends FieldMapperTestCase { +public class BooleanFieldMapperTests extends ESSingleNodeTestCase { + private IndexService indexService; private DocumentMapperParser parser; - @Override - protected Set unsupportedProperties() { - return Set.of("analyzer", "similarity"); - } - @Before public void setup() { indexService = createIndex("test"); @@ -285,9 +284,13 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } - @Override - protected BooleanFieldMapper.Builder newBuilder() { - return new BooleanFieldMapper.Builder("boolean"); + public void testBoosts() throws Exception { + String mapping = "{\"_doc\":{\"properties\":{\"field\":{\"type\":\"boolean\",\"boost\":2.0}}}}"; + DocumentMapper mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); + assertEquals(mapping, mapper.mappingSource().toString()); + + MappedFieldType ft = indexService.mapperService().fieldType("field"); + assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java index 0eb3b5776e630..d64f382209355 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldTypeTests.java @@ -22,13 +22,8 @@ import org.apache.lucene.search.TermQuery; import java.util.Collections; -import java.util.Map; -public class BooleanFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new BooleanFieldMapper.BooleanFieldType(name, true, true, meta); - } +public class BooleanFieldTypeTests extends FieldTypeTestCase { public void testValueFormat() { MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 4ea3a5d0beef8..274c8efd0e722 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; @@ -43,20 +42,15 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.search.suggest.completion.context.ContextBuilder; -import org.elasticsearch.search.suggest.completion.context.ContextMappings; +import org.elasticsearch.test.ESSingleNodeTestCase; import org.hamcrest.FeatureMatcher; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.hamcrest.core.CombinableMatcher; -import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.Map; -import java.util.Set; import java.util.function.Function; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -67,31 +61,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -public class CompletionFieldMapperTests extends FieldMapperTestCase { - - @Before - public void addModifiers() { - addBooleanModifier("preserve_separators", false, CompletionFieldMapper.Builder::preserveSeparators); - addBooleanModifier("preserve_position_increments", false, CompletionFieldMapper.Builder::preservePositionIncrements); - addModifier("context_mappings", false, (a, b) -> { - ContextMappings contextMappings = new ContextMappings(Arrays.asList(ContextBuilder.category("foo").build(), - ContextBuilder.geo("geo").build())); - a.contextMappings(contextMappings); - }); - } - - @Override - protected Set unsupportedProperties() { - return Set.of("doc_values", "index"); - } - - @Override - protected CompletionFieldMapper.Builder newBuilder() { - CompletionFieldMapper.Builder builder = new CompletionFieldMapper.Builder("completion"); - builder.indexAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); - builder.searchAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); - return builder; - } +public class CompletionFieldMapperTests extends ESSingleNodeTestCase { public void testDefaultConfiguration() throws IOException { String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1") @@ -175,7 +145,7 @@ public void testTypeParsing() throws Exception { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); XContentBuilder builder = jsonBuilder().startObject(); - fieldMapper.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); + fieldMapper.toXContent(builder, new ToXContent.MapParams(Map.of("include_defaults", "true"))).endObject(); builder.close(); Map serializedMap = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder)).map(); Map configMap = (Map) serializedMap.get("completion"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index e780516237aee..d551f7e5f7563 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -56,16 +56,8 @@ import java.time.Instant; import java.time.ZoneOffset; import java.util.Collections; -import java.util.Map; -public class DateFieldTypeTests extends FieldTypeTestCase { - - - @Override - protected DateFieldType createDefaultFieldType(String name, Map meta) { - return new DateFieldType(name, true, true, DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, meta); - } +public class DateFieldTypeTests extends FieldTypeTestCase { private static final long nowInMillis = 0; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index d77db2bbcffe1..38db532d9fb0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -76,15 +76,6 @@ static class FakeFieldType extends TermBasedFieldType { super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - FakeFieldType(FakeFieldType other) { - super(other); - } - - @Override - public MappedFieldType clone() { - return new FakeFieldType(this); - } - @Override public String typeName() { return "fake"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 9ab3ecabfd01e..d8b8b0d213a3b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -126,15 +126,6 @@ static class ExternalFieldType extends TermBasedFieldType { super(name, indexed, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected ExternalFieldType(ExternalFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new ExternalFieldType(this); - } - @Override public String typeName() { return "faketype"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 7f191164fc34a..a86a4a6d63f77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -91,14 +91,6 @@ public FakeStringFieldType(String name, TextSearchInfo textSearchInfo) { setIndexAnalyzer(Lucene.STANDARD_ANALYZER); } - protected FakeStringFieldType(FakeStringFieldType ref) { - super(ref); - } - - public FakeStringFieldType clone() { - return new FakeStringFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index d0b71cc2d5c46..f8293d2a18682 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.EqualsHashCodeTestUtils; import java.util.Collections; @@ -61,14 +60,4 @@ public void testTermQuery() { IllegalStateException e = expectThrows(IllegalStateException.class, () -> fieldNamesFieldType.termQuery("field_name", null)); assertEquals("Cannot run [exists] queries if the [_field_names] field is disabled", e.getMessage()); } - - public void testHashcodeAndEquals() { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(new FieldNamesFieldMapper.FieldNamesFieldType(), - FieldNamesFieldMapper.FieldNamesFieldType::clone, - t -> { - FieldNamesFieldMapper.FieldNamesFieldType m = t.clone(); - m.setEnabled(false); - return m; - }); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java deleted file mode 100644 index 30f361674c7e5..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.elasticsearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType; - -import java.util.Map; - -public class GeoPointFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new GeoPointFieldType(name, true, true, meta); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java index f321c25de070a..602287030dce9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldTypeTests.java @@ -26,22 +26,8 @@ import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.test.EqualsHashCodeTestUtils; -import java.util.Map; - -public class IgnoredFieldTypeTests extends FieldTypeTestCase { - - @Override - public void testEquals() { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(IgnoredFieldMapper.IgnoredFieldType.INSTANCE, - IgnoredFieldMapper.IgnoredFieldType::clone); - } - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return IgnoredFieldMapper.IgnoredFieldType.INSTANCE; - } +public class IgnoredFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index fb592b1d4d7da..11dcfe4f23fa8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.EqualsHashCodeTestUtils; import java.util.function.Predicate; @@ -36,11 +35,6 @@ public class IndexFieldTypeTests extends ESTestCase { - public void testEqualsHashCode() { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(IndexFieldMapper.IndexFieldType.INSTANCE, - IndexFieldMapper.IndexFieldType::new); - } - public void testPrefixQuery() { MappedFieldType ft = IndexFieldMapper.IndexFieldType.INSTANCE; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java index fd7bc36e73fde..b075ed965ad85 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldTypeTests.java @@ -29,13 +29,8 @@ import java.net.InetAddress; import java.util.Arrays; import java.util.Collections; -import java.util.Map; -public class IpFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new IpFieldMapper.IpFieldType(name, true, true, meta); - } +public class IpFieldTypeTests extends FieldTypeTestCase { public void testValueFormat() throws Exception { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 80a6466038d6c..a6f0a7ee5f00b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -48,14 +48,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; -public class KeywordFieldTypeTests extends FieldTypeTestCase { - - @Override - protected KeywordFieldType createDefaultFieldType(String name, Map meta) { - return new KeywordFieldMapper.KeywordFieldType(name, true, true, meta); - } +public class KeywordFieldTypeTests extends FieldTypeTestCase { public void testIsFieldWithinQuery() throws IOException { KeywordFieldType ft = new KeywordFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java index 4fbfe92f9900e..b34c998656262 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldTypeTests.java @@ -21,13 +21,7 @@ import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.GeoShapeFieldType; -import java.util.Map; - -public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new GeoShapeFieldType(name, true, true, meta); - } +public class LegacyGeoShapeFieldTypeTests extends FieldTypeTestCase { /** * Test for {@link LegacyGeoShapeFieldMapper.GeoShapeFieldType#setStrategy(SpatialStrategy)} that checks diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NullValueTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NullValueTests.java index f38f83b6e418f..fce744553456a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NullValueTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NullValueTests.java @@ -2,6 +2,14 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; +import static org.hamcrest.Matchers.equalTo; /* * Licensed to Elasticsearch under one or more contributor @@ -22,14 +30,6 @@ * under the License. */ - -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.test.ESSingleNodeTestCase; - -import static org.hamcrest.Matchers.equalTo; - public class NullValueTests extends ESSingleNodeTestCase { public void testNullNullValue() throws Exception { IndexService indexService = createIndex("test", Settings.builder().build()); @@ -52,7 +52,9 @@ public void testNullNullValue() throws Exception { indexService.mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); fail("Test should have failed because [null_value] was null."); } catch (MapperParsingException e) { - assertThat(e.getMessage(), equalTo("Property [null_value] cannot be null.")); + assertThat(e.getMessage(), + either(equalTo("Property [null_value] cannot be null.")) + .or(containsString("must not have a [null] value"))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 407fdca4d3b51..30b131d4f5ada 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -61,14 +61,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -public class NumberFieldTypeTests extends FieldTypeTestCase { +public class NumberFieldTypeTests extends FieldTypeTestCase { NumberType type; @@ -77,11 +76,6 @@ public void pickType() { type = RandomPicks.randomFrom(random(), NumberFieldMapper.NumberType.values()); } - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new NumberFieldMapper.NumberFieldType(name, type, true, true, meta); - } - public void testEqualsWithDifferentNumberTypes() { NumberType type = randomFrom(NumberType.values()); NumberFieldType fieldType = new NumberFieldType("foo", type); @@ -94,7 +88,7 @@ public void testEqualsWithDifferentNumberTypes() { } public void testIsFieldWithinQuery() throws IOException { - MappedFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + MappedFieldType ft = new NumberFieldType("field", NumberType.INTEGER); // current impl ignores args and should always return INTERSECTS assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(null, randomDouble(), randomDouble(), randomBoolean(), randomBoolean(), null, null, null)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java new file mode 100644 index 0000000000000..f330474f2dd79 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -0,0 +1,376 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ParametrizedMapperTests extends ESSingleNodeTestCase { + + public static class TestPlugin extends Plugin implements MapperPlugin { + @Override + public Map getMappers() { + return Map.of("test_mapper", new TypeParser()); + } + } + + @Override + protected Collection> getPlugins() { + return List.of(TestPlugin.class); + } + + private static class StringWrapper { + final String name; + + private StringWrapper(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + StringWrapper that = (StringWrapper) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + } + + private static TestMapper toType(Mapper in) { + return (TestMapper) in; + } + + public static class Builder extends ParametrizedFieldMapper.Builder { + + final Parameter fixed + = Parameter.boolParam("fixed", false, m -> toType(m).fixed, true); + final Parameter fixed2 + = Parameter.boolParam("fixed2", false, m -> toType(m).fixed2, false) + .addDeprecatedName("fixed2_old"); + final Parameter variable + = Parameter.stringParam("variable", true, m -> toType(m).variable, "default").acceptsNull(); + final Parameter wrapper + = new Parameter<>("wrapper", true, new StringWrapper("default"), + (n, c, o) -> { + if (o == null) return null; + return new StringWrapper(o.toString()); + }, + m -> toType(m).wrapper).setSerializer((b, n, v) -> b.field(n, v.name)); + final Parameter intValue = Parameter.intParam("int_value", true, m -> toType(m).intValue, 5) + .setValidator(n -> { + if (n > 50) { + throw new IllegalArgumentException("Value of [n] cannot be greater than 50"); + } + }); + final Parameter analyzer + = Parameter.analyzerParam("analyzer", true, m -> toType(m).analyzer, Lucene.KEYWORD_ANALYZER); + final Parameter index = Parameter.boolParam("index", false, m -> toType(m).index, true); + + protected Builder(String name) { + super(name); + } + + @Override + protected List> getParameters() { + return List.of(fixed, fixed2, variable, index, wrapper, intValue, analyzer); + } + + @Override + public ParametrizedFieldMapper build(Mapper.BuilderContext context) { + return new TestMapper(name(), buildFullName(context), + multiFieldsBuilder.build(this, context), copyTo.build(), this); + } + } + + public static class TypeParser implements Mapper.TypeParser { + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = new Builder(name); + builder.parse(name, parserContext, node); + return builder; + } + } + + public static class TestMapper extends ParametrizedFieldMapper { + + private final boolean fixed; + private final boolean fixed2; + private final String variable; + private final StringWrapper wrapper; + private final int intValue; + private final NamedAnalyzer analyzer; + private final boolean index; + + protected TestMapper(String simpleName, String fullName, MultiFields multiFields, CopyTo copyTo, + ParametrizedMapperTests.Builder builder) { + super(simpleName, new KeywordFieldMapper.KeywordFieldType(fullName), multiFields, copyTo); + this.fixed = builder.fixed.getValue(); + this.fixed2 = builder.fixed2.getValue(); + this.variable = builder.variable.getValue(); + this.wrapper = builder.wrapper.getValue(); + this.intValue = builder.intValue.getValue(); + this.analyzer = builder.analyzer.getValue(); + this.index = builder.index.getValue(); + } + + @Override + public Builder getMergeBuilder() { + return new ParametrizedMapperTests.Builder(simpleName()).init(this); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + + } + + @Override + protected String contentType() { + return "test_mapper"; + } + } + + private static TestMapper fromMapping(String mapping, Version version) { + MapperService mapperService = mock(MapperService.class); + IndexAnalyzers indexAnalyzers = new IndexAnalyzers( + Map.of("_standard", Lucene.STANDARD_ANALYZER, + "_keyword", Lucene.KEYWORD_ANALYZER, + "default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), + Collections.emptyMap(), Collections.emptyMap()); + when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); + Mapper.TypeParser.ParserContext pc = new Mapper.TypeParser.ParserContext(s -> null, mapperService, s -> { + if (Objects.equals("keyword", s)) { + return new KeywordFieldMapper.TypeParser(); + } + if (Objects.equals("binary", s)) { + return new BinaryFieldMapper.TypeParser(); + } + return null; + }, version, () -> null); + return (TestMapper) new TypeParser() + .parse("field", XContentHelper.convertToMap(JsonXContent.jsonXContent, mapping, true), pc) + .build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0))); + } + + private static TestMapper fromMapping(String mapping) { + return fromMapping(mapping, Version.CURRENT); + } + + // defaults - create empty builder config, and serialize with and without defaults + public void testDefaults() throws IOException { + String mapping = "{\"type\":\"test_mapper\"}"; + TestMapper mapper = fromMapping(mapping); + + assertTrue(mapper.fixed); + assertEquals("default", mapper.variable); + + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + XContentBuilder builder = JsonXContent.contentBuilder(); + ToXContent.Params params = new ToXContent.MapParams(Map.of("include_defaults", "true")); + builder.startObject(); + mapper.toXContent(builder, params); + builder.endObject(); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"fixed\":true," + + "\"fixed2\":false,\"variable\":\"default\",\"index\":true," + + "\"wrapper\":\"default\",\"int_value\":5,\"analyzer\":\"_keyword\"}}", + Strings.toString(builder)); + } + + // merging - try updating 'fixed' and 'fixed2' should get an error, try updating 'variable' and verify update + public void testMerging() { + String mapping = "{\"type\":\"test_mapper\",\"fixed\":false}"; + TestMapper mapper = fromMapping(mapping); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + TestMapper badMerge = fromMapping("{\"type\":\"test_mapper\",\"fixed\":true,\"fixed2\":true}"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.merge(badMerge)); + String expectedError = "Mapper for [field] conflicts with existing mapper:\n" + + "\tCannot update parameter [fixed] from [false] to [true]\n" + + "\tCannot update parameter [fixed2] from [false] to [true]"; + assertEquals(expectedError, e.getMessage()); + + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); // original mapping is unaffected + + // TODO: should we have to include 'fixed' here? Or should updates take as 'defaults' the existing values? + TestMapper goodMerge = fromMapping("{\"type\":\"test_mapper\",\"fixed\":false,\"variable\":\"updated\"}"); + TestMapper merged = (TestMapper) mapper.merge(goodMerge); + + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); // original mapping is unaffected + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"fixed\":false,\"variable\":\"updated\"}}", Strings.toString(merged)); + + } + + // add multifield, verify, add second multifield, verify, overwrite second multifield + public void testMultifields() { + String mapping = "{\"type\":\"test_mapper\",\"variable\":\"foo\",\"fields\":{\"sub\":{\"type\":\"keyword\"}}}"; + TestMapper mapper = fromMapping(mapping); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + String addSubField = "{\"type\":\"test_mapper\",\"variable\":\"foo\",\"fields\":{\"sub2\":{\"type\":\"keyword\"}}}"; + TestMapper toMerge = fromMapping(addSubField); + TestMapper merged = (TestMapper) mapper.merge(toMerge); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"variable\":\"foo\"," + + "\"fields\":{\"sub\":{\"type\":\"keyword\"},\"sub2\":{\"type\":\"keyword\"}}}}", Strings.toString(merged)); + + String badSubField = "{\"type\":\"test_mapper\",\"variable\":\"foo\",\"fields\":{\"sub2\":{\"type\":\"binary\"}}}"; + TestMapper badToMerge = fromMapping(badSubField); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> merged.merge(badToMerge)); + assertEquals("mapper [field.sub2] cannot be changed from type [keyword] to [binary]", e.getMessage()); + } + + // add copy_to, verify + public void testCopyTo() { + String mapping = "{\"type\":\"test_mapper\",\"variable\":\"foo\",\"copy_to\":[\"other\"]}"; + TestMapper mapper = fromMapping(mapping); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + // On update, copy_to is completely replaced + + TestMapper toMerge = fromMapping("{\"type\":\"test_mapper\",\"variable\":\"updated\",\"copy_to\":[\"foo\",\"bar\"]}"); + TestMapper merged = (TestMapper) mapper.merge(toMerge); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"variable\":\"updated\",\"copy_to\":[\"foo\",\"bar\"]}}", + Strings.toString(merged)); + + TestMapper removeCopyTo = fromMapping("{\"type\":\"test_mapper\",\"variable\":\"updated\"}"); + TestMapper noCopyTo = (TestMapper) merged.merge(removeCopyTo); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"variable\":\"updated\"}}", Strings.toString(noCopyTo)); + } + + public void testNullables() { + String mapping = "{\"type\":\"test_mapper\",\"fixed\":null}"; + MapperParsingException e = expectThrows(MapperParsingException.class, () -> fromMapping(mapping)); + assertEquals("[fixed] on mapper [field] of type [test_mapper] must not have a [null] value", e.getMessage()); + + String fine = "{\"type\":\"test_mapper\",\"variable\":null}"; + TestMapper mapper = fromMapping(fine); + assertEquals("{\"field\":" + fine + "}", Strings.toString(mapper)); + } + + public void testObjectSerialization() throws IOException { + + IndexService indexService = createIndex("test"); + + String mapping = "{\"_doc\":{" + + "\"properties\":{" + + "\"actual\":{\"type\":\"double\"}," + + "\"bucket_count\":{\"type\":\"long\"}," + + "\"bucket_influencers\":{\"type\":\"nested\",\"properties\":{" + + "\"anomaly_score\":{\"type\":\"double\"}," + + "\"bucket_span\":{\"type\":\"long\"}," + + "\"is_interim\":{\"type\":\"boolean\"}}}}}}"; + indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + assertEquals(mapping, Strings.toString(indexService.mapperService().documentMapper())); + + indexService.mapperService().merge("_doc", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + assertEquals(mapping, Strings.toString(indexService.mapperService().documentMapper())); + } + + // test custom serializer + public void testCustomSerialization() { + String mapping = "{\"type\":\"test_mapper\",\"wrapper\":\"wrapped value\"}"; + TestMapper mapper = fromMapping(mapping); + assertEquals("wrapped value", mapper.wrapper.name); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + } + + // test validator + public void testParameterValidation() { + String mapping = "{\"type\":\"test_mapper\",\"int_value\":10}"; + TestMapper mapper = fromMapping(mapping); + assertEquals(10, mapper.intValue); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> fromMapping("{\"type\":\"test_mapper\",\"int_value\":60}")); + assertEquals("Value of [n] cannot be greater than 50", e.getMessage()); + + } + + // test deprecations + public void testDeprecatedParameterName() { + String mapping = "{\"type\":\"test_mapper\",\"fixed2_old\":true}"; + TestMapper mapper = fromMapping(mapping); + assertTrue(mapper.fixed2); + assertWarnings("Parameter [fixed2_old] on mapper [field] is deprecated, use [fixed2]"); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"fixed2\":true}}", Strings.toString(mapper)); + } + + public void testAnalyzers() { + String mapping = "{\"type\":\"test_mapper\",\"analyzer\":\"_standard\"}"; + TestMapper mapper = fromMapping(mapping); + assertEquals(mapper.analyzer, Lucene.STANDARD_ANALYZER); + assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); + + String withDef = "{\"type\":\"test_mapper\",\"analyzer\":\"default\"}"; + mapper = fromMapping(withDef); + assertEquals(mapper.analyzer.name(), "default"); + assertThat(mapper.analyzer.analyzer(), instanceOf(StandardAnalyzer.class)); + assertEquals("{\"field\":" + withDef + "}", Strings.toString(mapper)); + + String badAnalyzer = "{\"type\":\"test_mapper\",\"analyzer\":\"wibble\"}"; + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fromMapping(badAnalyzer)); + assertEquals("analyzer [wibble] has not been configured in mappings", e.getMessage()); + } + + public void testDeprecatedParameters() throws IOException { + // 'index' is declared explicitly, 'store' is not, but is one of the previously always-accepted params + String mapping = "{\"type\":\"test_mapper\",\"index\":false,\"store\":true}"; + TestMapper mapper = fromMapping(mapping, Version.V_7_8_0); + assertWarnings("Parameter [store] has no effect on type [test_mapper] and will be removed in future"); + assertFalse(mapper.index); + assertEquals("{\"field\":{\"type\":\"test_mapper\",\"index\":false}}", Strings.toString(mapper)); + + MapperParsingException e = expectThrows(MapperParsingException.class, () -> fromMapping(mapping, Version.V_8_0_0)); + assertEquals("unknown parameter [store] on mapper [field] of type [test_mapper]", e.getMessage()); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 9dcdb4e32fc28..f9feb59700a42 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -48,12 +48,11 @@ import java.net.InetAddress; import java.util.Collections; -import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; -public class RangeFieldTypeTests extends FieldTypeTestCase { +public class RangeFieldTypeTests extends FieldTypeTestCase { RangeType type; protected static String FIELDNAME = "field"; protected static int DISTANCE = 10; @@ -65,17 +64,16 @@ public void setupProperties() { nowInMillis = randomNonNegativeLong(); } - @Override - protected RangeFieldType createDefaultFieldType(String name, Map meta) { + protected RangeFieldType createDefaultFieldType(String name) { if (type == RangeType.DATE) { - return new RangeFieldType(name, true, true, RangeFieldMapper.Defaults.DATE_FORMATTER, meta); + return new RangeFieldType(name, true, true, RangeFieldMapper.Defaults.DATE_FORMATTER, Collections.emptyMap()); } - return new RangeFieldType(name, type, true, true, meta); + return new RangeFieldType(name, type, true, true, Collections.emptyMap()); } public void testRangeQuery() throws Exception { QueryShardContext context = createContext(); - RangeFieldType ft = createDefaultFieldType(FIELDNAME, Collections.emptyMap()); + RangeFieldType ft = createDefaultFieldType(FIELDNAME); ShapeRelation relation = randomFrom(ShapeRelation.values()); boolean includeLower = randomBoolean(); @@ -97,7 +95,7 @@ public void testRangeQuery() throws Exception { public void testRangeQueryIntersectsAdjacentValues() throws Exception { QueryShardContext context = createContext(); ShapeRelation relation = randomFrom(ShapeRelation.values()); - RangeFieldType ft = createDefaultFieldType(FIELDNAME, Collections.emptyMap()); + RangeFieldType ft = createDefaultFieldType(FIELDNAME); Object from = null; Object to = null; @@ -154,7 +152,7 @@ public void testRangeQueryIntersectsAdjacentValues() throws Exception { */ public void testFromLargerToErrors() throws Exception { QueryShardContext context = createContext(); - RangeFieldType ft = createDefaultFieldType(FIELDNAME, Collections.emptyMap()); + RangeFieldType ft = createDefaultFieldType(FIELDNAME); final Object from; final Object to; @@ -474,7 +472,7 @@ public void testParseIp() { public void testTermQuery() throws Exception { // See https://github.com/elastic/elasticsearch/issues/25950 QueryShardContext context = createContext(); - RangeFieldType ft = createDefaultFieldType(FIELDNAME, Collections.emptyMap()); + RangeFieldType ft = createDefaultFieldType(FIELDNAME); Object value = nextFrom(); ShapeRelation relation = ShapeRelation.INTERSECTS; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index f5d7459f517d7..c714615fe29eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -432,8 +432,8 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { mapping.endObject(); MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE)); - assertThat(e.getRootCause(), instanceOf(IllegalArgumentException.class)); - assertThat(e.getRootCause().getMessage(), equalTo("Unused mapping attributes [{foo=bar}]")); + assertThat(e.getRootCause(), instanceOf(MapperParsingException.class)); + assertThat(e.getRootCause().getMessage(), equalTo("unknown parameter [foo] on mapper [__dummy__] of type [null]")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java index 8576c57a8bc01..49855b4096f81 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldTypeTests.java @@ -25,22 +25,8 @@ import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.test.EqualsHashCodeTestUtils; -import java.util.Map; - -public class RoutingFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return RoutingFieldMapper.RoutingFieldType.INSTANCE; - } - - @Override - public void testEquals() { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(RoutingFieldMapper.RoutingFieldType.INSTANCE, - RoutingFieldMapper.RoutingFieldType::new); - } +public class RoutingFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 4f505aaba23f2..ad97b8aaf98be 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -29,16 +29,14 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import java.io.IOException; import java.util.Collection; import java.util.Map; -import static org.hamcrest.Matchers.containsString; +import static org.elasticsearch.index.MapperTestUtils.assertConflicts; import static org.hamcrest.Matchers.equalTo; public class SourceFieldMapperTests extends ESSingleNodeTestCase { @@ -121,19 +119,6 @@ public void testExcludes() throws Exception { assertThat(sourceAsMap.containsKey("path2"), equalTo(true)); } - static void assertConflicts(String mapping1, String mapping2, DocumentMapperParser parser, String... conflicts) throws IOException { - DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping1)); - if (conflicts.length == 0) { - docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE); - } else { - Exception e = expectThrows(IllegalArgumentException.class, - () -> docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE)); - for (String conflict : conflicts) { - assertThat(e.getMessage(), containsString(conflict)); - } - } - } - public void testEnabledNotUpdateable() throws Exception { DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); // using default of true diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 49223cd99d17b..43a1ff09314f1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -38,47 +38,16 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; -import org.junit.Before; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; import static org.hamcrest.Matchers.equalTo; -public class TextFieldTypeTests extends FieldTypeTestCase { - - @Before - public void addModifiers() { - addModifier(t -> { - TextFieldType copy = t.clone(); - copy.setFielddata(t.fielddata() == false); - return copy; - }); - addModifier(t -> { - TextFieldType copy = t.clone(); - copy.setFielddataMaxFrequency(t.fielddataMaxFrequency() + 1); - return copy; - }); - addModifier(t -> { - TextFieldType copy = t.clone(); - copy.setFielddataMinFrequency(t.fielddataMinFrequency() + 1); - return copy; - }); - addModifier(t -> { - TextFieldType copy = t.clone(); - copy.setFielddataMinSegmentSize(t.fielddataMinSegmentSize() + 1); - return copy; - }); - } - - @Override - protected TextFieldType createDefaultFieldType(String name, Map meta) { - return new TextFieldType(name, true, meta); - } +public class TextFieldTypeTests extends FieldTypeTestCase { public void testTermQuery() { MappedFieldType ft = new TextFieldType("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java deleted file mode 100644 index ced04b0d07a09..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimestampFieldMapperTests.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESSingleNodeTestCase; - -import java.io.IOException; - -import static org.elasticsearch.index.mapper.SourceFieldMapperTests.assertConflicts; -import static org.hamcrest.Matchers.equalTo; - -public class TimestampFieldMapperTests extends ESSingleNodeTestCase { - - public void testPostParse() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", - randomBoolean() ? "date" : "date_nanos").endObject().endObject() - .endObject().endObject()); - DocumentMapper docMapper = createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - - ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", "2020-12-12") - .endObject()), - XContentType.JSON)); - assertThat(doc.rootDoc().getFields("@timestamp").length, equalTo(2)); - - Exception e = expectThrows(MapperException.class, () -> docMapper.parse(new SourceToParse("test", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp1", "2020-12-12") - .endObject()), - XContentType.JSON))); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); - - e = expectThrows(MapperException.class, () -> docMapper.parse(new SourceToParse("test", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .array("@timestamp", "2020-12-12", "2020-12-13") - .endObject()), - XContentType.JSON))); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); - } - - public void testValidateNonExistingField() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "non-existing-field").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date").endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), equalTo("the configured timestamp field [non-existing-field] does not exist")); - } - - public void testValidateInvalidFieldType() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "keyword").endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), - equalTo("the configured timestamp field [@timestamp] is of type [keyword], but [date,date_nanos] is expected")); - } - - public void testValidateNotIndexed() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date").field("index", "false").endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] is not indexed")); - } - - public void testValidateNotDocValues() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date").field("doc_values", "false").endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] doesn't have doc values")); - } - - public void testValidateNullValue() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date") - .field("null_value", "2020-12-12").endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), - equalTo("the configured timestamp field [@timestamp] has disallowed [null_value] attribute specified")); - } - - public void testValidateIgnoreMalformed() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date").field("ignore_malformed", "true") - .endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), - equalTo("the configured timestamp field [@timestamp] has disallowed [ignore_malformed] attribute specified")); - } - - public void testValidateNotDisallowedAttribute() throws IOException { - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("path", "@timestamp").endObject() - .startObject("properties").startObject("@timestamp").field("type", "date").field("store", "true") - .endObject().endObject() - .endObject().endObject()); - - Exception e = expectThrows(IllegalArgumentException.class, () -> createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE)); - assertThat(e.getMessage(), - equalTo("the configured timestamp field [@timestamp] has disallowed attributes: [store]")); - } - - public void testCannotUpdateTimestampField() throws IOException { - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - String mapping1 = "{\"type\":{\"_timestamp\":{\"path\":\"@timestamp\"}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; - String mapping2 = "{\"type\":{\"_timestamp\":{\"path\":\"@timestamp2\"}, \"properties\": {\"@timestamp2\": {\"type\": \"date\"}," + - "\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, parser, "cannot update path setting for [_timestamp]"); - - mapping1 = "{\"type\":{\"properties\":{\"@timestamp\": {\"type\": \"date\"}}}}}"; - mapping2 = "{\"type\":{\"_timestamp\":{\"path\":\"@timestamp2\"}, \"properties\": {\"@timestamp2\": {\"type\": \"date\"}," + - "\"@timestamp\": {\"type\": \"date\"}}}})"; - assertConflicts(mapping1, mapping2, parser, "cannot update path setting for [_timestamp]"); - } - - public void testDifferentTSField() throws IOException { - String mapping = "{\n" + - " \"_timestamp\": {\n" + - " \"path\": \"event.my_timestamp\"\n" + - " },\n" + - " \"properties\": {\n" + - " \"event\": {\n" + - " \"properties\": {\n" + - " \"my_timestamp\": {\n" + - " \"type\": \"date\"" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - " }"; - DocumentMapper docMapper = createIndex("test").mapperService() - .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - - ParsedDocument doc = docMapper.parse(new SourceToParse("test", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("event.my_timestamp", "2020-12-12") - .endObject()), - XContentType.JSON)); - assertThat(doc.rootDoc().getFields("event.my_timestamp").length, equalTo(2)); - - Exception e = expectThrows(MapperException.class, () -> docMapper.parse(new SourceToParse("test", "1", BytesReference - .bytes(XContentFactory.jsonBuilder() - .startObject() - .field("event.timestamp", "2020-12-12") - .endObject()), - XContentType.JSON))); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [event.my_timestamp] is missing")); - } - -} diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java index 7cfaf06f3e581..a1f08407c1ffe 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.seqno; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; @@ -106,7 +106,7 @@ public void testRetentionLeaseSyncActionOnPrimary() { threadPool, shardStateAction, new ActionFilters(Collections.emptySet()), - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); final RetentionLeases retentionLeases = mock(RetentionLeases.class); final RetentionLeaseSyncAction.Request request = new RetentionLeaseSyncAction.Request(indexShard.shardId(), retentionLeases); action.dispatchedShardOperationOnPrimary(request, indexShard, @@ -143,7 +143,7 @@ public void testRetentionLeaseSyncActionOnReplica() throws WriteStateException { threadPool, shardStateAction, new ActionFilters(Collections.emptySet()), - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); final RetentionLeases retentionLeases = mock(RetentionLeases.class); final RetentionLeaseSyncAction.Request request = new RetentionLeaseSyncAction.Request(indexShard.shardId(), retentionLeases); @@ -182,7 +182,7 @@ public void testBlocks() { threadPool, shardStateAction, new ActionFilters(Collections.emptySet()), - new WriteMemoryLimits()); + new IndexingPressure(Settings.EMPTY)); assertNull(action.indexBlockLevel()); } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 0d3d24b80643a..311b5fb6c2f32 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.indices; import org.elasticsearch.Version; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; @@ -88,8 +87,7 @@ public Map getMetadataMappers() { private static final String[] EXPECTED_METADATA_FIELDS = new String[]{ IgnoredFieldMapper.NAME, IdFieldMapper.NAME, RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, TypeFieldMapper.NAME, - NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, TimestampFieldMapper.NAME, - FieldNamesFieldMapper.NAME }; + NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, FieldNamesFieldMapper.NAME }; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 8a73b7d7aeaab..4d65b0529c67a 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -27,22 +27,38 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.LongSupplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; public class HierarchyCircuitBreakerServiceTests extends ESTestCase { @@ -270,6 +286,337 @@ long currentMemoryUsage() { assertEquals(0, requestBreaker.getTrippedCount()); } + /** + * "Integration test" checking that we ask the G1 over limit check before parent breaking. + * Given that it depends on GC, the main assertion that we do not get a circuit breaking exception in the threads towards + * the end of the test is not enabled. The following tests checks this in more unit test style. + */ + public void testParentTriggersG1GCBeforeBreaking() throws InterruptedException, TimeoutException, BrokenBarrierException { + assumeTrue("Only G1GC can utilize the over limit check", JvmInfo.jvmInfo().useG1GC().equals("true")); + long g1RegionSize = JvmInfo.jvmInfo().getG1RegionSize(); + assumeTrue("Must have region size", g1RegionSize > 0); + + Settings clusterSettings = Settings.builder() + .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.TRUE) + .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "50%") + .build(); + + AtomicInteger leaderTriggerCount = new AtomicInteger(); + AtomicReference> onOverLimit = new AtomicReference<>(leader -> {}); + AtomicLong time = new AtomicLong(randomLongBetween(Long.MIN_VALUE/2, Long.MAX_VALUE/2)); + long interval = randomLongBetween(1, 1000); + final HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, + Collections.emptyList(), + new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + trackRealMemoryUsage -> new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), + HierarchyCircuitBreakerService::realMemoryUsage, + HierarchyCircuitBreakerService.createYoungGcCountSupplier(), time::get, interval, TimeValue.timeValueSeconds(30)) { + + @Override + void overLimitTriggered(boolean leader) { + if (leader) { + leaderTriggerCount.incrementAndGet(); + } + onOverLimit.get().accept(leader); + } + }); + + long maxHeap = JvmInfo.jvmInfo().getConfiguredMaxHeapSize(); + int regionCount = Math.toIntExact((maxHeap / 2 + g1RegionSize - 1) / g1RegionSize); + + // First setup a host of large byte[]'s, must be Humongous objects since those are cleaned during a young phase (no concurrent cycle + // necessary, which is hard to control in the test). + List data = new ArrayList<>(); + for (int i = 0; i < regionCount; ++i) { + data.add(new byte[(int) (JvmInfo.jvmInfo().getG1RegionSize() / 2)]); + } + try { + service.checkParentLimit(0, "test"); + fail("must exceed memory limit"); + } catch (CircuitBreakingException e) { + // OK + } + + time.addAndGet(randomLongBetween(interval, interval + 10)); + onOverLimit.set(leader -> { + if (leader) { + data.clear(); + } + }); + + logger.trace("black hole [{}]", data.hashCode()); + + int threadCount = randomIntBetween(1, 10); + CyclicBarrier barrier = new CyclicBarrier(threadCount + 1); + List threads = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; ++i) { + threads.add( + new Thread(() -> { + try { + barrier.await(10, TimeUnit.SECONDS); + service.checkParentLimit(0, "test-thread"); + } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { + throw new AssertionError(e); + } catch (CircuitBreakingException e) { + // very rare + logger.info("Thread got semi-unexpected circuit breaking exception", e); + } + })); + } + + threads.forEach(Thread::start); + barrier.await(20, TimeUnit.SECONDS); + + for (Thread thread : threads) { + thread.join(10000); + } + threads.forEach(thread -> assertFalse(thread.isAlive())); + + assertThat(leaderTriggerCount.get(), equalTo(2)); + } + + public void testParentDoesOverLimitCheck() { + long g1RegionSize = JvmInfo.jvmInfo().getG1RegionSize(); + + Settings clusterSettings = Settings.builder() + .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.TRUE) + .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "50%") + .build(); + boolean saveTheDay = randomBoolean(); + AtomicBoolean overLimitTriggered = new AtomicBoolean(); + final HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService(clusterSettings, + Collections.emptyList(), + new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + trackRealMemoryUsage -> + memoryUsed -> { + assertTrue(overLimitTriggered.compareAndSet(false, true)); + if (saveTheDay) { + return new HierarchyCircuitBreakerService.MemoryUsage(memoryUsed.baseUsage / 2, + memoryUsed.totalUsage - (memoryUsed.baseUsage / 2), memoryUsed.transientChildUsage, + memoryUsed.permanentChildUsage); + } else { + return memoryUsed; + } + } + ); + + int allocationSize = g1RegionSize > 0 ? (int) (g1RegionSize / 2) : 1024 * 1024; + int allocationCount = (int) (JvmInfo.jvmInfo().getConfiguredMaxHeapSize() / allocationSize) + 1; + List data = new ArrayList<>(); + try { + for (int i = 0 ; i < allocationCount && overLimitTriggered.get() == false; ++i) { + data.add(new byte[allocationSize]); + service.checkParentLimit(0, "test"); + } + assertTrue(saveTheDay); + } catch (CircuitBreakingException e) { + assertFalse(saveTheDay); + } + + logger.trace("black hole [{}]", data.hashCode()); + } + + public void testFallbackG1RegionSize() { + assumeTrue("Only G1GC can utilize the over limit check", JvmInfo.jvmInfo().useG1GC().equals("true")); + assumeTrue("Must have region size", JvmInfo.jvmInfo().getG1RegionSize() > 0); + + assertThat(HierarchyCircuitBreakerService.G1OverLimitStrategy.fallbackRegionSize(JvmInfo.jvmInfo()), + equalTo(JvmInfo.jvmInfo().getG1RegionSize())); + } + + public void testG1OverLimitStrategyBreakOnMemory() { + AtomicLong time = new AtomicLong(randomLongBetween(Long.MIN_VALUE/2, Long.MAX_VALUE/2)); + AtomicInteger leaderTriggerCount = new AtomicInteger(); + AtomicInteger nonLeaderTriggerCount = new AtomicInteger(); + long interval = randomLongBetween(1, 1000); + AtomicLong memoryUsage = new AtomicLong(); + + HierarchyCircuitBreakerService.G1OverLimitStrategy strategy = + new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), memoryUsage::get, () -> 0, time::get, interval, + TimeValue.timeValueSeconds(30)) { + @Override + void overLimitTriggered(boolean leader) { + if (leader) { + leaderTriggerCount.incrementAndGet(); + } else { + nonLeaderTriggerCount.incrementAndGet(); + } + } + }; + memoryUsage.set(randomLongBetween(100, 110)); + HierarchyCircuitBreakerService.MemoryUsage input = new HierarchyCircuitBreakerService.MemoryUsage(100, randomLongBetween(100, 110), + randomLongBetween(0, 50), + randomLongBetween(0, 50)); + + assertThat(strategy.overLimit(input), sameInstance(input)); + assertThat(leaderTriggerCount.get(), equalTo(1)); + + memoryUsage.set(99); + HierarchyCircuitBreakerService.MemoryUsage output = strategy.overLimit(input); + assertThat(output, not(sameInstance(input))); + assertThat(output.baseUsage, equalTo(memoryUsage.get())); + assertThat(output.totalUsage, equalTo(99 + input.totalUsage - 100)); + assertThat(output.transientChildUsage, equalTo(input.transientChildUsage)); + assertThat(output.permanentChildUsage, equalTo(input.permanentChildUsage)); + assertThat(nonLeaderTriggerCount.get(), equalTo(1)); + + time.addAndGet(randomLongBetween(interval, interval * 2)); + output = strategy.overLimit(input); + assertThat(output, not(sameInstance(input))); + assertThat(output.baseUsage, equalTo(memoryUsage.get())); + assertThat(output.totalUsage, equalTo(99 + input.totalUsage - 100)); + assertThat(output.transientChildUsage, equalTo(input.transientChildUsage)); + assertThat(output.permanentChildUsage, equalTo(input.permanentChildUsage)); + assertThat(leaderTriggerCount.get(), equalTo(2)); + } + + public void testG1OverLimitStrategyBreakOnGcCount() { + AtomicLong time = new AtomicLong(randomLongBetween(Long.MIN_VALUE/2, Long.MAX_VALUE/2)); + AtomicInteger leaderTriggerCount = new AtomicInteger(); + AtomicInteger nonLeaderTriggerCount = new AtomicInteger(); + long interval = randomLongBetween(1, 1000); + AtomicLong memoryUsageCounter = new AtomicLong(); + AtomicLong gcCounter = new AtomicLong(); + LongSupplier memoryUsageSupplier = () -> { + memoryUsageCounter.incrementAndGet(); + return randomLongBetween(100, 110); + }; + HierarchyCircuitBreakerService.G1OverLimitStrategy strategy = + new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), + memoryUsageSupplier, + gcCounter::incrementAndGet, + time::get, interval, TimeValue.timeValueSeconds(30)) { + + @Override + void overLimitTriggered(boolean leader) { + if (leader) { + leaderTriggerCount.incrementAndGet(); + } else { + nonLeaderTriggerCount.incrementAndGet(); + } + } + }; + HierarchyCircuitBreakerService.MemoryUsage input = new HierarchyCircuitBreakerService.MemoryUsage(100, randomLongBetween(100, 110), + randomLongBetween(0, 50), + randomLongBetween(0, 50)); + + assertThat(strategy.overLimit(input), sameInstance(input)); + assertThat(leaderTriggerCount.get(), equalTo(1)); + assertThat(gcCounter.get(), equalTo(2L)); + assertThat(memoryUsageCounter.get(), equalTo(2L)); // 1 before gc count break and 1 to get resulting memory usage. + } + + public void testG1OverLimitStrategyThrottling() throws InterruptedException, BrokenBarrierException, TimeoutException { + AtomicLong time = new AtomicLong(randomLongBetween(Long.MIN_VALUE/2, Long.MAX_VALUE/2)); + AtomicInteger leaderTriggerCount = new AtomicInteger(); + long interval = randomLongBetween(1, 1000); + AtomicLong memoryUsage = new AtomicLong(); + HierarchyCircuitBreakerService.G1OverLimitStrategy strategy = + new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), memoryUsage::get, () -> 0, + time::get, interval, TimeValue.timeValueSeconds(30)) { + + @Override + void overLimitTriggered(boolean leader) { + if (leader) { + leaderTriggerCount.incrementAndGet(); + } + } + }; + + int threadCount = randomIntBetween(1, 10); + CyclicBarrier barrier = new CyclicBarrier(threadCount + 1); + AtomicReference countDown = new AtomicReference<>(new CountDownLatch(randomIntBetween(1, 20))); + List threads = IntStream.range(0, threadCount) + .mapToObj(i -> new Thread(() -> { + try { + barrier.await(10, TimeUnit.SECONDS); + } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { + throw new AssertionError(e); + } + do { + HierarchyCircuitBreakerService.MemoryUsage input = + new HierarchyCircuitBreakerService.MemoryUsage(randomLongBetween(0, 100), randomLongBetween(0, 100), + randomLongBetween(0, 100), randomLongBetween(0, 100)); + HierarchyCircuitBreakerService.MemoryUsage output = strategy.overLimit(input); + assertThat(output.totalUsage, equalTo(output.baseUsage + input.totalUsage - input.baseUsage)); + assertThat(output.transientChildUsage, equalTo(input.transientChildUsage)); + assertThat(output.permanentChildUsage, equalTo(input.permanentChildUsage)); + countDown.get().countDown(); + } while (!Thread.interrupted()); + })).collect(Collectors.toList()); + + threads.forEach(Thread::start); + barrier.await(20, TimeUnit.SECONDS); + + int iterationCount = randomIntBetween(1, 5); + for (int i = 0; i < iterationCount; ++i) { + memoryUsage.set(randomLongBetween(0, 100)); + assertTrue(countDown.get().await(20, TimeUnit.SECONDS)); + assertThat(leaderTriggerCount.get(), lessThanOrEqualTo(i + 1)); + assertThat(leaderTriggerCount.get(), greaterThanOrEqualTo(i / 2 + 1)); + time.addAndGet(randomLongBetween(interval, interval * 2)); + countDown.set(new CountDownLatch(randomIntBetween(1, 20))); + } + + threads.forEach(Thread::interrupt); + for (Thread thread : threads) { + thread.join(10000); + } + threads.forEach(thread -> assertFalse(thread.isAlive())); + } + + public void testCreateOverLimitStrategy() { + assertThat(HierarchyCircuitBreakerService.createOverLimitStrategy(false), + not(instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class))); + HierarchyCircuitBreakerService.OverLimitStrategy overLimitStrategy = HierarchyCircuitBreakerService.createOverLimitStrategy(true); + if (JvmInfo.jvmInfo().useG1GC().equals("true")) { + assertThat(overLimitStrategy, instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)); + assertThat(((HierarchyCircuitBreakerService.G1OverLimitStrategy) overLimitStrategy).getLockTimeout(), + equalTo(TimeValue.timeValueMillis(500))); + } else { + assertThat(overLimitStrategy, not(instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class))); + } + } + + public void testG1LockTimeout() throws Exception { + CountDownLatch startedBlocking = new CountDownLatch(1); + CountDownLatch blockingUntil = new CountDownLatch(1); + AtomicLong gcCounter = new AtomicLong(); + HierarchyCircuitBreakerService.G1OverLimitStrategy strategy = + new HierarchyCircuitBreakerService.G1OverLimitStrategy(JvmInfo.jvmInfo(), () -> 100, gcCounter::incrementAndGet, + () -> 0, 1, TimeValue.timeValueMillis(randomFrom(0, 5, 10))) { + + @Override + void overLimitTriggered(boolean leader) { + if (leader) { + startedBlocking.countDown(); + try { + // this is the central assertion - the overLimit call below should complete in a timely manner. + assertThat(blockingUntil.await(10, TimeUnit.SECONDS), is(true)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + } + } + }; + + HierarchyCircuitBreakerService.MemoryUsage input = new HierarchyCircuitBreakerService.MemoryUsage(100, 100, 0, 0); + Thread blocker = new Thread(() -> { + strategy.overLimit(input); + }); + blocker.start(); + try { + assertThat(startedBlocking.await(10, TimeUnit.SECONDS), is(true)); + + // this should complete in a timely manner, verified by the assertion in the thread. + assertThat(strategy.overLimit(input), sameInstance(input)); + } finally { + blockingUntil.countDown(); + blocker.join(10000); + assertThat(blocker.isAlive(), is(false)); + } + } + public void testTrippedCircuitBreakerDurability() { Settings clusterSettings = Settings.builder() .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), Boolean.FALSE) diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index c13717bbcb02e..20eab5a66107f 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -102,6 +102,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -110,6 +111,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.IntSupplier; +import java.util.stream.Collectors; import java.util.zip.CRC32; import static java.util.Collections.emptyMap; @@ -237,7 +239,7 @@ public void testSendSnapshotSendsOps() throws IOException { final long startingSeqNo = randomIntBetween(0, numberOfDocsWithValidSequenceNumbers - 1); final long endingSeqNo = randomLongBetween(startingSeqNo, numberOfDocsWithValidSequenceNumbers - 1); - final List shippedOps = new ArrayList<>(); + final Queue shippedOps = ConcurrentCollections.newQueue(); final AtomicLong checkpointOnTarget = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() { @Override @@ -256,10 +258,12 @@ public void indexTranslogOperations(List operations, int tot final int expectedOps = (int) (endingSeqNo - startingSeqNo + 1); RecoverySourceHandler.SendSnapshotResult result = future.actionGet(); assertThat(result.sentOperations, equalTo(expectedOps)); - shippedOps.sort(Comparator.comparing(Translog.Operation::seqNo)); + List sortedShippedOps = shippedOps.stream() + .sorted(Comparator.comparing(Translog.Operation::seqNo)) + .collect(Collectors.toList()); assertThat(shippedOps.size(), equalTo(expectedOps)); for (int i = 0; i < shippedOps.size(); i++) { - assertThat(shippedOps.get(i), equalTo(operations.get(i + (int) startingSeqNo + initialNumberOfDocs))); + assertThat(sortedShippedOps.get(i), equalTo(operations.get(i + (int) startingSeqNo + initialNumberOfDocs))); } assertThat(result.targetLocalCheckpoint, equalTo(checkpointOnTarget.get())); } diff --git a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java index 75c33c9cfb71a..04efdfdab808e 100644 --- a/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/fs/FsHealthServiceTests.java @@ -17,10 +17,8 @@ * under the License. */ - package org.elasticsearch.monitor.fs; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -57,7 +55,6 @@ import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.hamcrest.Matchers.is; - public class FsHealthServiceTests extends ESTestCase { private DeterministicTaskQueue deterministicTaskQueue; @@ -113,6 +110,7 @@ public void testFailsHealthOnIOException() throws IOException { assertEquals("health check passed", fsHealthService.getHealth().getInfo()); //disrupt file system + disruptFileSystemProvider.restrictPathPrefix(""); // disrupt all paths disruptFileSystemProvider.injectIOException.set(true); fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); fsHealthService.new FsHealthMonitor().run(); @@ -218,9 +216,9 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { assertEquals("health check passed", fsHealthService.getHealth().getInfo()); //disrupt file system writes on single path - disruptWritesFileSystemProvider.injectIOException.set(true); String disruptedPath = randomFrom(paths).toString(); disruptWritesFileSystemProvider.restrictPathPrefix(disruptedPath); + disruptWritesFileSystemProvider.injectIOException.set(true); fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); @@ -238,7 +236,7 @@ private static class FileSystemIOExceptionProvider extends FilterFileSystemProvi AtomicBoolean injectIOException = new AtomicBoolean(); AtomicInteger injectedPaths = new AtomicInteger(); - private String pathPrefix = "/"; + private String pathPrefix; FileSystemIOExceptionProvider(FileSystem inner) { super("disrupt_fs_health://", inner); @@ -255,6 +253,7 @@ public int getInjectedPathCount(){ @Override public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException { if (injectIOException.get()){ + assert pathPrefix != null : "must set pathPrefix before starting disruptions"; if (path.toString().startsWith(pathPrefix) && path.toString().endsWith(".es_temp_file")) { injectedPaths.incrementAndGet(); throw new IOException("fake IOException"); @@ -269,7 +268,7 @@ private static class FileSystemFsyncIOExceptionProvider extends FilterFileSystem AtomicBoolean injectIOException = new AtomicBoolean(); AtomicInteger injectedPaths = new AtomicInteger(); - private String pathPrefix = "/"; + private String pathPrefix = null; FileSystemFsyncIOExceptionProvider(FileSystem inner) { super("disrupt_fs_health://", inner); @@ -289,6 +288,7 @@ public FileChannel newFileChannel(Path path, Set options, @Override public void force(boolean metaData) throws IOException { if (injectIOException.get()) { + assert pathPrefix != null : "must set pathPrefix before starting disruptions"; if (path.toString().startsWith(pathPrefix) && path.toString().endsWith(".es_temp_file")) { injectedPaths.incrementAndGet(); throw new IOException("fake IOException"); @@ -305,8 +305,7 @@ private static class FileSystemFsyncHungProvider extends FilterFileSystemProvide AtomicBoolean injectIOException = new AtomicBoolean(); AtomicInteger injectedPaths = new AtomicInteger(); - private String pathPrefix = "/"; - private long delay; + private final long delay; private final ThreadPool threadPool; FileSystemFsyncHungProvider(FileSystem inner, long delay, ThreadPool threadPool) { @@ -325,7 +324,7 @@ public FileChannel newFileChannel(Path path, Set options, @Override public void force(boolean metaData) throws IOException { if (injectIOException.get()) { - if (path.toString().startsWith(pathPrefix) && path.toString().endsWith(".es_temp_file")) { + if (path.getFileName().toString().equals(FsHealthService.FsHealthMonitor.TEMP_FILE_NAME)) { injectedPaths.incrementAndGet(); final long startTimeMillis = threadPool.relativeTimeInMillis(); do { diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index 0ee651f219a35..6a7738c1c32e4 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -166,7 +166,7 @@ public void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryS @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { listener.onResponse(null); } diff --git a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java index 6c53624ab0cd3..17dfa08b93fd2 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptCacheTests.java @@ -19,35 +19,47 @@ package org.elasticsearch.script; import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; +import java.util.stream.Collectors; + public class ScriptCacheTests extends ESTestCase { // even though circuit breaking is allowed to be configured per minute, we actually weigh this over five minutes // simply by multiplying by five, so even setting it to one, requires five compilations to break public void testCompilationCircuitBreaking() throws Exception { - final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); - final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); - Tuple rate = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(Settings.EMPTY); - String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); - ScriptCache cache = new ScriptCache(size, expire, Tuple.tuple(1, TimeValue.timeValueMinutes(1)), settingName); + String context = randomFrom( + ScriptModule.CORE_CONTEXTS.values().stream().filter( + c -> c.maxCompilationRateDefault.equals(ScriptCache.UNLIMITED_COMPILATION_RATE) == false + ).collect(Collectors.toList()) + ).name; + final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); + final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); + Setting rateSetting = + ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context); + ScriptCache.CompilationRate rate = + ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); + String rateSettingName = rateSetting.getKey(); + ScriptCache cache = new ScriptCache(size, expire, + new ScriptCache.CompilationRate(1, TimeValue.timeValueMinutes(1)), rateSettingName); cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, (Tuple.tuple(2, TimeValue.timeValueMinutes(1))), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(2, TimeValue.timeValueMinutes(1)), rateSettingName); cache.checkCompilationLimit(); // should pass cache.checkCompilationLimit(); // should pass expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); int count = randomIntBetween(5, 50); - cache = new ScriptCache(size, expire, (Tuple.tuple(count, TimeValue.timeValueMinutes(1))), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(count, TimeValue.timeValueMinutes(1)), rateSettingName); for (int i = 0; i < count; i++) { cache.checkCompilationLimit(); // should pass } expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, (Tuple.tuple(0, TimeValue.timeValueMinutes(1))), settingName); + cache = new ScriptCache(size, expire, new ScriptCache.CompilationRate(0, TimeValue.timeValueMinutes(1)), rateSettingName); expectThrows(CircuitBreakingException.class, cache::checkCompilationLimit); - cache = new ScriptCache(size, expire, (Tuple.tuple(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1))), settingName); + cache = new ScriptCache(size, expire, + new ScriptCache.CompilationRate(Integer.MAX_VALUE, TimeValue.timeValueMinutes(1)), rateSettingName); int largeLimit = randomIntBetween(1000, 10000); for (int i = 0; i < largeLimit; i++) { cache.checkCompilationLimit(); @@ -55,9 +67,14 @@ public void testCompilationCircuitBreaking() throws Exception { } public void testUnlimitedCompilationRate() { - final Integer size = ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING.get(Settings.EMPTY); - final TimeValue expire = ScriptService.SCRIPT_GENERAL_CACHE_EXPIRE_SETTING.get(Settings.EMPTY); - String settingName = ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(); + String context = randomFrom( + ScriptModule.CORE_CONTEXTS.values().stream().filter( + c -> c.maxCompilationRateDefault.equals(ScriptCache.UNLIMITED_COMPILATION_RATE) == false + ).collect(Collectors.toList()) + ).name; + final Integer size = ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); + final TimeValue expire = ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(context).get(Settings.EMPTY); + String settingName = ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(context).getKey(); ScriptCache cache = new ScriptCache(size, expire, ScriptCache.UNLIMITED_COMPILATION_RATE, settingName); ScriptCache.TokenBucketState initialState = cache.tokenBucketState.get(); for(int i=0; i < 3000; i++) { diff --git a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 651dd57884476..2bc5d7105a01f 100644 --- a/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/server/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -26,9 +26,7 @@ import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; @@ -40,17 +38,15 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; -import static org.elasticsearch.script.ScriptService.CacheHolder; -import static org.elasticsearch.script.ScriptService.MAX_COMPILATION_RATE_FUNCTION; import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_EXPIRE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_CACHE_SIZE_SETTING; -import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_CACHE_SIZE_SETTING; -import static org.elasticsearch.script.ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING; import static org.elasticsearch.script.ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.is; @@ -107,8 +103,8 @@ protected StoredScriptSource getScriptFromClusterState(String id) { } public void testMaxCompilationRateSetting() throws Exception { - assertThat(MAX_COMPILATION_RATE_FUNCTION.apply("10/1m"), is(Tuple.tuple(10, TimeValue.timeValueMinutes(1)))); - assertThat(MAX_COMPILATION_RATE_FUNCTION.apply("10/60s"), is(Tuple.tuple(10, TimeValue.timeValueMinutes(1)))); + assertThat(new ScriptCache.CompilationRate("10/1m"), is(new ScriptCache.CompilationRate(10, TimeValue.timeValueMinutes(1)))); + assertThat(new ScriptCache.CompilationRate("10/60s"), is(new ScriptCache.CompilationRate(10, TimeValue.timeValueMinutes(1)))); assertException("10/m", IllegalArgumentException.class, "failed to parse [m]"); assertException("6/1.6m", IllegalArgumentException.class, "failed to parse [1.6m], fractional time values are not supported"); assertException("foo/bar", IllegalArgumentException.class, "could not parse [foo] as integer in value [foo/bar]"); @@ -128,7 +124,7 @@ public void testMaxCompilationRateSetting() throws Exception { } private void assertException(String rate, Class clazz, String message) { - Exception e = expectThrows(clazz, () -> MAX_COMPILATION_RATE_FUNCTION.apply(rate)); + Exception e = expectThrows(clazz, () -> new ScriptCache.CompilationRate(rate)); assertThat(e.getMessage(), is(message)); } @@ -219,6 +215,7 @@ public void testCompileCountedInCompilationStats() throws IOException { scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values())); assertEquals(1L, scriptService.stats().getCompilations()); } + public void testMultipleCompilationsCountedInCompilationStats() throws IOException { buildScriptService(Settings.EMPTY); int numberOfCompilations = randomIntBetween(1, 20); @@ -230,17 +227,12 @@ public void testMultipleCompilationsCountedInCompilationStats() throws IOExcepti } public void testCompilationStatsOnCacheHit() throws IOException { - Settings.Builder builder = Settings.builder() - .put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1) - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "2/1m"); - buildScriptService(builder.build()); + buildScriptService(Settings.EMPTY); Script script = new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()); ScriptContext context = randomFrom(contexts.values()); scriptService.compile(script, context); scriptService.compile(script, context); assertEquals(1L, scriptService.stats().getCompilations()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_CACHE_SIZE_SETTING, - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); } public void testIndexedScriptCountedInCompilationStats() throws IOException { @@ -248,22 +240,19 @@ public void testIndexedScriptCountedInCompilationStats() throws IOException { ScriptContext ctx = randomFrom(contexts.values()); scriptService.compile(new Script(ScriptType.STORED, null, "script", Collections.emptyMap()), ctx); assertEquals(1L, scriptService.stats().getCompilations()); - assertEquals(1L, scriptService.cacheStats().getContextStats().get(ctx.name).getCompilations()); + assertEquals(1L, getByContext(scriptService.stats(), ctx.name).getCompilations()); } public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException { - Settings.Builder builder = Settings.builder(); - builder.put(SCRIPT_GENERAL_CACHE_SIZE_SETTING.getKey(), 1); - builder.put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m"); - buildScriptService(builder.build()); - scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), randomFrom(contexts.values())); - scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), randomFrom(contexts.values())); + ScriptContext context = randomFrom(contexts.values()); + buildScriptService(Settings.builder() + .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(context.name).getKey(), 1) + .build() + ); + scriptService.compile(new Script(ScriptType.INLINE, "test", "1+1", Collections.emptyMap()), context); + scriptService.compile(new Script(ScriptType.INLINE, "test", "2+2", Collections.emptyMap()), context); assertEquals(2L, scriptService.stats().getCompilations()); - assertEquals(2L, scriptService.cacheStats().getGeneralStats().getCompilations()); assertEquals(1L, scriptService.stats().getCacheEvictions()); - assertEquals(1L, scriptService.cacheStats().getGeneralStats().getCacheEvictions()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_CACHE_SIZE_SETTING, - SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); } public void testContextCacheStats() throws IOException { @@ -304,15 +293,14 @@ public void testContextCacheStats() throws IOException { assertEquals(CircuitBreakingException.class, gse.getRootCause().getClass()); // Context specific - ScriptCacheStats stats = scriptService.cacheStats(); - assertEquals(2L, stats.getContextStats().get(contextA.name).getCompilations()); - assertEquals(1L, stats.getContextStats().get(contextA.name).getCacheEvictions()); - assertEquals(1L, stats.getContextStats().get(contextA.name).getCompilationLimitTriggered()); + ScriptStats stats = scriptService.stats(); + assertEquals(2L, getByContext(stats, contextA.name).getCompilations()); + assertEquals(1L, getByContext(stats, contextA.name).getCacheEvictions()); + assertEquals(1L, getByContext(stats, contextA.name).getCompilationLimitTriggered()); - assertEquals(3L, stats.getContextStats().get(contextB.name).getCompilations()); - assertEquals(1L, stats.getContextStats().get(contextB.name).getCacheEvictions()); - assertEquals(2L, stats.getContextStats().get(contextB.name).getCompilationLimitTriggered()); - assertNull(scriptService.cacheStats().getGeneralStats()); + assertEquals(3L, getByContext(stats, contextB.name).getCompilations()); + assertEquals(1L, getByContext(stats, contextB.name).getCacheEvictions()); + assertEquals(2L, getByContext(stats, contextB.name).getCompilationLimitTriggered()); // Summed up assertEquals(5L, scriptService.stats().getCompilations()); @@ -320,6 +308,13 @@ public void testContextCacheStats() throws IOException { assertEquals(3L, scriptService.stats().getCompilationLimitTriggered()); } + private ScriptContextStats getByContext(ScriptStats stats, String context) { + List maybeContextStats = stats.getContextStats().stream().filter(c -> c.getContext().equals(context)) + .collect(Collectors.toList()); + assertEquals(1, maybeContextStats.size()); + return maybeContextStats.get(0); + } + public void testStoreScript() throws Exception { BytesReference script = BytesReference.bytes(XContentFactory.jsonBuilder() .startObject() @@ -382,84 +377,9 @@ public void testMaxSizeLimit() throws Exception { iae.getMessage()); } - public void testConflictContextSettings() throws IOException { - IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> { - buildScriptService(Settings.builder() - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m") - .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123).build()); - }); - assertEquals("Context cache settings [script.context.field.cache_max_size] requires " + - "[script.max_compilations_rate] to be [use-context]", - illegal.getMessage() - ); - - illegal = expectThrows(IllegalArgumentException.class, () -> { - buildScriptService(Settings.builder() - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m") - .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m").build()); - }); - - assertEquals("Context cache settings [script.context.ingest.cache_expire] requires " + - "[script.max_compilations_rate] to be [use-context]", - illegal.getMessage() - ); - - illegal = expectThrows(IllegalArgumentException.class, () -> { - buildScriptService(Settings.builder() - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "10/1m") - .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m").build()); - }); - - assertEquals("Context cache settings [script.context.score.max_compilations_rate] requires " + - "[script.max_compilations_rate] to be [use-context]", - illegal.getMessage() - ); - - buildScriptService( - Settings.builder() - .put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), "5m") - .put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace("field").getKey(), 123) - .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("score").getKey(), "50/5m") - .build()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); - } - - public void testUseContextSettingValue() { - Settings s = Settings.builder() - .put(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY) - .put(ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("foo").getKey(), - ScriptService.USE_CONTEXT_RATE_KEY) - .build(); - - assertEquals(ScriptService.SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.get(s), ScriptService.USE_CONTEXT_RATE_VALUE); - - IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> { - ScriptService.SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getAsMap(s); - }); - - assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [use-context]", illegal.getMessage()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); - } - - public void testCacheHolderGeneralConstructor() throws IOException { - String compilationRate = "77/5m"; - Tuple rate = ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate); - buildScriptService( - Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate).build() - ); - - CacheHolder holder = scriptService.cacheHolder.get(); - - assertNotNull(holder.general); - assertNull(holder.contextCache); - assertEquals(holder.general.rate, rate); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); - } - public void testCacheHolderContextConstructor() throws IOException { String a = randomFrom(contexts.keySet()); String b = randomValueOtherThan(a, () -> randomFrom(contexts.keySet())); - String c = randomValueOtherThanMany(Set.of(a, b)::contains, () -> randomFrom(contexts.keySet())); String aCompilationRate = "77/5m"; String bCompilationRate = "78/6m"; @@ -468,35 +388,15 @@ public void testCacheHolderContextConstructor() throws IOException { .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), bCompilationRate) .build()); - assertNull(scriptService.cacheHolder.get().general); assertNotNull(scriptService.cacheHolder.get().contextCache); assertEquals(contexts.keySet(), scriptService.cacheHolder.get().contextCache.keySet()); - assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(aCompilationRate), + assertEquals(new ScriptCache.CompilationRate(aCompilationRate), scriptService.cacheHolder.get().contextCache.get(a).get().rate); - assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(bCompilationRate), + assertEquals(new ScriptCache.CompilationRate(bCompilationRate), scriptService.cacheHolder.get().contextCache.get(b).get().rate); } - public void testCompilationRateUnlimitedContextOnly() throws IOException { - IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> { - buildScriptService(Settings.builder() - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.UNLIMITED_COMPILATION_RATE_KEY) - .build()); - }); - assertEquals("parameter must contain a positive integer and a timevalue, i.e. 10/1m, but was [unlimited]", illegal.getMessage()); - - // Should not throw. - buildScriptService(Settings.builder() - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("ingest").getKey(), - ScriptService.UNLIMITED_COMPILATION_RATE_KEY) - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace("field").getKey(), - ScriptService.UNLIMITED_COMPILATION_RATE_KEY) - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), ScriptService.USE_CONTEXT_RATE_KEY) - .build()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); - } - public void testDisableCompilationRateSetting() throws IOException { IllegalArgumentException illegal = expectThrows(IllegalArgumentException.class, () -> { buildScriptService(Settings.builder() @@ -510,21 +410,9 @@ public void testDisableCompilationRateSetting() throws IOException { "[script.disable_max_compilations_rate]", illegal.getMessage()); - illegal = expectThrows(IllegalArgumentException.class, () -> { - buildScriptService(Settings.builder() - .put("script.disable_max_compilations_rate", true) - .put("script.max_compilations_rate", "76/10m") - .build()); - }); - assertEquals("Cannot set custom general compilation rates [script.max_compilations_rate] " + - "to [Tuple [v1=76, v2=10m]] if compile " + - "rates disabled via [script.disable_max_compilations_rate]", - illegal.getMessage()); - buildScriptService(Settings.builder() .put("script.disable_max_compilations_rate", true) .build()); - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); } public void testCacheHolderChangeSettings() throws IOException { @@ -534,41 +422,29 @@ public void testCacheHolderChangeSettings() throws IOException { String b = randomValueOtherThan(a, () -> randomFrom(contextNames)); String bRate = "78/6m"; String c = randomValueOtherThanMany(s -> a.equals(s) || b.equals(s), () -> randomFrom(contextNames)); - String compilationRate = "77/5m"; - Tuple generalRate = MAX_COMPILATION_RATE_FUNCTION.apply(compilationRate); - - Settings s = Settings.builder() - .put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), compilationRate) - .build(); - buildScriptService(s); - - assertNotNull(scriptService.cacheHolder.get().general); - // Set should not throw when using general cache - scriptService.cacheHolder.get().set(c, scriptService.contextCache(s, contexts.get(c))); - assertNull(scriptService.cacheHolder.get().contextCache); - assertEquals(generalRate, scriptService.cacheHolder.get().general.rate); + buildScriptService(Settings.EMPTY); - scriptService.setCacheHolder(Settings.builder() - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(a).getKey(), aRate) - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), bRate) - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(c).getKey(), - ScriptService.UNLIMITED_COMPILATION_RATE_KEY) - .build() - ); + Settings settings = Settings.builder() + .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(a).getKey(), aRate) + .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), bRate) + .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(c).getKey(), + ScriptService.UNLIMITED_COMPILATION_RATE_KEY) + .build(); - assertNull(scriptService.cacheHolder.get().general); assertNotNull(scriptService.cacheHolder.get().contextCache); + scriptService.cacheHolder.get().set(a, scriptService.contextCache(settings, contexts.get(a))); + scriptService.cacheHolder.get().set(b, scriptService.contextCache(settings, contexts.get(b))); + scriptService.cacheHolder.get().set(c, scriptService.contextCache(settings, contexts.get(c))); // get of missing context should be null assertNull(scriptService.cacheHolder.get().get( randomValueOtherThanMany(contexts.keySet()::contains, () -> randomAlphaOfLength(8))) ); assertEquals(contexts.keySet(), scriptService.cacheHolder.get().contextCache.keySet()); - String d = randomValueOtherThanMany(Set.of(a, b, c)::contains, () -> randomFrom(contextNames)); - assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(aRate), + assertEquals(new ScriptCache.CompilationRate(aRate), scriptService.cacheHolder.get().contextCache.get(a).get().rate); - assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(bRate), + assertEquals(new ScriptCache.CompilationRate(bRate), scriptService.cacheHolder.get().contextCache.get(b).get().rate); assertEquals(ScriptCache.UNLIMITED_COMPILATION_RATE, scriptService.cacheHolder.get().contextCache.get(c).get().rate); @@ -576,50 +452,30 @@ public void testCacheHolderChangeSettings() throws IOException { scriptService.cacheHolder.get().set(b, scriptService.contextCache(Settings.builder() .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(b).getKey(), aRate).build(), contexts.get(b))); - assertEquals(ScriptService.MAX_COMPILATION_RATE_FUNCTION.apply(aRate), + assertEquals(new ScriptCache.CompilationRate(aRate), scriptService.cacheHolder.get().contextCache.get(b).get().rate); - - scriptService.setCacheHolder(s); - assertNotNull(scriptService.cacheHolder.get().general); - assertNull(scriptService.cacheHolder.get().contextCache); - assertEquals(generalRate, scriptService.cacheHolder.get().general.rate); - - scriptService.setCacheHolder( - Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), bRate).build() - ); - - assertNotNull(scriptService.cacheHolder.get().general); - assertNull(scriptService.cacheHolder.get().contextCache); - assertEquals(MAX_COMPILATION_RATE_FUNCTION.apply(bRate), scriptService.cacheHolder.get().general.rate); - - CacheHolder holder = scriptService.cacheHolder.get(); - scriptService.setCacheHolder( - Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), bRate).build() - ); - assertEquals(holder, scriptService.cacheHolder.get()); - - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); } public void testFallbackToContextDefaults() throws IOException { String contextRateStr = randomIntBetween(10, 1024) + "/" + randomIntBetween(10, 200) + "m"; - Tuple contextRate = MAX_COMPILATION_RATE_FUNCTION.apply(contextRateStr); + ScriptCache.CompilationRate contextRate = new ScriptCache.CompilationRate(contextRateStr); int contextCacheSize = randomIntBetween(1, 1024); TimeValue contextExpire = TimeValue.timeValueMinutes(randomIntBetween(10, 200)); - buildScriptService( - Settings.builder().put(SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING.getKey(), "75/5m").build() - ); + buildScriptService(Settings.EMPTY); String name = "ingest"; // Use context specific - scriptService.setCacheHolder(Settings.builder() - .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextCacheSize) - .put(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextExpire) - .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextRateStr) - .build() - ); + scriptService.cacheHolder.get().set( + name, + scriptService.contextCache(Settings.builder() + .put(SCRIPT_CACHE_SIZE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextCacheSize) + .put(SCRIPT_CACHE_EXPIRE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextExpire) + .put(SCRIPT_MAX_COMPILATIONS_RATE_SETTING.getConcreteSettingForNamespace(name).getKey(), contextRateStr) + .build(), + contexts.get(name) + )); ScriptService.CacheHolder holder = scriptService.cacheHolder.get(); assertNotNull(holder.contextCache); @@ -639,11 +495,9 @@ public void testFallbackToContextDefaults() throws IOException { assertNotNull(holder.contextCache.get(name)); assertNotNull(holder.contextCache.get(name).get()); - assertEquals(ingest.maxCompilationRateDefault, holder.contextCache.get(name).get().rate); + assertEquals(ingest.maxCompilationRateDefault, holder.contextCache.get(name).get().rate.asTuple()); assertEquals(ingest.cacheSizeDefault, holder.contextCache.get(name).get().cacheSize); assertEquals(ingest.cacheExpireDefault, holder.contextCache.get(name).get().cacheExpire); - - assertSettingDeprecationsAndWarnings(new Setting[]{SCRIPT_GENERAL_MAX_COMPILATIONS_RATE_SETTING}); } private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) { diff --git a/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java new file mode 100644 index 0000000000000..9ce212d92c56b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/script/ScriptStatsTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class ScriptStatsTests extends ESTestCase { + public void testXContent() throws IOException { + List contextStats = List.of( + new ScriptContextStats("contextB", 100, 201, 302), + new ScriptContextStats("contextA", 1000, 2010, 3020) + ); + ScriptStats stats = new ScriptStats(contextStats); + final XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + builder.startObject(); + stats.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + String expected = "{\n" + + " \"script\" : {\n" + + " \"compilations\" : 1100,\n" + + " \"cache_evictions\" : 2211,\n" + + " \"compilation_limit_triggered\" : 3322,\n" + + " \"contexts\" : [\n" + + " {\n" + + " \"context\" : \"contextA\",\n" + + " \"compilations\" : 1000,\n" + + " \"cache_evictions\" : 2010,\n" + + " \"compilation_limit_triggered\" : 3020\n" + + " },\n" + + " {\n" + + " \"context\" : \"contextB\",\n" + + " \"compilations\" : 100,\n" + + " \"cache_evictions\" : 201,\n" + + " \"compilation_limit_triggered\" : 302\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + assertThat(Strings.toString(builder), equalTo(expected)); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java new file mode 100644 index 0000000000000..4673d46fad192 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorBaseTests.java @@ -0,0 +1,192 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AggregatorBaseTests extends ESSingleNodeTestCase { + + class BogusAggregator extends AggregatorBase { + BogusAggregator(SearchContext searchContext, Aggregator parent) throws IOException { + super("bogus", AggregatorFactories.EMPTY, searchContext, parent, CardinalityUpperBound.NONE, Map.of()); + } + + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + throw new UnsupportedOperationException(); + } + } + + private SearchContext mockSearchContext(Query query) { + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.query()).thenReturn(query); + BigArrays mockBigArrays = mock(BigArrays.class); + CircuitBreakerService mockCBS = mock(CircuitBreakerService.class); + when(mockCBS.getBreaker(CircuitBreaker.REQUEST)).thenReturn(mock(CircuitBreaker.class)); + when(mockBigArrays.breakerService()).thenReturn(mockCBS); + when(searchContext.bigArrays()).thenReturn(mockBigArrays); + return searchContext; + } + + private Function pointReaderShim(SearchContext context, Aggregator parent, ValuesSourceConfig config) + throws IOException { + BogusAggregator aggregator = new BogusAggregator(context, parent); + return aggregator.pointReaderIfAvailable(config); + } + + private Aggregator mockAggregator() { + return mock(Aggregator.class); + } + + private ValuesSourceConfig getVSConfig( + String fieldName, + NumberFieldMapper.NumberType numType, + boolean indexed, + QueryShardContext context + ) { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(fieldName, numType, indexed, true, Collections.emptyMap()); + return ValuesSourceConfig.resolveFieldOnly(ft, context); + } + + private ValuesSourceConfig getVSConfig( + String fieldName, + DateFieldMapper.Resolution resolution, + boolean indexed, + QueryShardContext context + ) { + Mapper.BuilderContext builderContext = new Mapper.BuilderContext( + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(), + new ContentPath() + ); + MappedFieldType ft = new DateFieldMapper.Builder(fieldName).index(indexed) + .withResolution(resolution) + .build(builderContext) + .fieldType(); + return ValuesSourceConfig.resolveFieldOnly(ft, context); + } + + public void testShortcutIsApplicable() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", "bytes", "type=keyword"); + client().prepareIndex("index").setId("1").setSource("bytes", "abc").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) { + assertNotNull( + pointReaderShim(mockSearchContext(new MatchAllDocsQuery()), null, getVSConfig("number", type, true, context)) + ); + assertNotNull(pointReaderShim(mockSearchContext(null), null, getVSConfig("number", type, true, context))); + assertNull(pointReaderShim(mockSearchContext(null), mockAggregator(), getVSConfig("number", type, true, context))); + assertNull( + pointReaderShim( + mockSearchContext(new TermQuery(new Term("foo", "bar"))), + null, + getVSConfig("number", type, true, context) + ) + ); + assertNull(pointReaderShim(mockSearchContext(null), mockAggregator(), getVSConfig("number", type, true, context))); + assertNull(pointReaderShim(mockSearchContext(null), null, getVSConfig("number", type, false, context))); + } + for (DateFieldMapper.Resolution resolution : DateFieldMapper.Resolution.values()) { + assertNull( + pointReaderShim( + mockSearchContext(new MatchAllDocsQuery()), + mockAggregator(), + getVSConfig("number", resolution, true, context) + ) + ); + assertNull( + pointReaderShim( + mockSearchContext(new TermQuery(new Term("foo", "bar"))), + null, + getVSConfig("number", resolution, true, context) + ) + ); + assertNull(pointReaderShim(mockSearchContext(null), mockAggregator(), getVSConfig("number", resolution, true, context))); + assertNull(pointReaderShim(mockSearchContext(null), null, getVSConfig("number", resolution, false, context))); + } + // Check that we decode a dates "just like" the doc values instance. + Instant expected = Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2020-01-01T00:00:00Z")); + byte[] scratch = new byte[8]; + LongPoint.encodeDimension(DateFieldMapper.Resolution.MILLISECONDS.convert(expected), scratch, 0); + assertThat( + pointReaderShim( + mockSearchContext(new MatchAllDocsQuery()), + null, + getVSConfig("number", DateFieldMapper.Resolution.MILLISECONDS, true, context) + ).apply(scratch), + equalTo(expected.toEpochMilli()) + ); + LongPoint.encodeDimension(DateFieldMapper.Resolution.NANOSECONDS.convert(expected), scratch, 0); + assertThat( + pointReaderShim( + mockSearchContext(new MatchAllDocsQuery()), + null, + getVSConfig("number", DateFieldMapper.Resolution.NANOSECONDS, true, context) + ).apply(scratch), + equalTo(expected.toEpochMilli()) + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index da7d898b3b9ba..4d84a854fc57f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -1132,6 +1132,29 @@ public void testLegacyThenNew() throws IOException { assertWarnings("[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future."); } + + public void testOverlappingBounds() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchAllDocsQuery(), + Arrays.asList( + "2017-02-01", + "2017-02-02", + "2017-02-02", + "2017-02-03", + "2017-02-03", + "2017-02-03", + "2017-02-05" + ), + aggregation -> aggregation .calendarInterval(DateHistogramInterval.DAY) + .hardBounds(new LongBounds("2010-01-01", "2020-01-01")) + .extendedBounds(new LongBounds("2009-01-01", "2021-01-01")) + .field(AGGREGABLE_DATE), + histogram -> {}, false + )); + + assertThat(ex.getMessage(), equalTo("Extended bounds have to be inside hard bounds, " + + "hard bounds: [2010-01-01--2020-01-01], extended bounds: [2009-01-01--2021-01-01]")); + } + public void testIllegalInterval() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testSearchCase(new MatchAllDocsQuery(), Collections.emptyList(), diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java index f3c48f36fab3d..faefcd3e3a68b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramTests.java @@ -59,7 +59,7 @@ protected DateHistogramAggregationBuilder createTestAggregatorBuilder() { } } if (randomBoolean()) { - factory.extendedBounds(ExtendedBoundsTests.randomExtendedBounds()); + factory.extendedBounds(LongBoundsTests.randomExtendedBounds()); } if (randomBoolean()) { factory.format("###.##"); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java index 23f3baf284a2c..9d4920b64a443 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateRangeHistogramAggregatorTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -46,6 +47,7 @@ import java.util.function.Consumer; import static java.util.Collections.singleton; +import static org.hamcrest.Matchers.equalTo; public class DateRangeHistogramAggregatorTests extends AggregatorTestCase { @@ -658,6 +660,150 @@ public void testWithinQuery() throws Exception { ); } + public void testHardBounds() throws Exception { + RangeFieldMapper.Range range1 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T02:15:00"), + asLong("2019-08-02T05:45:00"), true, true); + RangeFieldMapper.Range range2 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T05:15:00"), + asLong("2019-08-02T17:45:00"), true, true); + + testCase( + Queries.newMatchAllQuery(), + builder -> builder.calendarInterval(DateHistogramInterval.HOUR).hardBounds( + new LongBounds("2019-08-02T03:00:00", "2019-08-02T10:00:00")), + writer -> { + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range1))))); + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range2))))); + }, + histo -> { + assertEquals(8, histo.getBuckets().size()); + + assertEquals(asZDT("2019-08-02T03:00:00"), histo.getBuckets().get(0).getKey()); + assertEquals(1, histo.getBuckets().get(0).getDocCount()); + + assertEquals(asZDT("2019-08-02T05:00:00"), histo.getBuckets().get(2).getKey()); + assertEquals(2, histo.getBuckets().get(2).getDocCount()); + + assertEquals(asZDT("2019-08-02T10:00:00"), histo.getBuckets().get(7).getKey()); + assertEquals(1, histo.getBuckets().get(7).getDocCount()); + + assertTrue(AggregationInspectionHelper.hasValue(histo)); + } + ); + } + public void testHardBoundsWithOpenRanges() throws Exception { + RangeFieldMapper.Range range1 = new RangeFieldMapper.Range(RangeType.DATE, Long.MIN_VALUE, + asLong("2019-08-02T05:45:00"), true, true); + RangeFieldMapper.Range range2 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T05:15:00"), + Long.MAX_VALUE, true, true); + + testCase( + Queries.newMatchAllQuery(), + builder -> builder.calendarInterval(DateHistogramInterval.HOUR).hardBounds( + new LongBounds("2019-08-02T03:00:00", "2019-08-02T10:00:00")), + writer -> { + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range1))))); + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range2))))); + }, + histo -> { + assertEquals(8, histo.getBuckets().size()); + + assertEquals(asZDT("2019-08-02T03:00:00"), histo.getBuckets().get(0).getKey()); + assertEquals(1, histo.getBuckets().get(0).getDocCount()); + + assertEquals(asZDT("2019-08-02T05:00:00"), histo.getBuckets().get(2).getKey()); + assertEquals(2, histo.getBuckets().get(2).getDocCount()); + + assertEquals(asZDT("2019-08-02T10:00:00"), histo.getBuckets().get(7).getKey()); + assertEquals(1, histo.getBuckets().get(7).getDocCount()); + + assertTrue(AggregationInspectionHelper.hasValue(histo)); + } + ); + } + + public void testBothBounds() throws Exception { + RangeFieldMapper.Range range1 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T02:15:00"), + asLong("2019-08-02T05:45:00"), true, true); + RangeFieldMapper.Range range2 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T05:15:00"), + asLong("2019-08-02T17:45:00"), true, true); + + testCase( + Queries.newMatchAllQuery(), + builder -> builder.calendarInterval(DateHistogramInterval.HOUR) + .hardBounds(new LongBounds("2019-08-02T00:00:00", "2019-08-02T10:00:00")) + .extendedBounds(new LongBounds("2019-08-02T01:00:00", "2019-08-02T08:00:00")), + writer -> { + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range1))))); + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range2))))); + }, + histo -> { + assertEquals(10, histo.getBuckets().size()); + + assertEquals(asZDT("2019-08-02T01:00:00"), histo.getBuckets().get(0).getKey()); + assertEquals(0, histo.getBuckets().get(0).getDocCount()); + + assertEquals(asZDT("2019-08-02T02:00:00"), histo.getBuckets().get(1).getKey()); + assertEquals(1, histo.getBuckets().get(1).getDocCount()); + + assertEquals(asZDT("2019-08-02T10:00:00"), histo.getBuckets().get(9).getKey()); + assertEquals(1, histo.getBuckets().get(9).getDocCount()); + + assertTrue(AggregationInspectionHelper.hasValue(histo)); + } + ); + } + + public void testOverlappingBounds() throws Exception { + + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> testCase( + Queries.newMatchAllQuery(), + builder -> builder.calendarInterval(DateHistogramInterval.HOUR) + .hardBounds(new LongBounds("2019-08-02T01:00:00", "2019-08-02T08:00:00")) + .extendedBounds(new LongBounds("2019-08-02T00:00:00", "2019-08-02T10:00:00")), + writer -> { + + }, + histo -> { + fail("Shouldn't be here"); + } + )); + + assertThat(ex.getMessage(), equalTo("Extended bounds have to be inside hard bounds, " + + "hard bounds: [2019-08-02T01:00:00--2019-08-02T08:00:00], extended bounds: [2019-08-02T00:00:00--2019-08-02T10:00:00]")); + } + + public void testEqualBounds() throws Exception { + RangeFieldMapper.Range range1 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T02:15:00"), + asLong("2019-08-02T05:45:00"), true, true); + RangeFieldMapper.Range range2 = new RangeFieldMapper.Range(RangeType.DATE, asLong("2019-08-02T05:15:00"), + asLong("2019-08-02T17:45:00"), true, true); + + testCase( + Queries.newMatchAllQuery(), + builder -> builder.calendarInterval(DateHistogramInterval.HOUR) + .hardBounds(new LongBounds("2019-08-02T00:00:00", "2019-08-02T10:00:00")) + .extendedBounds(new LongBounds("2019-08-02T00:00:00", "2019-08-02T10:00:00")), + writer -> { + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range1))))); + writer.addDocument(singleton(new BinaryDocValuesField(FIELD_NAME, RangeType.DATE.encodeRanges(singleton(range2))))); + }, + histo -> { + assertEquals(11, histo.getBuckets().size()); + + assertEquals(asZDT("2019-08-02T00:00:00"), histo.getBuckets().get(0).getKey()); + assertEquals(0, histo.getBuckets().get(0).getDocCount()); + + assertEquals(asZDT("2019-08-02T02:00:00"), histo.getBuckets().get(2).getKey()); + assertEquals(1, histo.getBuckets().get(2).getDocCount()); + + assertEquals(asZDT("2019-08-02T10:00:00"), histo.getBuckets().get(10).getKey()); + assertEquals(1, histo.getBuckets().get(10).getDocCount()); + + assertTrue(AggregationInspectionHelper.hasValue(histo)); + } + ); + } + private void testCase(Query query, Consumer configure, CheckedConsumer buildIndex, @@ -673,19 +819,18 @@ private void testCase(Query query, private void testCase(DateHistogramAggregationBuilder aggregationBuilder, Query query, CheckedConsumer buildIndex, Consumer verify, MappedFieldType fieldType) throws IOException { - Directory directory = newDirectory(); - RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); - buildIndex.accept(indexWriter); - indexWriter.close(); - - IndexReader indexReader = DirectoryReader.open(directory); - IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + try(Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + buildIndex.accept(indexWriter); + indexWriter.close(); - InternalDateHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); - verify.accept(histogram); + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = newSearcher(indexReader, true, true); - indexReader.close(); - directory.close(); + InternalDateHistogram histogram = searchAndReduce(indexSearcher, query, aggregationBuilder, fieldType); + verify.accept(histogram); + } + } } private static long asLong(String dateTime) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java new file mode 100644 index 0000000000000..bf4b9d6494aa9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DoubleBoundsTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.histogram; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleBoundsTests extends ESTestCase { + /** + * Construct a random {@link DoubleBounds} + */ + public static DoubleBounds randomBounds() { + Double min = randomDouble(); + Double max = randomValueOtherThan(min, ESTestCase::randomDouble); + if (min > max) { + double temp = min; + min = max; + max = temp; + } + if (randomBoolean()) { + // Construct with one missing bound + if (randomBoolean()) { + return new DoubleBounds(null, max); + } + return new DoubleBounds(min, null); + } + return new DoubleBounds(min, max); + } + + public void testTransportRoundTrip() throws IOException { + DoubleBounds orig = randomBounds(); + + BytesReference origBytes; + try (BytesStreamOutput out = new BytesStreamOutput()) { + orig.writeTo(out); + origBytes = out.bytes(); + } + + DoubleBounds read; + try (StreamInput in = origBytes.streamInput()) { + read = new DoubleBounds(in); + assertEquals("read fully", 0, in.available()); + } + assertEquals(orig, read); + + BytesReference readBytes; + try (BytesStreamOutput out = new BytesStreamOutput()) { + read.writeTo(out); + readBytes = out.bytes(); + } + + assertEquals(origBytes, readBytes); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/59475") + public void testXContentRoundTrip() throws Exception { + DoubleBounds orig = randomBounds(); + + try (XContentBuilder out = JsonXContent.contentBuilder()) { + out.startObject(); + orig.toXContent(out, ToXContent.EMPTY_PARAMS); + out.endObject(); + + try (XContentParser in = createParser(JsonXContent.jsonXContent, BytesReference.bytes(out))) { + XContentParser.Token token = in.currentToken(); + assertNull(token); + + token = in.nextToken(); + assertThat(token, equalTo(XContentParser.Token.START_OBJECT)); + + DoubleBounds read = DoubleBounds.PARSER.apply(in, null); + assertEquals(orig, read); + } catch (Exception e) { + throw new Exception("Error parsing [" + BytesReference.bytes(out).utf8ToString() + "]", e); + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java index d1d402e9d5e98..492bc5adfde44 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogramTests.java @@ -65,13 +65,13 @@ public void setUp() throws Exception { emptyBucketInfo = null; } else { minDocCount = 0; - ExtendedBounds extendedBounds = null; + LongBounds extendedBounds = null; if (randomBoolean()) { //it's ok if min and max are outside the range of the generated buckets, that will just mean that //empty buckets won't be added before the first bucket and/or after the last one long min = baseMillis - intervalMillis * randomNumberOfBuckets(); long max = baseMillis + randomNumberOfBuckets() * intervalMillis; - extendedBounds = new ExtendedBounds(min, max); + extendedBounds = new LongBounds(min, max); } emptyBucketInfo = new InternalDateHistogram.EmptyBucketInfo(rounding, InternalAggregations.EMPTY, extendedBounds); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBoundsTests.java similarity index 78% rename from server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java rename to server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBoundsTests.java index be9c9947b6b37..69fbcdde27e80 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/ExtendedBoundsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/LongBoundsTests.java @@ -44,12 +44,12 @@ import static java.lang.Math.min; import static org.hamcrest.Matchers.equalTo; -public class ExtendedBoundsTests extends ESTestCase { +public class LongBoundsTests extends ESTestCase { /** - * Construct a random {@link ExtendedBounds}. + * Construct a random {@link LongBounds}. */ - public static ExtendedBounds randomExtendedBounds() { - ExtendedBounds bounds = randomParsedExtendedBounds(); + public static LongBounds randomExtendedBounds() { + LongBounds bounds = randomParsedExtendedBounds(); if (randomBoolean()) { bounds = unparsed(bounds); } @@ -57,17 +57,17 @@ public static ExtendedBounds randomExtendedBounds() { } /** - * Construct a random {@link ExtendedBounds} in pre-parsed form. + * Construct a random {@link LongBounds} in pre-parsed form. */ - public static ExtendedBounds randomParsedExtendedBounds() { + public static LongBounds randomParsedExtendedBounds() { long maxDateValue = 253402300799999L; // end of year 9999 long minDateValue = -377705116800000L; // beginning of year -9999 if (randomBoolean()) { // Construct with one missing bound if (randomBoolean()) { - return new ExtendedBounds(null, maxDateValue); + return new LongBounds(null, maxDateValue); } - return new ExtendedBounds(minDateValue, null); + return new LongBounds(minDateValue, null); } long a = randomLongBetween(minDateValue, maxDateValue); long b; @@ -76,18 +76,18 @@ public static ExtendedBounds randomParsedExtendedBounds() { } while (a == b); long min = min(a, b); long max = max(a, b); - return new ExtendedBounds(min, max); + return new LongBounds(min, max); } /** * Convert an extended bounds in parsed for into one in unparsed form. */ - public static ExtendedBounds unparsed(ExtendedBounds template) { + public static LongBounds unparsed(LongBounds template) { // It'd probably be better to randomize the formatter DateFormatter formatter = DateFormatter.forPattern("strict_date_time").withZone(ZoneOffset.UTC); String minAsStr = template.getMin() == null ? null : formatter.formatMillis(template.getMin()); String maxAsStr = template.getMax() == null ? null : formatter.formatMillis(template.getMax()); - return new ExtendedBounds(minAsStr, maxAsStr); + return new LongBounds(minAsStr, maxAsStr); } public void testParseAndValidate() { @@ -101,33 +101,33 @@ BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry DateFormatter formatter = DateFormatter.forPattern("dateOptionalTime"); DocValueFormat format = new DocValueFormat.DateTime(formatter, ZoneOffset.UTC, DateFieldMapper.Resolution.MILLISECONDS); - ExtendedBounds expected = randomParsedExtendedBounds(); - ExtendedBounds parsed = unparsed(expected).parseAndValidate("test", qsc, format); + LongBounds expected = randomParsedExtendedBounds(); + LongBounds parsed = unparsed(expected).parseAndValidate("test", "extended_bounds", qsc, format); // parsed won't *equal* expected because equal includes the String parts assertEquals(expected.getMin(), parsed.getMin()); assertEquals(expected.getMax(), parsed.getMax()); - parsed = new ExtendedBounds("now", null).parseAndValidate("test", qsc, format); + parsed = new LongBounds("now", null).parseAndValidate("test", "extended_bounds", qsc, format); assertEquals(now, (long) parsed.getMin()); assertNull(parsed.getMax()); - parsed = new ExtendedBounds(null, "now").parseAndValidate("test", qsc, format); + parsed = new LongBounds(null, "now").parseAndValidate("test", "extended_bounds", qsc, format); assertNull(parsed.getMin()); assertEquals(now, (long) parsed.getMax()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new ExtendedBounds(100L, 90L).parseAndValidate("test", qsc, format)); + () -> new LongBounds(100L, 90L).parseAndValidate("test", "extended_bounds", qsc, format)); assertEquals("[extended_bounds.min][100] cannot be greater than [extended_bounds.max][90] for histogram aggregation [test]", e.getMessage()); e = expectThrows(IllegalArgumentException.class, - () -> unparsed(new ExtendedBounds(100L, 90L)).parseAndValidate("test", qsc, format)); + () -> unparsed(new LongBounds(100L, 90L)).parseAndValidate("test", "extended_bounds", qsc, format)); assertEquals("[extended_bounds.min][100] cannot be greater than [extended_bounds.max][90] for histogram aggregation [test]", e.getMessage()); } public void testTransportRoundTrip() throws IOException { - ExtendedBounds orig = randomExtendedBounds(); + LongBounds orig = randomExtendedBounds(); BytesReference origBytes; try (BytesStreamOutput out = new BytesStreamOutput()) { @@ -135,9 +135,9 @@ public void testTransportRoundTrip() throws IOException { origBytes = out.bytes(); } - ExtendedBounds read; + LongBounds read; try (StreamInput in = origBytes.streamInput()) { - read = new ExtendedBounds(in); + read = new LongBounds(in); assertEquals("read fully", 0, in.available()); } assertEquals(orig, read); @@ -152,7 +152,7 @@ public void testTransportRoundTrip() throws IOException { } public void testXContentRoundTrip() throws Exception { - ExtendedBounds orig = randomExtendedBounds(); + LongBounds orig = randomExtendedBounds(); try (XContentBuilder out = JsonXContent.contentBuilder()) { out.startObject(); @@ -166,11 +166,7 @@ public void testXContentRoundTrip() throws Exception { token = in.nextToken(); assertThat(token, equalTo(XContentParser.Token.START_OBJECT)); - token = in.nextToken(); - assertThat(token, equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(in.currentName(), equalTo(ExtendedBounds.EXTENDED_BOUNDS_FIELD.getPreferredName())); - - ExtendedBounds read = ExtendedBounds.PARSER.apply(in, null); + LongBounds read = LongBounds.PARSER.apply(in, null); assertEquals(orig, read); } catch (Exception e) { throw new Exception("Error parsing [" + BytesReference.bytes(out).utf8ToString() + "]", e); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java index e3ac46de76e62..10d11af74648b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/VariableWidthHistogramAggregatorTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalStats; @@ -218,25 +217,25 @@ public void testDoubles() throws Exception { // Once the cache limit is reached, cached documents are collected into (3/4 * shard_size) buckets // A new bucket should be added when there is a document that is distant from all existing buckets public void testNewBucketCreation() throws Exception { - final List dataset = Arrays.asList(-1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 40, 30, 25, 32, 38, 80, 50, 75); + final List dataset = Arrays.asList(-1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 40, 30, 25, 32, 36, 80, 50, 75, 60); double doubleError = 1d / 10000d; // Search (no reduce) // Expected clusters: [ (-1), (1), (3), (5), (7), (9), (11), (13), (15), (17), - // (19), (25, 30, 32), (38, 40), (50), (75, 80) ] - // Corresponding keys (centroids): [ -1, 1, 3, ..., 17, 19, 29, 39, 50, 77.5] + // (19), (25, 30, 32), (36, 40, 50), (60), (75, 80) ] + // Corresponding keys (centroids): [ -1, 1, 3, ..., 17, 19, 29, 42, 77.5] // Note: New buckets are created for 30, 50, and 80 because they are distant from the other buckets - final List keys = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 29d, 39d, 50d, 77.5d); - final List mins = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 25d, 38d, 50d, 75d); - final List maxes = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 32d, 40d, 50d, 80d); - final List docCounts = Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 2, 1, 2); + final List keys = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 29d, 42d, 60d, 77.5d); + final List mins = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 25d, 36d, 60d, 75d); + final List maxes = Arrays.asList(-1d, 1d, 3d, 5d, 7d, 9d, 11d, 13d, 15d, 17d, 19d, 32d, 50d, 60d, 80d); + final List docCounts = Arrays.asList(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 2); assert keys.size() == docCounts.size() && keys.size() == keys.size(); final Map expectedDocCountOnlySearch = new HashMap<>(); final Map expectedMinsOnlySearch = new HashMap<>(); final Map expectedMaxesOnlySearch = new HashMap<>(); - for(int i=0; i expectedDocCountBigKeys = new HashMap<>(); + final Map expectedMinsBigKeys = new HashMap<>(); + final Map expectedMaxesBigKeys = new HashMap<>(); + for(int i=0; i< keys.size(); i++){ + expectedDocCountBigKeys.put(Long.MAX_VALUE * keys.get(i), docCounts.get(i)); + expectedMinsBigKeys.put(Long.MAX_VALUE * keys.get(i), Long.MAX_VALUE * mins.get(i)); + expectedMaxesBigKeys.put(Long.MAX_VALUE * keys.get(i), Long.MAX_VALUE * maxes.get(i)); + } + + testSearchCase(DEFAULT_QUERY, dataset.stream().map(n -> Double.valueOf(n.doubleValue() * Long.MAX_VALUE)).collect(toList()), false, + aggregation -> aggregation.field(NUMERIC_FIELD).setNumBuckets(2).setShardSize(16).setInitialBuffer(12), + histogram -> { + final List buckets = histogram.getBuckets(); + assertEquals(expectedDocCountOnlySearch.size(), buckets.size()); + buckets.forEach(bucket -> { + long expectedDocCount = expectedDocCountBigKeys.getOrDefault(bucket.getKey(), 0).longValue(); + double expectedCentroid = expectedMinsBigKeys.getOrDefault(bucket.getKey(), 0d).doubleValue(); + double expectedMax = expectedMaxesBigKeys.getOrDefault(bucket.getKey(), 0d).doubleValue(); assertEquals(expectedDocCount, bucket.getDocCount()); assertEquals(expectedCentroid, bucket.min(), doubleError); assertEquals(expectedMax, bucket.max(), doubleError); @@ -308,7 +332,6 @@ public void testSimpleSubAggregations() throws IOException{ .setShardSize(4) .subAggregation(AggregationBuilders.stats("stats").field(NUMERIC_FIELD)), histogram -> { - final List buckets = histogram.getBuckets(); double deltaError = 1d/10000d; // Expected clusters: [ (1, 2), (5), (8,9) ] @@ -343,7 +366,6 @@ public void testComplexSubAggregations() throws IOException{ .setShardSize(4) .subAggregation(new StatsAggregationBuilder("stats").field(NUMERIC_FIELD)), histogram -> { - final List buckets = histogram.getBuckets(); double deltaError = 1d / 10000d; // Expected clusters: [ (0, 1, 2, 3), (4, 5, 6, 7), (8, 9, 10, 11) ] @@ -381,16 +403,15 @@ public void testSubAggregationReduction() throws IOException{ .shardSize(2) .size(1)), histogram -> { - final List buckets = histogram.getBuckets(); double deltaError = 1d / 10000d; // This is a test to make sure that the sub aggregations get reduced // This terms sub aggregation has shardSize (2) != size (1), so we will get 1 bucket only if // InternalVariableWidthHistogram reduces the sub aggregations. - InternalTerms terms = histogram.getBuckets().get(0).getAggregations().get("terms"); + LongTerms terms = histogram.getBuckets().get(0).getAggregations().get("terms"); assertEquals(1L, terms.getBuckets().size(), deltaError); - assertEquals(1L, ((InternalTerms.Bucket) terms.getBuckets().get(0)).getKey()); + assertEquals(1L, terms.getBuckets().get(0).getKey()); }); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java index 8991639f5ac7e..0886ad91015f1 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MinAggregatorTests.java @@ -43,22 +43,16 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; @@ -70,7 +64,6 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.filter.Filter; @@ -84,13 +77,9 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; -import org.elasticsearch.search.aggregations.support.FieldContext; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.LeafDocLookup; import java.io.IOException; -import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -106,8 +95,6 @@ import static java.util.Collections.singleton; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class MinAggregatorTests extends AggregatorTestCase { @@ -690,103 +677,6 @@ public void testScriptCaching() throws IOException { } } - public void testShortcutIsApplicable() { - for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) { - assertNotNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new MatchAllDocsQuery()), - null, - mockNumericValuesSourceConfig("number", type, true) - ) - ); - assertNotNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - null, - mockNumericValuesSourceConfig("number", type, true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - mockAggregator(), - mockNumericValuesSourceConfig("number", type, true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new TermQuery(new Term("foo", "bar"))), - null, - mockNumericValuesSourceConfig("number", type, true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - mockAggregator(), - mockNumericValuesSourceConfig("number", type, true) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - null, - mockNumericValuesSourceConfig("number", type, false) - ) - ); - } - for (DateFieldMapper.Resolution resolution : DateFieldMapper.Resolution.values()) { - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new MatchAllDocsQuery()), - mockAggregator(), - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new TermQuery(new Term("foo", "bar"))), - null, - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - mockAggregator(), - mockDateValuesSourceConfig("number", true, resolution) - ) - ); - assertNull( - MinAggregator.getPointReaderOrNull( - mockSearchContext(null), - null, - mockDateValuesSourceConfig("number", false, resolution) - ) - ); - } - // Check that we decode a dates "just like" the doc values instance. - Instant expected = Instant.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse("2020-01-01T00:00:00Z")); - byte[] scratch = new byte[8]; - LongPoint.encodeDimension(DateFieldMapper.Resolution.MILLISECONDS.convert(expected), scratch, 0); - assertThat( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new MatchAllDocsQuery()), - null, - mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.MILLISECONDS) - ).apply(scratch), equalTo(expected.toEpochMilli()) - ); - LongPoint.encodeDimension(DateFieldMapper.Resolution.NANOSECONDS.convert(expected), scratch, 0); - assertThat( - MinAggregator.getPointReaderOrNull( - mockSearchContext(new MatchAllDocsQuery()), - null, - mockDateValuesSourceConfig("number", true, DateFieldMapper.Resolution.NANOSECONDS) - ).apply(scratch), equalTo(expected.toEpochMilli()) - ); - - } - public void testMinShortcutRandom() throws Exception { testMinShortcutCase( () -> randomLongBetween(Integer.MIN_VALUE, Integer.MAX_VALUE), @@ -862,40 +752,6 @@ private void testMinShortcutCase(Supplier randomNumber, directory.close(); } - private SearchContext mockSearchContext(Query query) { - SearchContext searchContext = mock(SearchContext.class); - when(searchContext.query()).thenReturn(query); - return searchContext; - } - - private Aggregator mockAggregator() { - return mock(Aggregator.class); - } - - private ValuesSourceConfig mockNumericValuesSourceConfig(String fieldName, - NumberFieldMapper.NumberType numType, - boolean indexed) { - ValuesSourceConfig config = mock(ValuesSourceConfig.class); - MappedFieldType ft = new NumberFieldMapper.NumberFieldType(fieldName, numType, indexed, true, Collections.emptyMap()); - when(config.fieldContext()).thenReturn(new FieldContext(fieldName, null, ft)); - return config; - } - - private ValuesSourceConfig mockDateValuesSourceConfig(String fieldName, boolean indexed, - DateFieldMapper.Resolution resolution) { - ValuesSourceConfig config = mock(ValuesSourceConfig.class); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext( - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(), - new ContentPath()); - MappedFieldType ft = new DateFieldMapper.Builder(fieldName) - .index(indexed) - .withResolution(resolution) - .build(builderContext) - .fieldType(); - when(config.fieldContext()).thenReturn(new FieldContext(fieldName, null, ft)); - return config; - } - private void testCase(Query query, CheckedConsumer buildIndex, Consumer verify) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistryTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistryTests.java new file mode 100644 index 0000000000000..b156facf4b4f4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceRegistryTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.support; + +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.test.ESTestCase; +import org.mockito.Mockito; + +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ValuesSourceRegistryTests extends ESTestCase { + + public void testAggregatorNotFoundException() { + final QueryShardContext queryShardContext = mock(QueryShardContext.class); + final AggregationScript.Factory mockAggScriptFactory = mock(AggregationScript.Factory.class); + when(mockAggScriptFactory.newFactory(Mockito.any(), Mockito.any())).thenReturn(mock(AggregationScript.LeafFactory.class)); + when(queryShardContext.compile(Mockito.any(), Mockito.any())).thenReturn(mockAggScriptFactory); + + ValuesSourceConfig fieldOnly = ValuesSourceConfig.resolve( + queryShardContext, + null, + "field", + null, + null, + null, + null, + CoreValuesSourceType.BYTES + ); + + ValuesSourceConfig scriptOnly = ValuesSourceConfig.resolve( + queryShardContext, + null, + null, + mockScript("fakeScript"), + null, + null, + null, + CoreValuesSourceType.BYTES + ); + ValuesSourceRegistry registry = new ValuesSourceRegistry(Map.of("bogus", List.of()), null); + expectThrows(IllegalArgumentException.class, () -> registry.getAggregator(fieldOnly, "bogus")); + expectThrows(IllegalArgumentException.class, () -> registry.getAggregator(scriptOnly, "bogus")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java index b545ab468a541..20ac204ae6599 100644 --- a/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/collapse/CollapseBuilderTests.java @@ -198,11 +198,6 @@ public void testBuildWithExceptions() { { MappedFieldType fieldType = new MappedFieldType("field", true, true, TextSearchInfo.NONE, Collections.emptyMap()) { - @Override - public MappedFieldType clone() { - return null; - } - @Override public String typeName() { return null; diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 95b5baeb99023..18253f5aaee6f 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -120,10 +120,6 @@ private ShardSearchRequest createRequest(int shardId, String[] routings, String private QueryShardContext createShardContext(Version indexVersionCreated, IndexReader reader, String fieldName, DocValuesType dvType, int numShards, int shardId) { MappedFieldType fieldType = new MappedFieldType(fieldName, true, dvType != null, TextSearchInfo.NONE, Collections.emptyMap()) { - @Override - public MappedFieldType clone() { - return null; - } @Override public String typeName() { diff --git a/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java index 556adb0813fc1..cdc45eebec029 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.CompletionFieldMapper; import org.elasticsearch.index.mapper.CompletionFieldMapper.CompletionFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.suggest.AbstractSuggestionBuilderTestCase; @@ -169,14 +168,11 @@ protected void mutateSpecificParameters(CompletionSuggestionBuilder builder) thr @Override protected MappedFieldType mockFieldType(String fieldName, boolean analyzerSet) { if (analyzerSet == false) { - CompletionFieldType completionFieldType = new CompletionFieldType(fieldName, - CompletionFieldMapper.Defaults.FIELD_TYPE, null, null, Collections.emptyMap()); + CompletionFieldType completionFieldType = new CompletionFieldType(fieldName, null, Collections.emptyMap()); completionFieldType.setContextMappings(new ContextMappings(contextMappings)); return completionFieldType; } CompletionFieldType completionFieldType = new CompletionFieldType(fieldName, - CompletionFieldMapper.Defaults.FIELD_TYPE, - new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()), new NamedAnalyzer("fieldSearchAnalyzer", AnalyzerScope.INDEX, new SimpleAnalyzer()), Collections.emptyMap()); completionFieldType.setContextMappings(new ContextMappings(contextMappings)); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index fd9876b08ec56..a6f0f56b6cd04 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -65,7 +65,7 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; import org.elasticsearch.action.bulk.BulkAction; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -210,6 +210,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import static java.util.Collections.emptyMap; @@ -224,7 +225,6 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.iterableWithSize; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -552,31 +552,19 @@ public void testConcurrentSnapshotCreateAndDeleteOther() { createSnapshotResponse -> client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-2") .execute(createOtherSnapshotResponseStepListener)); - final StepListener deleteSnapshotStepListener = new StepListener<>(); + final StepListener deleteSnapshotStepListener = new StepListener<>(); continueOrDie(createOtherSnapshotResponseStepListener, - createSnapshotResponse -> client().admin().cluster().deleteSnapshot( - new DeleteSnapshotRequest(repoName, snapshotName), ActionListener.wrap( - resp -> deleteSnapshotStepListener.onResponse(true), - e -> { - final Throwable unwrapped = - ExceptionsHelper.unwrap(e, ConcurrentSnapshotExecutionException.class); - assertThat(unwrapped, instanceOf(ConcurrentSnapshotExecutionException.class)); - deleteSnapshotStepListener.onResponse(false); - }))); + createSnapshotResponse -> client().admin().cluster().prepareDeleteSnapshot( + repoName, snapshotName).execute(deleteSnapshotStepListener)); final StepListener createAnotherSnapshotResponseStepListener = new StepListener<>(); continueOrDie(deleteSnapshotStepListener, deleted -> { - if (deleted) { - // The delete worked out, creating a third snapshot - client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true) + client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true) .execute(createAnotherSnapshotResponseStepListener); - continueOrDie(createAnotherSnapshotResponseStepListener, createSnapshotResponse -> + continueOrDie(createAnotherSnapshotResponseStepListener, createSnapshotResponse -> assertEquals(createSnapshotResponse.getSnapshotInfo().state(), SnapshotState.SUCCESS)); - } else { - createAnotherSnapshotResponseStepListener.onResponse(null); - } }); deterministicTaskQueue.runAllRunnableTasks(); @@ -614,11 +602,16 @@ public void testBulkSnapshotDeleteWithAbort() { createIndexResponse -> client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) .setWaitForCompletion(true).execute(createSnapshotResponseStepListener)); - final StepListener createOtherSnapshotResponseStepListener = new StepListener<>(); + final int inProgressSnapshots = randomIntBetween(1, 5); + final StepListener> createOtherSnapshotResponseStepListener = new StepListener<>(); + final ActionListener createSnapshotListener = + new GroupedActionListener<>(createOtherSnapshotResponseStepListener, inProgressSnapshots); - continueOrDie(createSnapshotResponseStepListener, - createSnapshotResponse -> client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-2") - .execute(createOtherSnapshotResponseStepListener)); + continueOrDie(createSnapshotResponseStepListener, createSnapshotResponse -> { + for (int i = 0; i < inProgressSnapshots; i++) { + client().admin().cluster().prepareCreateSnapshot(repoName, "other-" + i).execute(createSnapshotListener); + } + }); final StepListener deleteSnapshotStepListener = new StepListener<>(); @@ -1009,6 +1002,66 @@ public void testSuccessfulSnapshotWithConcurrentDynamicMappingUpdates() { assertEquals(0, snapshotInfo.failedShards()); } + public void testRunConcurrentSnapshots() { + setupTestCluster(randomFrom(1, 3, 5), randomIntBetween(2, 10)); + + final String repoName = "repo"; + final List snapshotNames = IntStream.range(1, randomIntBetween(2, 4)) + .mapToObj(i -> "snapshot-" + i).collect(Collectors.toList()); + final String index = "test"; + final int shards = randomIntBetween(1, 10); + final int documents = randomIntBetween(1, 100); + + final TestClusterNodes.TestClusterNode masterNode = + testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state()); + + final StepListener> allSnapshotsListener = new StepListener<>(); + final ActionListener snapshotListener = + new GroupedActionListener<>(allSnapshotsListener, snapshotNames.size()); + final AtomicBoolean doneIndexing = new AtomicBoolean(false); + continueOrDie(createRepoAndIndex(repoName, index, shards), createIndexResponse -> { + for (String snapshotName : snapshotNames) { + scheduleNow(() -> client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName) + .setWaitForCompletion(true).execute(snapshotListener)); + } + final BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < documents; ++i) { + bulkRequest.add(new IndexRequest(index).source(Collections.singletonMap("foo", "bar" + i))); + } + final StepListener bulkResponseStepListener = new StepListener<>(); + client().bulk(bulkRequest, bulkResponseStepListener); + continueOrDie(bulkResponseStepListener, bulkResponse -> { + assertFalse("Failures in bulk response: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures()); + assertEquals(documents, bulkResponse.getItems().length); + doneIndexing.set(true); + }); + }); + + final AtomicBoolean doneSnapshotting = new AtomicBoolean(false); + continueOrDie(allSnapshotsListener, createSnapshotResponses -> { + for (CreateSnapshotResponse createSnapshotResponse : createSnapshotResponses) { + final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS)); + } + doneSnapshotting.set(true); + }); + + runUntil(() -> doneIndexing.get() && doneSnapshotting.get(), TimeUnit.MINUTES.toMillis(5L)); + SnapshotsInProgress finalSnapshotsInProgress = masterNode.clusterService.state().custom(SnapshotsInProgress.TYPE); + assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); + final Repository repository = masterNode.repositoriesService.repository(repoName); + Collection snapshotIds = getRepositoryData(repository).getSnapshotIds(); + assertThat(snapshotIds, hasSize(snapshotNames.size())); + + for (SnapshotId snapshotId : snapshotIds) { + final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId); + assertEquals(SnapshotState.SUCCESS, snapshotInfo.state()); + assertThat(snapshotInfo.indices(), containsInAnyOrder(index)); + assertEquals(shards, snapshotInfo.successfulShards()); + assertEquals(0, snapshotInfo.failedShards()); + } + } + private RepositoryData getRepositoryData(Repository repository) { final PlainActionFuture res = PlainActionFuture.newFuture(); repository.getRepositoryData(res); @@ -1498,7 +1551,7 @@ public void onFailure(final Exception e) { threadPool, shardStateAction, actionFilters, - new WriteMemoryLimits())), + new IndexingPressure(settings))), RetentionLeaseSyncer.EMPTY, client); final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService); @@ -1513,7 +1566,7 @@ allocationService, new AliasValidator(), shardLimitValidator, environment, index actionFilters, indexNameExpressionResolver )); final MappingUpdatedAction mappingUpdatedAction = new MappingUpdatedAction(settings, clusterSettings, clusterService); - final WriteMemoryLimits indexingMemoryLimits = new WriteMemoryLimits(); + final IndexingPressure indexingMemoryLimits = new IndexingPressure(settings); mappingUpdatedAction.setClient(client); actions.put(BulkAction.INSTANCE, new TransportBulkAction(threadPool, transportService, clusterService, @@ -1523,7 +1576,7 @@ allocationService, new AliasValidator(), shardLimitValidator, environment, index Collections.emptyList(), client), client, actionFilters, indexNameExpressionResolver, new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver), - new WriteMemoryLimits() + new IndexingPressure(settings) )); final TransportShardBulkAction transportShardBulkAction = new TransportShardBulkAction(settings, transportService, clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedAction, new UpdateHelper(scriptService), diff --git a/test/fixtures/azure-fixture/build.gradle b/test/fixtures/azure-fixture/build.gradle index 33025686ba6dd..3c292c78bd947 100644 --- a/test/fixtures/azure-fixture/build.gradle +++ b/test/fixtures/azure-fixture/build.gradle @@ -27,7 +27,7 @@ dependencies { } preProcessFixture { - dependsOn jar + dependsOn jar, configurations.runtimeClasspath doLast { file("${testFixturesDir}/shared").mkdirs() project.copy { diff --git a/test/fixtures/gcs-fixture/build.gradle b/test/fixtures/gcs-fixture/build.gradle index 3d6493f5f3c05..df0977af7e4a2 100644 --- a/test/fixtures/gcs-fixture/build.gradle +++ b/test/fixtures/gcs-fixture/build.gradle @@ -27,7 +27,7 @@ dependencies { } preProcessFixture { - dependsOn jar + dependsOn jar, configurations.runtimeClasspath doLast { file("${testFixturesDir}/shared").mkdirs() project.copy { diff --git a/test/fixtures/s3-fixture/build.gradle b/test/fixtures/s3-fixture/build.gradle index 5bb7e114a1500..d986410441481 100644 --- a/test/fixtures/s3-fixture/build.gradle +++ b/test/fixtures/s3-fixture/build.gradle @@ -27,7 +27,7 @@ dependencies { } preProcessFixture { - dependsOn jar + dependsOn jar, configurations.runtimeClasspath doLast { file("${testFixturesDir}/shared").mkdirs() project.copy { diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java index ba9d59282c7d6..387a6f0031117 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java @@ -55,4 +55,27 @@ public static IndexMetadata.Builder getIndexMetadataBuilderForIndex(Index index) public static DataStream.TimestampField createTimestampField(String fieldName) { return new DataStream.TimestampField(fieldName); } + + public static String generateMapping(String timestampFieldName) { + return "{\n" + + " \"properties\": {\n" + + " \"" + timestampFieldName + "\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + " }"; + } + + public static String generateMapping(String timestampFieldName, String type) { + return "{\n" + + " \"_data_stream_timestamp\": {\n" + + " \"path\": \"" + timestampFieldName + "\"\n" + + " }," + + " \"properties\": {\n" + + " \"" + timestampFieldName + "\": {\n" + + " \"type\": \"" + type + "\"\n" + + " }\n" + + " }\n" + + " }"; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java index 9b0181f355005..fd5f4f74bafef 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java @@ -84,7 +84,7 @@ List adjustNodesStats(List nodesStats) { .map(fsInfoPath -> diskUsageFunction.apply(discoveryNode, fsInfoPath)) .toArray(FsInfo.Path[]::new)), nodeStats.getTransport(), nodeStats.getHttp(), nodeStats.getBreaker(), nodeStats.getScriptStats(), nodeStats.getDiscoveryStats(), - nodeStats.getIngestStats(), nodeStats.getAdaptiveSelectionStats(), nodeStats.getScriptCacheStats()); + nodeStats.getIngestStats(), nodeStats.getAdaptiveSelectionStats(), nodeStats.getIndexingPressureStats()); }).collect(Collectors.toList()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 04a066e3361ad..f8057d7bfa8a2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -21,11 +21,15 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.mapper.MapperRegistry; @@ -35,7 +39,10 @@ import java.nio.file.Path; import java.util.Collections; +import static org.apache.lucene.util.LuceneTestCase.expectThrows; import static org.elasticsearch.test.ESTestCase.createTestAnalysis; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; public class MapperTestUtils { @@ -68,4 +75,20 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis mapperRegistry, () -> null, () -> false); } + + public static void assertConflicts(String mapping1, + String mapping2, + DocumentMapperParser + parser, String... conflicts) throws IOException { + DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping1)); + if (conflicts.length == 0) { + docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE); + } else { + Exception e = expectThrows(IllegalArgumentException.class, + () -> docMapper.merge(parser.parse("type", new CompressedXContent(mapping2)).mapping(), MergeReason.MAPPING_UPDATE)); + for (String conflict : conflicts) { + assertThat(e.getMessage(), containsString(conflict)); + } + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index 2cf5fec0b1bcb..fe6cbe3276614 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -20,50 +20,16 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.EqualsHashCodeTestUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** Base test case for subclasses of MappedFieldType */ -public abstract class FieldTypeTestCase extends ESTestCase { +public abstract class FieldTypeTestCase extends ESTestCase { public static final QueryShardContext MOCK_QSC = createMockQueryShardContext(true); public static final QueryShardContext MOCK_QSC_DISALLOW_EXPENSIVE = createMockQueryShardContext(false); - /** Create a default constructed fieldtype */ - protected abstract T createDefaultFieldType(String name, Map meta); - - @SuppressWarnings("unchecked") - private final List> modifiers = new ArrayList<>(List.of( - t -> createDefaultFieldType(t.name() + "-mutated", t.meta()), - t -> { - MappedFieldType copy = t.clone(); - copy.setBoost(t.boost() + 1); - return (T) copy; - }, - t -> { - MappedFieldType copy = t.clone(); - copy.setEagerGlobalOrdinals(t.eagerGlobalOrdinals() == false); - return (T) copy; - }, - t -> { - Map meta = new HashMap<>(t.meta()); - meta.put("bogus", "bogus"); - return createDefaultFieldType(t.name(), meta); - } - )); - - protected void addModifier(EqualsHashCodeTestUtils.MutateFunction modifier) { - modifiers.add(modifier); - } - protected QueryShardContext randomMockShardContext() { return randomFrom(MOCK_QSC, MOCK_QSC_DISALLOW_EXPENSIVE); } @@ -74,17 +40,4 @@ static QueryShardContext createMockQueryShardContext(boolean allowExpensiveQueri return queryShardContext; } - public void testClone() { - MappedFieldType fieldType = createDefaultFieldType("foo", Collections.emptyMap()); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(fieldType, MappedFieldType::clone); - } - - @SuppressWarnings("unchecked") - public void testEquals() { - for (EqualsHashCodeTestUtils.MutateFunction modifier : modifiers) { - EqualsHashCodeTestUtils.checkEqualsAndHashCode(createDefaultFieldType("foo", Collections.emptyMap()), - t -> (T) t.clone(), modifier); - } - } - } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 12130b5bf337b..223af014c955e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -52,15 +52,6 @@ public FakeFieldType(String name) { super(name, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); } - protected FakeFieldType(FakeFieldType ref) { - super(ref); - } - - @Override - public MappedFieldType clone() { - return new FakeFieldType(this); - } - @Override public String typeName() { return "faketype"; diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java index c115996809327..6d1168f419b12 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java @@ -105,7 +105,7 @@ public void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryS @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { listener.onResponse(null); } diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java index 2c1834f5e60c0..8e07290bda18d 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -139,7 +139,7 @@ protected void disableRepoConsistencyCheck(String reason) { } protected RepositoryData getRepositoryData(String repository) { - return getRepositoryData(internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(repository)); + return getRepositoryData(internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repository)); } protected RepositoryData getRepositoryData(Repository repository) { @@ -232,6 +232,12 @@ public static String blockMasterFromFinalizingSnapshotOnIndexFile(final String r return masterName; } + public static void blockMasterFromDeletingIndexNFile(String repositoryName) { + final String masterName = internalCluster().getMasterName(); + ((MockRepository)internalCluster().getInstance(RepositoriesService.class, masterName) + .repository(repositoryName)).setBlockOnDeleteIndexFile(); + } + public static String blockMasterFromFinalizingSnapshotOnSnapFile(final String repositoryName) { final String masterName = internalCluster().getMasterName(); ((MockRepository)internalCluster().getInstance(RepositoriesService.class, masterName) @@ -249,6 +255,11 @@ public static String blockNodeWithIndex(final String repositoryName, final Strin return null; } + public static void blockNodeOnAnyFiles(String repository, String nodeName) { + ((MockRepository) internalCluster().getInstance(RepositoriesService.class, nodeName) + .repository(repository)).setBlockOnAnyFiles(true); + } + public static void blockDataNode(String repository, String nodeName) { ((MockRepository) internalCluster().getInstance(RepositoriesService.class, nodeName) .repository(repository)).blockOnDataFiles(true); @@ -280,7 +291,8 @@ public static void waitForBlockOnAnyDataNode(String repository, TimeValue timeou assertTrue("No repository is blocked waiting on a data node", blocked); } - public static void unblockNode(final String repository, final String node) { + public void unblockNode(final String repository, final String node) { + logger.info("--> unblocking [{}] on node [{}]", repository, node); ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repository)).unblock(); } @@ -416,7 +428,7 @@ protected void awaitNoMoreRunningOperations(String viaNode) throws Exception { state.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY).hasDeletionsInProgress() == false); } - private void awaitClusterState(String viaNode, Predicate statePredicate) throws Exception { + protected void awaitClusterState(String viaNode, Predicate statePredicate) throws Exception { final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, viaNode); final ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, viaNode); final ClusterStateObserver observer = new ClusterStateObserver(clusterService, logger, threadPool.getThreadContext()); diff --git a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index 14e6daef1c775..b8cff6d52d653 100644 --- a/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/test/framework/src/main/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -42,6 +42,7 @@ import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.plugins.RepositoryPlugin; import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; import java.io.IOException; @@ -104,10 +105,12 @@ public long getFailureCount() { private final Environment env; - private volatile boolean blockOnControlFiles; + private volatile boolean blockOnAnyFiles; private volatile boolean blockOnDataFiles; + private volatile boolean blockOnDeleteIndexN; + /** Allows blocking on writing the index-N blob; this is a way to enforce blocking the * finalization of a snapshot, while permitting other IO operations to proceed unblocked. */ private volatile boolean blockOnWriteIndexFile; @@ -125,7 +128,7 @@ public MockRepository(RepositoryMetadata metadata, Environment environment, randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0); useLuceneCorruptionException = metadata.settings().getAsBoolean("use_lucene_corruption", false); maximumNumberOfFailures = metadata.settings().getAsLong("max_failure_number", 100L); - blockOnControlFiles = metadata.settings().getAsBoolean("block_on_control", false); + blockOnAnyFiles = metadata.settings().getAsBoolean("block_on_control", false); blockOnDataFiles = metadata.settings().getAsBoolean("block_on_data", false); blockAndFailOnWriteSnapFile = metadata.settings().getAsBoolean("block_on_snap", false); randomPrefix = metadata.settings().get("random", "default"); @@ -171,9 +174,10 @@ public synchronized void unblock() { blocked = false; // Clean blocking flags, so we wouldn't try to block again blockOnDataFiles = false; - blockOnControlFiles = false; + blockOnAnyFiles = false; blockOnWriteIndexFile = false; blockAndFailOnWriteSnapFile = false; + blockOnDeleteIndexN = false; this.notifyAll(); } @@ -181,6 +185,10 @@ public void blockOnDataFiles(boolean blocked) { blockOnDataFiles = blocked; } + public void setBlockOnAnyFiles(boolean blocked) { + blockOnAnyFiles = blocked; + } + public void setBlockAndFailOnWriteSnapFiles(boolean blocked) { blockAndFailOnWriteSnapFile = blocked; } @@ -189,6 +197,10 @@ public void setBlockOnWriteIndexFile(boolean blocked) { blockOnWriteIndexFile = blocked; } + public void setBlockOnDeleteIndexFile() { + blockOnDeleteIndexN = true; + } + public boolean blocked() { return blocked; } @@ -197,8 +209,8 @@ private synchronized boolean blockExecution() { logger.debug("[{}] Blocking execution", metadata.name()); boolean wasBlocked = false; try { - while (blockOnDataFiles || blockOnControlFiles || blockOnWriteIndexFile || - blockAndFailOnWriteSnapFile) { + while (blockOnDataFiles || blockOnAnyFiles || blockOnWriteIndexFile || + blockAndFailOnWriteSnapFile || blockOnDeleteIndexN) { blocked = true; this.wait(); wasBlocked = true; @@ -275,7 +287,7 @@ private void maybeIOExceptionOrBlock(String blobName) throws IOException { if (shouldFail(blobName, randomControlIOExceptionRate) && (incrementAndGetFailureCount() < maximumNumberOfFailures)) { logger.info("throwing random IOException for file [{}] at path [{}]", blobName, path()); throw new IOException("Random IOException"); - } else if (blockOnControlFiles) { + } else if (blockOnAnyFiles) { blockExecutionAndMaybeWait(blobName); } else if (blobName.startsWith("snap-") && blockAndFailOnWriteSnapFile) { blockExecutionAndFail(blobName); @@ -339,6 +351,15 @@ public DeleteResult delete() throws IOException { return deleteResult.add(deleteBlobCount, deleteByteCount); } + @Override + public void deleteBlobsIgnoringIfNotExists(List blobNames) throws IOException { + if (blockOnDeleteIndexN && blobNames.stream().anyMatch( + name -> name.startsWith(BlobStoreRepository.INDEX_FILE_PREFIX))) { + blockExecutionAndMaybeWait("index-{N}"); + } + super.deleteBlobsIgnoringIfNotExists(blobNames); + } + @Override public Map listBlobs() throws IOException { maybeIOExceptionOrBlock(""); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 410cc3cb6796e..9aa7b65330699 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -117,7 +117,6 @@ import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.node.NodeMocksPlugin; -import org.elasticsearch.node.RecoverySettingsChunkSizePlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -1902,9 +1901,6 @@ protected Collection> getMockPlugins() { mocks.add(MockFieldFilterPlugin.class); } } - if (randomBoolean()) { - mocks.add(RecoverySettingsChunkSizePlugin.class); - } if (addMockTransportService()) { mocks.add(getTestTransportPlugin()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 2c0e226845585..4f14ac5815507 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -36,7 +36,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterName; @@ -99,7 +99,6 @@ import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeService; import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.node.RecoverySettingsChunkSizePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; @@ -151,7 +150,6 @@ import static org.elasticsearch.discovery.FileBasedSeedHostsProvider.UNICAST_HOSTS_FILE; import static org.elasticsearch.node.Node.INITIAL_STATE_TIMEOUT_SETTING; import static org.elasticsearch.test.ESTestCase.assertBusy; -import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.NodeRoles.dataOnlyNode; import static org.elasticsearch.test.NodeRoles.masterOnlyNode; @@ -395,10 +393,6 @@ public InternalTestCluster( RandomNumbers.randomIntBetween(random, 1, 5)); builder.put(RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING.getKey(), RandomNumbers.randomIntBetween(random, 1, 4)); - if (mockPlugins.contains(RecoverySettingsChunkSizePlugin.class) && randomBoolean()) { - builder.put(RecoverySettingsChunkSizePlugin.CHUNK_SIZE_SETTING.getKey(), - new ByteSizeValue(RandomNumbers.randomIntBetween(random, 256, 10 * 1024 * 1024))); - } defaultSettings = builder.build(); executor = EsExecutors.newScaling("internal_test_cluster_executor", 0, Integer.MAX_VALUE, 0, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory("test_" + clusterName), new ThreadContext(Settings.EMPTY)); @@ -1171,13 +1165,13 @@ public void beforeIndexDeletion() throws Exception { private void assertAllPendingWriteLimitsReleased() throws Exception { assertBusy(() -> { for (NodeAndClient nodeAndClient : nodes.values()) { - WriteMemoryLimits writeMemoryLimits = getInstance(WriteMemoryLimits.class, nodeAndClient.name); - final long writeBytes = writeMemoryLimits.getWriteBytes(); + IndexingPressure indexingPressure = getInstance(IndexingPressure.class, nodeAndClient.name); + final long writeBytes = indexingPressure.getCurrentCoordinatingAndPrimaryBytes(); if (writeBytes > 0) { throw new AssertionError("pending write bytes [" + writeBytes + "] bytes on node [" + nodeAndClient.name + "]."); } - final long replicaWriteBytes = writeMemoryLimits.getReplicaWriteBytes(); + final long replicaWriteBytes = indexingPressure.getCurrentReplicaBytes(); if (replicaWriteBytes > 0) { throw new AssertionError("pending replica write bytes [" + writeBytes + "] bytes on node [" + nodeAndClient.name + "]."); @@ -2265,7 +2259,7 @@ public void ensureEstimatedStats() { NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node); CommonStatsFlags flags = new CommonStatsFlags(Flag.FieldData, Flag.QueryCache, Flag.Segments); NodeStats stats = nodeService.stats(flags, - false, false, false, false, false, false, false, false, false, false, false, false, false); + false, false, false, false, false, false, false, false, false, false, false, false, false, false); assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L)); assertThat("Query cache size must be 0 on node: " + stats.getNode(), diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index c73a458a20470..0acec6f9cf239 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -126,6 +126,19 @@ public static Map entityAsMap(Response response) throws IOExcept } } + /** + * Convert the entity from a {@link Response} into a list of maps. + */ + public static List entityAsList(Response response) throws IOException { + XContentType xContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); + // EMPTY and THROW are fine here because `.map` doesn't use named x content or deprecation + try (XContentParser parser = xContentType.xContent().createParser( + NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent())) { + return parser.list(); + } + } + /** * Does any node in the cluster being tested have x-pack installed? */ diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index 420e25cd30a10..631d2c6795f3e 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -76,4 +76,9 @@ protected boolean preserveILMPoliciesUponCompletion() { protected boolean preserveSLMPoliciesUponCompletion() { return true; } + + @Override + protected boolean preserveDataStreamsUponCompletion() { + return true; + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/MatchAssertionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/MatchAssertionTests.java index ddcffa7ac5adc..5d751702d5516 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/MatchAssertionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/MatchAssertionTests.java @@ -52,4 +52,14 @@ public void testNullInMap() { matchAssertion.doAssert(emptyMap(), matchAssertion.getExpectedValue())); assertThat(e.getMessage(), containsString("expected [null] but not found")); } + + public void testNullInMap2() { + XContentLocation xContentLocation = new XContentLocation(0, 0); + MatchAssertion matchAssertion = + new MatchAssertion(xContentLocation, "test_index.mappings.test_type.text.mapping.text.type", "type"); + matchAssertion.doAssert(singletonMap("a", null), matchAssertion.getExpectedValue()); + AssertionError e = expectThrows(AssertionError.class, () -> + matchAssertion.doAssert(emptyMap(), matchAssertion.getExpectedValue())); + assertThat(e.getMessage(), containsString("expected [null] but not found")); + } } diff --git a/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc index fea1b35f53476..c93bb546e409a 100644 --- a/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc +++ b/x-pack/docs/en/rest-api/security/get-builtin-privileges.asciidoc @@ -100,6 +100,7 @@ A successful call returns an object with "cluster" and "index" fields. ], "index" : [ "all", + "auto_configure", "create", "create_doc", "create_index", diff --git a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc index 2683388d10725..09ed883f2e679 100644 --- a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc +++ b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc @@ -440,7 +440,7 @@ PUT /_security/role_mapping/oidc-example <1> The `example_role` role is *not* a builtin Elasticsearch role. This example assumes that you have created a custom role of your own, with -appropriate access to your <> and +appropriate access to your <> and {kibana-ref}/kibana-privileges.html#kibana-feature-privileges[Kibana features]. The user properties that are mapped via the realm configuration are used to process diff --git a/x-pack/docs/en/security/authentication/realm-chains.asciidoc b/x-pack/docs/en/security/authentication/realm-chains.asciidoc index b130baa45a553..6c609dfd9223f 100644 --- a/x-pack/docs/en/security/authentication/realm-chains.asciidoc +++ b/x-pack/docs/en/security/authentication/realm-chains.asciidoc @@ -93,5 +93,6 @@ realms, authentication fails. See <> for more details. -NOTE: Delegated authorization requires a -https://www.elastic.co/subscriptions[Platinum or Trial license]. +NOTE: Delegated authorization requires that you have a +{subscriptions}[subscription] that includes custom authentication and +authorization realms. diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc index 40dfbabc56767..90b4b5fc76138 100644 --- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc +++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc @@ -656,7 +656,7 @@ PUT /_security/role_mapping/saml-example <1> The `example_role` role is *not* a builtin Elasticsearch role. This example assumes that you have created a custom role of your own, with -appropriate access to your <> and +appropriate access to your <> and {kibana-ref}/kibana-privileges.html#kibana-feature-privileges[Kibana features]. The attributes that are mapped via the realm configuration are used to process diff --git a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc index 403e333ea0b73..edee5cddca851 100644 --- a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc +++ b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc @@ -1,10 +1,118 @@ [role="xpack"] [[securing-aliases]] -=== Granting privileges for indices and aliases +=== Granting privileges for data streams and index aliases -Elasticsearch allows to execute operations against -<>, -which are effectively virtual indices. An alias points to one or more indices, +{es} {security-features} allow you to secure operations executed against +<> and <>. + +[[data-stream-privileges]] +==== Data stream privileges + +A data stream consists of one or more backing indices, which store the stream's +data. Most requests sent to a data stream are routed to one or more of these +backing indices. + +Similar to an index, you can use <> +to control access to a data stream. Any role or user granted privileges to a +data stream are automatically granted the same privileges to its backing +indices. + +.*Example* +[%collapsible] +==== +`logs` is a data stream that consists of two backing indices: `.ds-logs-000001` +and `.ds-logs-000002`. + +A user is granted the `read` privilege to the `logs` data stream. + +[source,js] +-------------------------------------------------- +{ + "names" : [ "logs" ], + "privileges" : [ "read" ] +} +-------------------------------------------------- +// NOTCONSOLE + +Because the user is automatically granted the same privileges to the stream's +backing indices, the user can retrieve a document directly from `.ds-logs-000002`: + +//// +[source,console] +---- +PUT /_index_template/logs_data_stream +{ + "index_patterns": [ "logs*" ], + "data_stream": { + "timestamp_field": "@timestamp" + }, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } +} + +PUT /_data_stream/logs + +POST /logs/_rollover/ + +PUT /logs/_create/2?refresh=wait_for +{ + "@timestamp": "2020-12-07T11:06:07.000Z" +} +---- +//// + +[source,console] +---- +GET /.ds-logs-000002/_doc/2 +---- +// TEST[continued] + +Later the `logs` data stream <>. +This creates a new backing index: `.ds-logs-000003`. Because the user still has +the `read` privilege for the `logs` data stream, the user can retrieve documents +directly from `.ds-logs-000003`: + +//// +[source,console] +---- +POST /logs/_rollover/ + +PUT /logs/_create/2?refresh=wait_for +{ + "@timestamp": "2020-12-07T11:06:07.000Z" +} +---- +// TEST[continued] +//// + +[source,console] +---- +GET /.ds-logs-000003/_doc/2 +---- +// TEST[continued] + +//// +[source,console] +---- +DELETE /_data_stream/* + +DELETE /_index_template/* +---- +// TEST[continued] +//// +==== + +[[index-alias-privileges]] +==== Index alias privileges + +An index alias points to one or more indices, holds metadata and potentially a filter. The {es} {security-features} treat aliases and indices the same. Privileges for indices actions are granted on specific indices or diff --git a/x-pack/docs/en/security/authorization/built-in-roles.asciidoc b/x-pack/docs/en/security/authorization/built-in-roles.asciidoc index 6fbaed6a88bcc..297839d53502c 100644 --- a/x-pack/docs/en/security/authorization/built-in-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/built-in-roles.asciidoc @@ -152,7 +152,7 @@ Grants the necessary privileges to create snapshots of **all** the indices and to view their metadata. This role enables users to view the configuration of existing snapshot repositories and snapshot details. It does not grant authority to remove or add repositories or to restore snapshots. It also does not enable -to change index settings or to read or update index data. +to change index settings or to read or update data stream or index data. [[built-in-roles-superuser]] `superuser`:: Grants full access to the cluster, including all indices and data. A user with diff --git a/x-pack/docs/en/security/authorization/document-level-security.asciidoc b/x-pack/docs/en/security/authorization/document-level-security.asciidoc index bef024975dcaa..b294cd36910ae 100644 --- a/x-pack/docs/en/security/authorization/document-level-security.asciidoc +++ b/x-pack/docs/en/security/authorization/document-level-security.asciidoc @@ -6,13 +6,13 @@ Document level security restricts the documents that users have read access to. In particular, it restricts which documents can be accessed from document-based read APIs. -To enable document level security, you use a query to specify the documents that -each role can access. The document query is associated with a particular index -or index pattern and operates in conjunction with the privileges specified for -the indices. +To enable document level security, you use a query to specify the documents that +each role can access. The document query is associated with a particular data +stream, index, or wildcard (`*`) pattern and operates in conjunction with the +privileges specified for the data streams and indices. The following role definition grants read access only to documents that -belong to the `click` category within all the `events-*` indices: +belong to the `click` category within all the `events-*` data streams and indices: [source,console] -------------------------------------------------- diff --git a/x-pack/docs/en/security/authorization/field-and-document-access-control.asciidoc b/x-pack/docs/en/security/authorization/field-and-document-access-control.asciidoc index 280cbec8b00b8..227e30073a46e 100644 --- a/x-pack/docs/en/security/authorization/field-and-document-access-control.asciidoc +++ b/x-pack/docs/en/security/authorization/field-and-document-access-control.asciidoc @@ -2,16 +2,16 @@ [[field-and-document-access-control]] === Setting up field and document level security -You can control access to data within an index by adding field and document level +You can control access to data within a data stream or index by adding field and document level security permissions to a role. <> restrict access to particular fields within a document. <> restrict access -to particular documents within an index. +to particular documents. NOTE: Document and field level security is currently meant to operate with read-only privileged accounts. Users with document and field level -security enabled for an index should not perform write operations. +security enabled for a data stream or index should not perform write operations. A role can define both field and document level permissions on a per-index basis. A role that doesn’t specify field level permissions grants access to ALL fields. @@ -22,7 +22,7 @@ to ALL documents in the index. ===================================================================== When assigning users multiple roles, be careful that you don't inadvertently grant wider access than intended. Each user has a single set of field level and -document level permissions per index. See <>. +document level permissions per data stream or index. See <>. ===================================================================== NOTE: Document- and field-level security disables the @@ -32,11 +32,11 @@ NOTE: Document- and field-level security disables the ==== Multiple roles with document and field level security A user can have many roles and each role can define different permissions on the -same index. It is important to understand the behavior of document and field +same data stream or index. It is important to understand the behavior of document and field level security in this scenario. Document level security takes into account each role held by the user and -combines each document level security query for a given index with an "OR". This +combines each document level security query for a given data stream or index with an "OR". This means that only one of the role queries must match for a document to be returned. For example, if a role grants access to an index without document level security and another grants access with document level security, document level security @@ -44,7 +44,7 @@ is not applied; the user with both roles has access to all of the documents in the index. Field level security takes into account each role the user has and combines -all of the fields listed into a single set for each index. For example, if a +all of the fields listed into a single set for each data stream or index. For example, if a role grants access to an index without field level security and another grants access with field level security, field level security is not be applied for that index; the user with both roles has access to all of the fields in the diff --git a/x-pack/docs/en/security/authorization/field-level-security.asciidoc b/x-pack/docs/en/security/authorization/field-level-security.asciidoc index e31247a7b7f5e..2137bc7687ff6 100644 --- a/x-pack/docs/en/security/authorization/field-level-security.asciidoc +++ b/x-pack/docs/en/security/authorization/field-level-security.asciidoc @@ -8,11 +8,11 @@ read APIs. To enable field level security, specify the fields that each role can access as part of the indices permissions in a role definition. Field level security is -thus bound to a well-defined set of indices (and potentially a set of +thus bound to a well-defined set of data streams or indices (and potentially a set of <>). The following role definition grants read access only to the `category`, -`@timestamp`, and `message` fields in all the `events-*` indices. +`@timestamp`, and `message` fields in all the `events-*` data streams and indices. [source,console] -------------------------------------------------- @@ -162,7 +162,7 @@ An empty array for `grant` (for example, `"grant" : []`) means that access has not been granted to any fields. When a user has several roles that specify field level permissions, the -resulting field level permissions per index are the union of the individual role +resulting field level permissions per data stream or index are the union of the individual role permissions. For example, if these two roles are merged: [source,console] diff --git a/x-pack/docs/en/security/authorization/managing-roles.asciidoc b/x-pack/docs/en/security/authorization/managing-roles.asciidoc index 4d8329c2cf0ef..7a9a7a9eeed6a 100644 --- a/x-pack/docs/en/security/authorization/managing-roles.asciidoc +++ b/x-pack/docs/en/security/authorization/managing-roles.asciidoc @@ -55,14 +55,14 @@ The following describes the structure of an indices permissions entry: ------- // NOTCONSOLE -<1> A list of indices (or index name patterns) to which the permissions in this - entry apply. +<1> A list of data streams, indices, and index aliases to which the permissions + in this entry apply. Wildcard (`*`) expressions are supported. <2> The index level privileges the owners of the role have on the associated - indices (those indices that are specified in the `names` field) + data streams and indices specified in the `names` argument. <3> Specification for document fields the owners of the role have read access to. See <> for details. <4> A search query that defines the documents the owners of the role have read - access to. A document within the associated indices must match this query + access to. A document within the associated data streams and indices must match this query in order for it to be accessible by the owners of the role. <5> Restricted indices are a special category of indices that are used internally to store configuration data. Only internal system @@ -75,8 +75,8 @@ The following describes the structure of an indices permissions entry: [TIP] ============================================================================== -When specifying index names, you can use indices and aliases with their full -names or regular expressions that refer to multiple indices. +The `names` parameter accepts wildcard and regular expressions that may refer to +multiple data streams, indices, and index aliases. * Wildcard (default) - simple wildcard matching where `*` is a placeholder for zero or more characters, `?` is a placeholder for a single character diff --git a/x-pack/docs/en/security/authorization/overview.asciidoc b/x-pack/docs/en/security/authorization/overview.asciidoc index feb2014e30ee3..97ad0ead69f8d 100644 --- a/x-pack/docs/en/security/authorization/overview.asciidoc +++ b/x-pack/docs/en/security/authorization/overview.asciidoc @@ -34,7 +34,7 @@ see <>. _Permissions_:: A set of one or more privileges against a secured resource. Permissions can easily be described in words, here are few examples: - * `read` privilege on the `products` index + * `read` privilege on the `products` data stream or index * `manage` privilege on the cluster * `run_as` privilege on `john` user * `read` privilege on documents that match query X diff --git a/x-pack/docs/en/security/authorization/privileges.asciidoc b/x-pack/docs/en/security/authorization/privileges.asciidoc index 0db5300b66bc5..936461a05acb6 100644 --- a/x-pack/docs/en/security/authorization/privileges.asciidoc +++ b/x-pack/docs/en/security/authorization/privileges.asciidoc @@ -155,7 +155,18 @@ cluster to enable <>. [horizontal] `all`:: -Any action on an index +Any action on an index or data stream. + +`auto_configure`:: +Permits auto-creation of indices and data streams. An auto-create action is the +result of an <> or <> request that targets a +non-existent index or data stream rather than an explicit +<> or +<> request. Also permits +auto-update of mappings on indices and data streams if they do not contradict +existing mappings. An auto-update mapping action is the result of an index or +bulk request on an index or data stream that contains new fields that may +be mapped rather than an explicit <> request. `create`:: Privilege to index documents. Also grants access to the update mapping @@ -190,15 +201,16 @@ privilege (and no higher privilege such as `index` or `write`), you must ensure -- `create_index`:: -Privilege to create an index. A create index request may contain aliases to be -added to the index once created. In that case the request requires the `manage` -privilege as well, on both the index and the aliases names. +Privilege to create an index or data stream. A create index request may contain +aliases to be added to the index once created. In that case the request +requires the `manage` privilege as well, on both the index and the aliases +names. `delete`:: Privilege to delete documents. `delete_index`:: -Privilege to delete an index. +Privilege to delete an index or data stream. `index`:: Privilege to index and update documents. Also grants access to the update @@ -209,9 +221,9 @@ Permits refresh, flush, synced flush and force merge index administration operat No privilege to read or write index data or otherwise manage the index. `manage`:: -All `monitor` privileges plus index administration (aliases, analyze, cache clear, -close, delete, exists, flush, mapping, open, force merge, refresh, settings, -search shards, templates, validate). +All `monitor` privileges plus index and data stream administration (aliases, +analyze, cache clear, close, delete, exists, flush, mapping, open, force merge, +refresh, settings, search shards, templates, validate). `manage_follow_index`:: All actions that are required to manage the lifecycle of a follower index, which @@ -220,8 +232,8 @@ index. This privilege is necessary only on clusters that contain follower indice `manage_ilm`:: All {Ilm} operations relating to managing the execution of policies of an index -This includes operations like retrying policies, and removing a policy -from an index. +or data stream. This includes operations such as retrying policies and removing +a policy from an index or data stream. `manage_leader_index`:: All actions that are required to manage the lifecycle of a leader index, which @@ -241,9 +253,10 @@ clear_scroll, search, suggest, tv). Read-only access to the search action from a <>. `view_index_metadata`:: -Read-only access to index metadata (aliases, aliases exists, get index, exists, field mappings, -mappings, search shards, type exists, validate, warmers, settings, ilm). This -privilege is primarily available for use by {kib} users. +Read-only access to index and data stream metadata (aliases, aliases exists, +get index, get data stream, exists, field mappings, mappings, search shards, +type exists, validate, warmers, settings, ilm). This privilege is available +for use primarily by {kib} users. `write`:: Privilege to perform all write operations to documents, which includes the diff --git a/x-pack/docs/en/security/authorization/set-security-user.asciidoc b/x-pack/docs/en/security/authorization/set-security-user.asciidoc index 7602b37ffa18d..d80134e94d1a8 100644 --- a/x-pack/docs/en/security/authorization/set-security-user.asciidoc +++ b/x-pack/docs/en/security/authorization/set-security-user.asciidoc @@ -12,7 +12,7 @@ role query for document level security. This is a situation where the <> ingest processor can help. NOTE: Document level security doesn't apply to write APIs. You must use unique -ids for each user that uses the same index, otherwise they might overwrite other +ids for each user that uses the same data stream or index, otherwise they might overwrite other users' documents. The ingest processor just adds properties for the current authenticated user to the documents that are being indexed. diff --git a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster-kibana.asciidoc b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster-kibana.asciidoc index f9bcd86889c17..4ba29afcb4aad 100644 --- a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster-kibana.asciidoc +++ b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster-kibana.asciidoc @@ -2,21 +2,20 @@ ==== {ccs-cap} and {kib} When {kib} is used to search across multiple clusters, a two-step authorization -process determines whether or not the user can access indices on a remote +process determines whether or not the user can access data streams and indices on a remote cluster: * First, the local cluster determines if the user is authorized to access remote clusters. (The local cluster is the cluster {kib} is connected to.) * If they are, the remote cluster then determines if the user has access -to the specified indices. +to the specified data streams and indices. To grant {kib} users access to remote clusters, assign them a local role -with read privileges to indices on the remote clusters. You specify remote -cluster indices as `:`. +with read privileges to indices on the remote clusters. You specify data streams and indices in a remote cluster as `:`. -To enable users to actually read the remote indices, you must create a matching +To enable users to actually read the remote data streams and indices, you must create a matching role on the remote clusters that grants the `read_cross_cluster` privilege -and access to the appropriate indices. +and access to the appropriate data streams and indices. For example, if {kib} is connected to the cluster where you're actively indexing {ls} data (your _local cluster_) and you're periodically diff --git a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc index 8fe479805b253..0aaac8f837f98 100644 --- a/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc +++ b/x-pack/docs/en/security/ccs-clients-integrations/cross-cluster.asciidoc @@ -76,7 +76,8 @@ PUT _cluster/settings ==== Example configuration of cross cluster search In the following example, we will configure the user `alice` to have permissions -to search any index starting with `logs-` in cluster `two` from cluster `one`. +to search any data stream or index starting with `logs-` in cluster `two` from +cluster `one`. First, enable cluster `one` to perform cross cluster search on remote cluster `two` by running the following request as the superuser on cluster `one`: diff --git a/x-pack/docs/en/security/fips-140-compliance.asciidoc b/x-pack/docs/en/security/fips-140-compliance.asciidoc index 35188a8218ba5..bfc3fb4725584 100644 --- a/x-pack/docs/en/security/fips-140-compliance.asciidoc +++ b/x-pack/docs/en/security/fips-140-compliance.asciidoc @@ -21,23 +21,24 @@ For {es}, adherence to FIPS 140-2 is ensured by [float] === Upgrade considerations -If you plan to upgrade your existing Cluster to a version that can be run in +If you plan to upgrade your existing cluster to a version that can be run in a FIPS 140-2 enabled JVM, the suggested approach is to first perform a rolling upgrade to the new version in your existing JVM and perform all necessary configuration changes in preparation for running in fips mode. You can then perform a rolling restart of the nodes, this time starting each node in the FIPS -140-2 JVM. This will allow {es} to take care of a couple of things automatically for you: +140-2 JVM. This enables {es} to take care of a couple of things automatically for you: -- <> will be upgraded to the latest format version as +- <> will be upgraded to the latest format version as previous format versions cannot be loaded in a FIPS 140-2 JVM. - Self-generated trial licenses will be upgraded to the latest format that is compliant with FIPS 140-2. -If you are on a appropriate license level (platinum) you can elect to perform -a rolling upgrade while at the same time running each upgraded node in a -FIPS 140-2 JVM. In this case, you would need to also regenerate your -`elasticsearch.keystore` and migrate all secure settings to it, in addition to the -necessary configuration changes outlined below, before starting each node. +If you have a {subscriptions}[subscription] that supports FIPS 140-2 mode, you +can elect to perform a rolling upgrade while at the same time running each +upgraded node in a FIPS 140-2 JVM. In this case, you would need to also +regenerate your `elasticsearch.keystore` and migrate all secure settings to it, +in addition to the necessary configuration changes outlined below, before +starting each node. [float] === Configuring {es} for FIPS 140-2 diff --git a/x-pack/docs/en/security/get-started-security.asciidoc b/x-pack/docs/en/security/get-started-security.asciidoc index 949cfad6b8fc6..9e4f53c351b1f 100644 --- a/x-pack/docs/en/security/get-started-security.asciidoc +++ b/x-pack/docs/en/security/get-started-security.asciidoc @@ -155,7 +155,7 @@ themselves, and run the `authenticate` API. If you want them to do more than that, you need to give them one or more _roles_. Each role defines a specific set of actions (such as read, create, or delete) -that can be performed on specific secured resources (such as indices, aliases, +that can be performed on specific secured resources (such as data streams, indices, aliases, documents, fields, or clusters). To help you get up and running, there are built-in roles. diff --git a/x-pack/docs/en/security/limitations.asciidoc b/x-pack/docs/en/security/limitations.asciidoc index d3cd6e589e2bb..da3f0acf4340e 100644 --- a/x-pack/docs/en/security/limitations.asciidoc +++ b/x-pack/docs/en/security/limitations.asciidoc @@ -18,18 +18,18 @@ source or not) and therefore we cannot guarantee their compliance with officially supported on clusters with {security-features} enabled. [float] -=== Changes in index wildcard behavior +=== Changes in wildcard behavior Elasticsearch clusters with the {security-features} enabled apply the `/_all` -wildcard, and all other wildcards, to the indices that the current user has -privileges for, not the set of all indices on the cluster. +wildcard, and all other wildcards, to the data streams, indices, and index aliases that the current user has +privileges for, not all data streams, indices, and index aliases on the cluster. [float] === Multi document APIs Multi get and multi term vectors API throw IndexNotFoundException when trying to access non existing indices that the user is -not authorized for. By doing that they leak information regarding the fact that the index doesn't exist, while the user is not -authorized to know anything about those indices. +not authorized for. By doing that they leak information regarding the fact that the data stream or index doesn't exist, while the user is not +authorized to know anything about those data streams or indices. [float] === Filtered index aliases @@ -44,14 +44,14 @@ documents through the [float] === Field and document level security limitations -When a user's role enables document or field level security for an index: +When a user's role enables document or field level security for a data stream or index: * The user cannot perform write operations: ** The update API isn't supported. ** Update requests included in bulk requests aren't supported. * The request cache is disabled for search requests. -When a user's role enables document level security for an index: +When a user's role enables document level security for a data stream or index: * Document level security isn't applied for APIs that aren't document based. An example is the field stats API. diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 97fd3aed8b704..392e75b97a751 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.analytics.action.AnalyticsInfoTransportAction; import org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction; import org.elasticsearch.xpack.analytics.action.TransportAnalyticsStatsAction; -import org.elasticsearch.xpack.analytics.aggregations.metrics.AnalyticsAggregatorFactory; +import org.elasticsearch.xpack.analytics.aggregations.AnalyticsAggregatorFactory; import org.elasticsearch.xpack.analytics.normalize.NormalizePipelineAggregationBuilder; import org.elasticsearch.xpack.analytics.boxplot.BoxplotAggregationBuilder; import org.elasticsearch.xpack.analytics.boxplot.InternalBoxplot; @@ -147,7 +147,8 @@ public List> getAggregationExtentions() { AnalyticsAggregatorFactory::registerPercentileRanksAggregator, AnalyticsAggregatorFactory::registerHistoBackedSumAggregator, AnalyticsAggregatorFactory::registerHistoBackedValueCountAggregator, - AnalyticsAggregatorFactory::registerHistoBackedAverageAggregator + AnalyticsAggregatorFactory::registerHistoBackedAverageAggregator, + AnalyticsAggregatorFactory::registerHistoBackedHistogramAggregator ); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsAggregatorFactory.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java similarity index 78% rename from x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsAggregatorFactory.java rename to x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java index 24e25793c5866..9ae2eb6109813 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/AnalyticsAggregatorFactory.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/AnalyticsAggregatorFactory.java @@ -3,8 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.analytics.aggregations.metrics; +package org.elasticsearch.xpack.analytics.aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorSupplier; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.MetricAggregatorSupplier; import org.elasticsearch.search.aggregations.metrics.PercentileRanksAggregationBuilder; @@ -15,6 +17,14 @@ import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.xpack.analytics.aggregations.bucket.histogram.HistoBackedHistogramAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedAvgAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedHDRPercentileRanksAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedHDRPercentilesAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedSumAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedTDigestPercentileRanksAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedTDigestPercentilesAggregator; +import org.elasticsearch.xpack.analytics.aggregations.metrics.HistoBackedValueCountAggregator; import org.elasticsearch.xpack.analytics.aggregations.support.AnalyticsValuesSourceType; public class AnalyticsAggregatorFactory { @@ -83,4 +93,11 @@ public static void registerHistoBackedAverageAggregator(ValuesSourceRegistry.Bui (MetricAggregatorSupplier) HistoBackedAvgAggregator::new ); } + + public static void registerHistoBackedHistogramAggregator(ValuesSourceRegistry.Builder builder) { + builder.register(HistogramAggregationBuilder.NAME, + AnalyticsValuesSourceType.HISTOGRAM, + (HistogramAggregatorSupplier) HistoBackedHistogramAggregator::new + ); + } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java new file mode 100644 index 0000000000000..d99bda737e2ed --- /dev/null +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregator.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.bucket.histogram; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.HistogramValue; +import org.elasticsearch.index.fielddata.HistogramValues; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregator; +import org.elasticsearch.search.aggregations.bucket.histogram.DoubleBounds; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.analytics.aggregations.support.HistogramValuesSource; + +import java.io.IOException; +import java.util.Map; + +public class HistoBackedHistogramAggregator extends AbstractHistogramAggregator { + + private final HistogramValuesSource.Histogram valuesSource; + + public HistoBackedHistogramAggregator( + String name, + AggregatorFactories factories, + double interval, + double offset, + BucketOrder order, + boolean keyed, + long minDocCount, + double minBound, + double maxBound, + DoubleBounds hardBounds, + ValuesSourceConfig valuesSourceConfig, + SearchContext context, + Aggregator parent, + CardinalityUpperBound cardinalityUpperBound, + Map metadata) throws IOException { + super(name, factories, interval, offset, order, keyed, minDocCount, minBound, maxBound, hardBounds, + valuesSourceConfig.format(), context, parent, cardinalityUpperBound, metadata); + + // TODO: Stop using null here + this.valuesSource = valuesSourceConfig.hasValues() ? (HistogramValuesSource.Histogram) valuesSourceConfig.getValuesSource() : null; + + // Sub aggregations are not allowed when running histogram agg over histograms + if (subAggregators().length > 0) { + throw new IllegalArgumentException("Histogram aggregation on histogram fields does not support sub-aggregations"); + } + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, + final LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + + final HistogramValues values = valuesSource.getHistogramValues(ctx); + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long owningBucketOrd) throws IOException { + if (values.advanceExact(doc)) { + final HistogramValue sketch = values.histogram(); + + double previousKey = Double.NEGATIVE_INFINITY; + while (sketch.next()) { + final double value = sketch.value(); + final int count = sketch.count(); + + double key = Math.floor((value - offset) / interval); + assert key >= previousKey; + long bucketOrd = bucketOrds.add(owningBucketOrd, Double.doubleToLongBits(key)); + if (bucketOrd < 0) { // already seen + bucketOrd = -1 - bucketOrd; + collectExistingBucket(sub, doc, bucketOrd); + } else { + collectBucket(sub, doc, bucketOrd); + } + // We have added the document already. We should increment doc_count by count - 1 + // so that we have added it count times. + incrementBucketDocCount(bucketOrd, count - 1); + previousKey = key; + } + } + } + }; + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregator.java index 424c17e4f91b0..ae3bfdb128f18 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedAvgAggregator.java @@ -32,7 +32,7 @@ * Average aggregator operating over histogram datatypes {@link HistogramValuesSource} * The aggregation computes weighted average by taking counts into consideration for each value */ -class HistoBackedAvgAggregator extends NumericMetricsAggregator.SingleValue { +public class HistoBackedAvgAggregator extends NumericMetricsAggregator.SingleValue { private final HistogramValuesSource.Histogram valuesSource; @@ -41,7 +41,7 @@ class HistoBackedAvgAggregator extends NumericMetricsAggregator.SingleValue { DoubleArray compensations; DocValueFormat format; - HistoBackedAvgAggregator( + public HistoBackedAvgAggregator( String name, ValuesSourceConfig valuesSourceConfig, SearchContext context, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java index c88be4a3d27b2..1cf231e58d56b 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentileRanksAggregator.java @@ -17,9 +17,9 @@ import java.io.IOException; import java.util.Map; -class HistoBackedHDRPercentileRanksAggregator extends AbstractHistoBackedHDRPercentilesAggregator { +public class HistoBackedHDRPercentileRanksAggregator extends AbstractHistoBackedHDRPercentilesAggregator { - HistoBackedHDRPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + public HistoBackedHDRPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] percents, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat format, Map metadata) throws IOException { super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, format, metadata); diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java index a10d3086f28e0..2c50b6e14b37a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedHDRPercentilesAggregator.java @@ -19,8 +19,8 @@ public class HistoBackedHDRPercentilesAggregator extends AbstractHistoBackedHDRPercentilesAggregator { - HistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, double[] percents, - int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, + public HistoBackedHDRPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, + double[] percents, int numberOfSignificantValueDigits, boolean keyed, DocValueFormat formatter, Map metadata) throws IOException { super(name, valuesSource, context, parent, percents, numberOfSignificantValueDigits, keyed, formatter, metadata); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregator.java index 662c21a6bf20f..f7796ac02377d 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedSumAggregator.java @@ -33,7 +33,7 @@ * The aggregator sums each histogram value multiplied by its count. * Eg for a histogram of response times, this is an approximate "total time spent". */ -class HistoBackedSumAggregator extends NumericMetricsAggregator.SingleValue { +public class HistoBackedSumAggregator extends NumericMetricsAggregator.SingleValue { private final HistogramValuesSource.Histogram valuesSource; private final DocValueFormat format; @@ -41,7 +41,7 @@ class HistoBackedSumAggregator extends NumericMetricsAggregator.SingleValue { private DoubleArray sums; private DoubleArray compensations; - HistoBackedSumAggregator( + public HistoBackedSumAggregator( String name, ValuesSourceConfig valuesSourceConfig, SearchContext context, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java index d2481e05d7207..7ec2e602c5d9a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentileRanksAggregator.java @@ -17,9 +17,9 @@ import java.io.IOException; import java.util.Map; -class HistoBackedTDigestPercentileRanksAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { +public class HistoBackedTDigestPercentileRanksAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { - HistoBackedTDigestPercentileRanksAggregator(String name, + public HistoBackedTDigestPercentileRanksAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java index 677242ce51f42..d72b7f48547dd 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/aggregations/metrics/HistoBackedTDigestPercentilesAggregator.java @@ -17,9 +17,9 @@ import java.io.IOException; import java.util.Map; -class HistoBackedTDigestPercentilesAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { +public class HistoBackedTDigestPercentilesAggregator extends AbstractHistoBackedTDigestPercentilesAggregator { - HistoBackedTDigestPercentilesAggregator(String name, + public HistoBackedTDigestPercentilesAggregator(String name, ValuesSource valuesSource, SearchContext context, Aggregator parent, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 6d07c70974e21..685eda193d0b9 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -121,7 +121,6 @@ public static class TypeParser implements Mapper.TypeParser { public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new HistogramFieldMapper.Builder(name); - TypeParsers.parseMeta(builder, name, node); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String propName = entry.getKey(); @@ -130,6 +129,10 @@ public Mapper.Builder parse(String name, Map node, Pars builder.ignoreMalformed(XContentMapValues.nodeBooleanValue(propNode, name + "." + Names.IGNORE_MALFORMED)); iterator.remove(); } + if (propName.equals("meta")) { + builder.meta(TypeParsers.parseMeta(propName, propNode)); + iterator.remove(); + } } return builder; } @@ -167,20 +170,11 @@ public HistogramFieldType(String name, boolean hasDocValues, Map super(name, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); } - HistogramFieldType(HistogramFieldType ref) { - super(ref); - } - @Override public String typeName() { return CONTENT_TYPE; } - @Override - public MappedFieldType clone() { - return new HistogramFieldType(this); - } - @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java new file mode 100644 index 0000000000000..a0c296b63542a --- /dev/null +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/bucket/histogram/HistoBackedHistogramAggregatorTests.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.aggregations.bucket.histogram; + +import com.tdunning.math.stats.Centroid; +import com.tdunning.math.stats.TDigest; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; +import org.elasticsearch.xpack.analytics.mapper.HistogramFieldMapper; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static java.util.Collections.singleton; + +public class HistoBackedHistogramAggregatorTests extends AggregatorTestCase { + + private static final String FIELD_NAME = "field"; + + public void testHistograms() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30}))); + + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(FIELD_NAME) + .interval(5); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, defaultFieldType(FIELD_NAME)); + assertEquals(9, histogram.getBuckets().size()); + assertEquals(-10d, histogram.getBuckets().get(0).getKey()); + assertEquals(1, histogram.getBuckets().get(0).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(2).getKey()); + assertEquals(3, histogram.getBuckets().get(2).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(3).getKey()); + assertEquals(3, histogram.getBuckets().get(3).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(4).getKey()); + assertEquals(4, histogram.getBuckets().get(4).getDocCount()); + assertEquals(15d, histogram.getBuckets().get(5).getKey()); + assertEquals(0, histogram.getBuckets().get(5).getDocCount()); + assertEquals(20d, histogram.getBuckets().get(6).getKey()); + assertEquals(2, histogram.getBuckets().get(6).getDocCount()); + assertEquals(25d, histogram.getBuckets().get(7).getKey()); + assertEquals(0, histogram.getBuckets().get(7).getDocCount()); + assertEquals(30d, histogram.getBuckets().get(8).getKey()); + assertEquals(1, histogram.getBuckets().get(8).getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); + } + } + } + + public void testMinDocCount() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {0, 1.2, 10, 12, 24}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {5.3, 6, 6, 20}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-10, 0.01, 10, 10, 30, 90}))); + + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(FIELD_NAME) + .interval(5) + .minDocCount(2); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, defaultFieldType(FIELD_NAME)); + assertEquals(4, histogram.getBuckets().size()); + assertEquals(0d, histogram.getBuckets().get(0).getKey()); + assertEquals(3, histogram.getBuckets().get(0).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(1).getKey()); + assertEquals(3, histogram.getBuckets().get(1).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(2).getKey()); + assertEquals(4, histogram.getBuckets().get(2).getDocCount()); + assertEquals(20d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); + } + } + } + + public void testRandomOffset() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + // Note, these values are carefully chosen to ensure that no matter what offset we pick, no two can end up in the same bucket + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {3.2, 9.3}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + + final double offset = randomDouble(); + final double interval = 5; + final double expectedOffset = offset % interval; + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(FIELD_NAME) + .interval(interval) + .offset(offset) + .minDocCount(1); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, defaultFieldType(FIELD_NAME)); + + assertEquals(3, histogram.getBuckets().size()); + assertEquals(-10 + expectedOffset, histogram.getBuckets().get(0).getKey()); + assertEquals(1, histogram.getBuckets().get(0).getDocCount()); + + assertEquals(expectedOffset, histogram.getBuckets().get(1).getKey()); + assertEquals(2, histogram.getBuckets().get(1).getDocCount()); + + assertEquals(5 + expectedOffset, histogram.getBuckets().get(2).getKey()); + assertEquals(1, histogram.getBuckets().get(2).getDocCount()); + + assertTrue(AggregationInspectionHelper.hasValue(histogram)); + } + } + } + + public void testExtendedBounds() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(FIELD_NAME) + .interval(5) + .extendedBounds(-12, 13); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalHistogram histogram = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, defaultFieldType(FIELD_NAME)); + assertEquals(6, histogram.getBuckets().size()); + assertEquals(-15d, histogram.getBuckets().get(0).getKey()); + assertEquals(0, histogram.getBuckets().get(0).getDocCount()); + assertEquals(-10d, histogram.getBuckets().get(1).getKey()); + assertEquals(0, histogram.getBuckets().get(1).getDocCount()); + assertEquals(-5d, histogram.getBuckets().get(2).getKey()); + assertEquals(2, histogram.getBuckets().get(2).getDocCount()); + assertEquals(0d, histogram.getBuckets().get(3).getKey()); + assertEquals(2, histogram.getBuckets().get(3).getDocCount()); + assertEquals(5d, histogram.getBuckets().get(4).getKey()); + assertEquals(0, histogram.getBuckets().get(4).getDocCount()); + assertEquals(10d, histogram.getBuckets().get(5).getKey()); + assertEquals(0, histogram.getBuckets().get(5).getDocCount()); + assertTrue(AggregationInspectionHelper.hasValue(histogram)); + } + } + } + + /** + * Test that sub-aggregations are not supported + */ + public void testSubAggs() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-4.5, 4.3}))); + w.addDocument(singleton(getDocValue(FIELD_NAME, new double[] {-5, 3.2 }))); + + HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("my_agg") + .field(FIELD_NAME) + .interval(5) + .extendedBounds(-12, 13) + .subAggregation(new TopHitsAggregationBuilder("top_hits")); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, defaultFieldType(FIELD_NAME)) + ); + + assertEquals("Histogram aggregation on histogram fields does not support sub-aggregations", e.getMessage()); + } + } + } + + private BinaryDocValuesField getDocValue(String fieldName, double[] values) throws IOException { + TDigest histogram = new TDigestState(100.0); //default + for (double value : values) { + histogram.add(value); + } + BytesStreamOutput streamOutput = new BytesStreamOutput(); + histogram.compress(); + Collection centroids = histogram.centroids(); + Iterator iterator = centroids.iterator(); + while ( iterator.hasNext()) { + Centroid centroid = iterator.next(); + streamOutput.writeVInt(centroid.count()); + streamOutput.writeDouble(centroid.mean()); + } + return new BinaryDocValuesField(fieldName, streamOutput.bytes().toBytesRef()); + } + + @Override + protected List getSearchPlugins() { + return List.of(new AnalyticsPlugin()); + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new HistogramAggregationBuilder("_name").field(fieldName); + } + + private MappedFieldType defaultFieldType(String fieldName) { + return new HistogramFieldMapper.HistogramFieldType(fieldName, true, Collections.emptyMap()); + } + +} diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldTypeTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldTypeTests.java deleted file mode 100644 index 78acdb9fa7c73..0000000000000 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldTypeTests.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - -package org.elasticsearch.xpack.analytics.mapper; - -import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; - -import java.util.Map; - -public class HistogramFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new HistogramFieldMapper.HistogramFieldType(name, true, meta); - } -} diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 6bc6e0e24e9aa..570281a43ac76 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -51,7 +51,6 @@ protected boolean reuseClusters() { return false; } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/58534") public void testFailOverOnFollower() throws Exception { final String leaderIndex = "leader_test_failover"; final String followerIndex = "follower_test_failover"; diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 86b41e59daeb8..e6d7ff6627b9a 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -239,6 +239,7 @@ public void testFollowIndex() throws Exception { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/59273") public void testFollowIndexWithConcurrentMappingChanges() throws Exception { final int numberOfPrimaryShards = randomIntBetween(1, 3); final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, between(0, 1)); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 8b55572d19b8b..08e309cf443c9 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.ccr; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -117,12 +117,12 @@ public void testWriteLimitsIncremented() throws Exception { final PutFollowAction.Request followRequest = getPutFollowRequest("leader", "follower"); client().execute(PutFollowAction.INSTANCE, followRequest).get(); - WriteMemoryLimits memoryLimits = getInstanceFromNode(WriteMemoryLimits.class); + IndexingPressure memoryLimits = getInstanceFromNode(IndexingPressure.class); final long finalSourceSize = sourceSize; assertBusy(() -> { // The actual write bytes will be greater due to other request fields. However, this test is // just spot checking that the bytes are incremented at all. - assertTrue(memoryLimits.getWriteBytes() > finalSourceSize); + assertTrue(memoryLimits.getCurrentCoordinatingAndPrimaryBytes() > finalSourceSize); }); blocker.countDown(); assertBusy(() -> { diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java new file mode 100644 index 0000000000000..b773b43f54feb --- /dev/null +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/PrimaryFollowerAllocationIT.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ccr; + +import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanation; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationDecision; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationResult; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.NodeRoles; +import org.elasticsearch.xpack.CcrIntegTestCase; +import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.in; + +public class PrimaryFollowerAllocationIT extends CcrIntegTestCase { + + @Override + protected boolean reuseClusters() { + return false; + } + + public void testDoNotAllocateFollowerPrimaryToNodesWithoutRemoteClusterClientRole() throws Exception { + final String leaderIndex = "leader-not-allow-index"; + final String followerIndex = "follower-not-allow-index"; + final List dataOnlyNodes = getFollowerCluster().startNodes(between(1, 2), + NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE))); + assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex) + .setSource(getIndexSettings(between(1, 2), between(0, 1)), XContentType.JSON)); + final PutFollowAction.Request putFollowRequest = putFollow(leaderIndex, followerIndex); + putFollowRequest.setSettings(Settings.builder() + .put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes)) + .build()); + putFollowRequest.waitForActiveShards(ActiveShardCount.ONE); + putFollowRequest.timeout(TimeValue.timeValueSeconds(2)); + final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get(); + assertFalse(response.isFollowIndexShardsAcked()); + assertFalse(response.isIndexFollowingStarted()); + final ClusterAllocationExplanation explanation = followerClient().admin().cluster().prepareAllocationExplain() + .setIndex(followerIndex).setShard(0).setPrimary(true).get().getExplanation(); + for (NodeAllocationResult nodeDecision : explanation.getShardAllocationDecision().getAllocateDecision().getNodeDecisions()) { + assertThat(nodeDecision.getNodeDecision(), equalTo(AllocationDecision.NO)); + if (dataOnlyNodes.contains(nodeDecision.getNode().getName())) { + final List decisions = nodeDecision.getCanAllocateDecision().getDecisions() + .stream().map(Object::toString).collect(Collectors.toList()); + assertThat("NO(shard is a primary follower and being bootstrapped, but node does not have the remote_cluster_client role)", + in(decisions)); + } + } + } + + public void testAllocateFollowerPrimaryToNodesWithRemoteClusterClientRole() throws Exception { + final String leaderIndex = "leader-allow-index"; + final String followerIndex = "follower-allow-index"; + final List dataOnlyNodes = getFollowerCluster().startNodes(between(2, 3), + NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE))); + final List dataAndRemoteNodes = getFollowerCluster().startNodes(between(1, 2), + NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE))); + assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex) + .setSource(getIndexSettings(between(1, 2), between(0, 1)), XContentType.JSON)); + final PutFollowAction.Request putFollowRequest = putFollow(leaderIndex, followerIndex); + putFollowRequest.setSettings(Settings.builder() + .put("index.routing.rebalance.enable", "none") + .put("index.routing.allocation.include._name", + Stream.concat(dataOnlyNodes.stream(), dataAndRemoteNodes.stream()).collect(Collectors.joining(","))) + .build()); + final PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, putFollowRequest).get(); + assertTrue(response.isFollowIndexShardsAcked()); + assertTrue(response.isIndexFollowingStarted()); + ensureFollowerGreen(followerIndex); + int numDocs = between(0, 20); + for (int i = 0; i < numDocs; i++) { + leaderClient().prepareIndex(leaderIndex).setSource("f", i).get(); + } + // Empty follower primaries must be assigned to nodes with the remote cluster client role + assertBusy(() -> { + final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(followerIndex)) { + final ShardRouting primaryShard = shardRoutingTable.primaryShard(); + assertTrue(primaryShard.assignedToNode()); + final DiscoveryNode assignedNode = state.nodes().get(primaryShard.currentNodeId()); + assertThat(assignedNode.getName(), in(dataAndRemoteNodes)); + } + }); + // Follower primaries can be relocated to nodes without the remote cluster client role + followerClient().admin().indices().prepareUpdateSettings(followerIndex) + .setSettings(Settings.builder().put("index.routing.allocation.include._name", String.join(",", dataOnlyNodes))) + .get(); + assertBusy(() -> { + final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(followerIndex)) { + for (ShardRouting shard : shardRoutingTable) { + assertNotNull(shard.currentNodeId()); + final DiscoveryNode assignedNode = state.nodes().get(shard.currentNodeId()); + assertThat(assignedNode.getName(), in(dataOnlyNodes)); + } + } + }); + assertIndexFullyReplicatedToFollower(leaderIndex, followerIndex); + // Follower primaries can be recovered from the existing copies on nodes without the remote cluster client role + getFollowerCluster().fullRestart(); + ensureFollowerGreen(followerIndex); + assertBusy(() -> { + final ClusterState state = getFollowerCluster().client().admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(followerIndex)) { + for (ShardRouting shard : shardRoutingTable) { + assertNotNull(shard.currentNodeId()); + final DiscoveryNode assignedNode = state.nodes().get(shard.currentNodeId()); + assertThat(assignedNode.getName(), in(dataOnlyNodes)); + } + } + }); + int moreDocs = between(0, 20); + for (int i = 0; i < moreDocs; i++) { + leaderClient().prepareIndex(leaderIndex).setSource("f", i).get(); + } + assertIndexFullyReplicatedToFollower(leaderIndex, followerIndex); + } +} diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index 359bf42339453..0b0fcaa315021 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -36,6 +37,7 @@ import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.ClusterPlugin; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.PersistentTaskPlugin; import org.elasticsearch.plugins.Plugin; @@ -75,6 +77,7 @@ import org.elasticsearch.xpack.ccr.action.repositories.GetCcrRestoreFileChunkAction; import org.elasticsearch.xpack.ccr.action.repositories.PutCcrRestoreSessionAction; import org.elasticsearch.xpack.ccr.action.repositories.PutInternalCcrRepositoryAction; +import org.elasticsearch.xpack.ccr.allocation.CcrPrimaryFollowerAllocationDecider; import org.elasticsearch.xpack.ccr.index.engine.FollowingEngineFactory; import org.elasticsearch.xpack.ccr.repository.CcrRepository; import org.elasticsearch.xpack.ccr.repository.CcrRestoreSourceService; @@ -125,7 +128,7 @@ /** * Container class for CCR functionality. */ -public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, EnginePlugin, RepositoryPlugin { +public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, EnginePlugin, RepositoryPlugin, ClusterPlugin { public static final String CCR_THREAD_POOL_NAME = "ccr"; public static final String CCR_CUSTOM_METADATA_KEY = "ccr"; @@ -370,4 +373,9 @@ public Collection> mapping public Collection> indicesAliasesRequestValidators() { return Collections.singletonList(CcrRequests.CCR_INDICES_ALIASES_REQUEST_VALIDATOR); } + + @Override + public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { + return List.of(new CcrPrimaryFollowerAllocationDecider()); + } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java index 4f75318b50b2d..9da48580702fa 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/bulk/TransportBulkShardOperationsAction.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.bulk.WriteMemoryLimits; +import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.TransportWriteAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; @@ -38,7 +38,7 @@ public class TransportBulkShardOperationsAction extends TransportWriteAction { - private final WriteMemoryLimits writeMemoryLimits; + private final IndexingPressure indexingPressure; @Inject public TransportBulkShardOperationsAction( @@ -49,7 +49,7 @@ public TransportBulkShardOperationsAction( final ThreadPool threadPool, final ShardStateAction shardStateAction, final ActionFilters actionFilters, - final WriteMemoryLimits writeMemoryLimits) { + final IndexingPressure indexingPressure) { super( settings, BulkShardOperationsAction.NAME, @@ -61,14 +61,14 @@ public TransportBulkShardOperationsAction( actionFilters, BulkShardOperationsRequest::new, BulkShardOperationsRequest::new, - ThreadPool.Names.WRITE, false, writeMemoryLimits); - this.writeMemoryLimits = writeMemoryLimits; + ThreadPool.Names.WRITE, false, indexingPressure); + this.indexingPressure = indexingPressure; } @Override protected void doExecute(Task task, BulkShardOperationsRequest request, ActionListener listener) { // This is executed on the follower coordinator node and we need to mark the bytes. - Releasable releasable = writeMemoryLimits.markWriteOperationStarted(primaryOperationSize(request)); + Releasable releasable = indexingPressure.markIndexingOperationStarted(primaryOperationSize(request)); ActionListener releasingListener = ActionListener.runBefore(listener, releasable::close); try { super.doExecute(task, request, releasingListener); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDecider.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDecider.java new file mode 100644 index 0000000000000..3aa2121245990 --- /dev/null +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDecider.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.ccr.allocation; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.xpack.ccr.CcrSettings; + +/** + * An allocation decider that ensures primary shards of follower indices that are being bootstrapped are assigned to nodes that have the + * remote cluster client role. This is necessary as those nodes reach out to the leader shards on the remote cluster to copy Lucene segment + * files and periodically renew retention leases during the bootstrap. + */ +public final class CcrPrimaryFollowerAllocationDecider extends AllocationDecider { + static final String NAME = "ccr_primary_follower"; + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + final IndexMetadata indexMetadata = allocation.metadata().index(shardRouting.index()); + if (CcrSettings.CCR_FOLLOWING_INDEX_SETTING.get(indexMetadata.getSettings()) == false) { + return allocation.decision(Decision.YES, NAME, "shard is not a follower and is not under the purview of this decider"); + } + if (shardRouting.primary() == false) { + return allocation.decision(Decision.YES, NAME, "shard is a replica follower and is not under the purview of this decider"); + } + final RecoverySource recoverySource = shardRouting.recoverySource(); + if (recoverySource == null || recoverySource.getType() != RecoverySource.Type.SNAPSHOT) { + return allocation.decision(Decision.YES, NAME, + "shard is a primary follower but was bootstrapped already; hence is not under the purview of this decider"); + } + if (node.node().isRemoteClusterClient() == false) { + return allocation.decision(Decision.NO, NAME, "shard is a primary follower and being bootstrapped, but node does not have the " + + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"); + } + return allocation.decision(Decision.YES, NAME, + "shard is a primary follower and node has the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"); + } +} diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index e7d7d71091faf..611d11f1e2e08 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -266,7 +266,7 @@ public void finalizeSnapshot(ShardGenerations shardGenerations, long repositoryS @Override public void deleteSnapshots(Collection snapshotIds, long repositoryStateId, Version repositoryMetaVersion, - ActionListener listener) { + ActionListener listener) { throw new UnsupportedOperationException("Unsupported for repository of type: " + TYPE); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java index 778ec9489641d..f943f083d3861 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardChangesActionTests.java @@ -143,8 +143,8 @@ public void testGetOperationsExceedByteLimit() throws Exception { final IndexShard indexShard = indexService.getShard(0); final Translog.Operation[] operations = ShardChangesAction.getOperations(indexShard, indexShard.getLastKnownGlobalCheckpoint(), - 0, 12, indexShard.getHistoryUUID(), new ByteSizeValue(256, ByteSizeUnit.BYTES)); - assertThat(operations.length, equalTo(12)); + 0, randomIntBetween(100, 500), indexShard.getHistoryUUID(), new ByteSizeValue(256, ByteSizeUnit.BYTES)); + assertThat(operations.length, equalTo(8)); assertThat(operations[0].seqNo(), equalTo(0L)); assertThat(operations[1].seqNo(), equalTo(1L)); assertThat(operations[2].seqNo(), equalTo(2L)); @@ -153,10 +153,6 @@ public void testGetOperationsExceedByteLimit() throws Exception { assertThat(operations[5].seqNo(), equalTo(5L)); assertThat(operations[6].seqNo(), equalTo(6L)); assertThat(operations[7].seqNo(), equalTo(7L)); - assertThat(operations[8].seqNo(), equalTo(8L)); - assertThat(operations[9].seqNo(), equalTo(9L)); - assertThat(operations[10].seqNo(), equalTo(10L)); - assertThat(operations[11].seqNo(), equalTo(11L)); } public void testGetOperationsAlwaysReturnAtLeastOneOp() throws Exception { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java new file mode 100644 index 0000000000000..92cf1f4516885 --- /dev/null +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/allocation/CcrPrimaryFollowerAllocationDeciderTests.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.ccr.allocation; + +import com.carrotsearch.hppc.IntHashSet; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterInfo; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.xpack.ccr.CcrSettings; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import static org.hamcrest.Matchers.equalTo; + +public class CcrPrimaryFollowerAllocationDeciderTests extends ESAllocationTestCase { + + public void testRegularIndex() { + String index = "test-index"; + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(index).settings(settings(Version.CURRENT)) + .numberOfShards(1).numberOfReplicas(1); + List nodes = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + final Set roles = new HashSet<>(); + roles.add(DiscoveryNodeRole.DATA_ROLE); + if (randomBoolean()) { + roles.add(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE); + } + nodes.add(newNode("node" + i, roles)); + } + DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder(); + nodes.forEach(discoveryNodes::add); + Metadata metadata = Metadata.builder().put(indexMetadata).build(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + if (randomBoolean()) { + routingTable.addAsNew(metadata.index(index)); + } else if (randomBoolean()) { + routingTable.addAsRecovery(metadata.index(index)); + } else if (randomBoolean()) { + routingTable.addAsNewRestore(metadata.index(index), newSnapshotRecoverySource(), new IntHashSet()); + } else { + routingTable.addAsRestore(metadata.index(index), newSnapshotRecoverySource()); + } + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .nodes(DiscoveryNodes.EMPTY_NODES).metadata(metadata).routingTable(routingTable.build()).build(); + for (int i = 0; i < clusterState.routingTable().index(index).shards().size(); i++) { + IndexShardRoutingTable shardRouting = clusterState.routingTable().index(index).shard(i); + assertThat(shardRouting.size(), equalTo(2)); + assertThat(shardRouting.primaryShard().state(), equalTo(UNASSIGNED)); + Decision decision = executeAllocation(clusterState, shardRouting.primaryShard(), randomFrom(nodes)); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + assertThat(decision.getExplanation(), equalTo("shard is not a follower and is not under the purview of this decider")); + for (ShardRouting replica : shardRouting.replicaShards()) { + assertThat(replica.state(), equalTo(UNASSIGNED)); + decision = executeAllocation(clusterState, replica, randomFrom(nodes)); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + assertThat(decision.getExplanation(), equalTo("shard is not a follower and is not under the purview of this decider")); + } + } + } + + public void testAlreadyBootstrappedFollowerIndex() { + String index = "test-index"; + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(index) + .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) + .numberOfShards(1).numberOfReplicas(1); + List nodes = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + final Set roles = new HashSet<>(); + roles.add(DiscoveryNodeRole.DATA_ROLE); + if (randomBoolean()) { + roles.add(DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE); + } + nodes.add(newNode("node" + i, roles)); + } + DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder(); + nodes.forEach(discoveryNodes::add); + Metadata metadata = Metadata.builder().put(indexMetadata).build(); + RoutingTable.Builder routingTable = RoutingTable.builder().addAsRecovery(metadata.index(index)); + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .nodes(discoveryNodes).metadata(metadata).routingTable(routingTable.build()).build(); + for (int i = 0; i < clusterState.routingTable().index(index).shards().size(); i++) { + IndexShardRoutingTable shardRouting = clusterState.routingTable().index(index).shard(i); + assertThat(shardRouting.size(), equalTo(2)); + assertThat(shardRouting.primaryShard().state(), equalTo(UNASSIGNED)); + Decision decision = executeAllocation(clusterState, shardRouting.primaryShard(), randomFrom(nodes)); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + assertThat(decision.getExplanation(), + equalTo("shard is a primary follower but was bootstrapped already; hence is not under the purview of this decider")); + for (ShardRouting replica : shardRouting.replicaShards()) { + assertThat(replica.state(), equalTo(UNASSIGNED)); + decision = executeAllocation(clusterState, replica, randomFrom(nodes)); + assertThat(decision.type(), equalTo(Decision.Type.YES)); + assertThat(decision.getExplanation(), equalTo("shard is a replica follower and is not under the purview of this decider")); + } + } + } + + public void testBootstrappingFollowerIndex() { + String index = "test-index"; + IndexMetadata.Builder indexMetadata = IndexMetadata.builder(index) + .settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true)) + .numberOfShards(1).numberOfReplicas(1); + DiscoveryNode dataOnlyNode = newNode("d1", Set.of(DiscoveryNodeRole.DATA_ROLE)); + DiscoveryNode dataAndRemoteNode = newNode("dr1", Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)); + DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(dataOnlyNode).add(dataAndRemoteNode).build(); + Metadata metadata = Metadata.builder().put(indexMetadata).build(); + RoutingTable.Builder routingTable = RoutingTable.builder() + .addAsNewRestore(metadata.index(index), newSnapshotRecoverySource(), new IntHashSet()); + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .nodes(discoveryNodes).metadata(metadata).routingTable(routingTable.build()).build(); + for (int i = 0; i < clusterState.routingTable().index(index).shards().size(); i++) { + IndexShardRoutingTable shardRouting = clusterState.routingTable().index(index).shard(i); + assertThat(shardRouting.size(), equalTo(2)); + assertThat(shardRouting.primaryShard().state(), equalTo(UNASSIGNED)); + Decision noDecision = executeAllocation(clusterState, shardRouting.primaryShard(), dataOnlyNode); + assertThat(noDecision.type(), equalTo(Decision.Type.NO)); + assertThat(noDecision.getExplanation(), + equalTo("shard is a primary follower and being bootstrapped, but node does not have the remote_cluster_client role")); + Decision yesDecision = executeAllocation(clusterState, shardRouting.primaryShard(), dataAndRemoteNode); + assertThat(yesDecision.type(), equalTo(Decision.Type.YES)); + assertThat(yesDecision.getExplanation(), equalTo("shard is a primary follower and node has the remote_cluster_client role")); + for (ShardRouting replica : shardRouting.replicaShards()) { + assertThat(replica.state(), equalTo(UNASSIGNED)); + yesDecision = executeAllocation(clusterState, replica, randomFrom(dataOnlyNode, dataAndRemoteNode)); + assertThat(yesDecision.type(), equalTo(Decision.Type.YES)); + assertThat(yesDecision.getExplanation(), + equalTo("shard is a replica follower and is not under the purview of this decider")); + } + } + } + + static Decision executeAllocation(ClusterState clusterState, ShardRouting shardRouting, DiscoveryNode node) { + final AllocationDecider decider = new CcrPrimaryFollowerAllocationDecider(); + final RoutingAllocation routingAllocation = new RoutingAllocation(new AllocationDeciders(List.of(decider)), + new RoutingNodes(clusterState), clusterState, ClusterInfo.EMPTY, System.nanoTime()); + routingAllocation.debugDecision(true); + return decider.canAllocate(shardRouting, new RoutingNode(node.getId(), node), routingAllocation); + } + + static RecoverySource.SnapshotRecoverySource newSnapshotRecoverySource() { + Snapshot snapshot = new Snapshot("repo", new SnapshotId("name", "_uuid")); + return new RecoverySource.SnapshotRecoverySource(UUIDs.randomBase64UUID(), snapshot, Version.CURRENT, + new IndexId("test", UUIDs.randomBase64UUID(random()))); + } +} diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index c37faf84afbea..2b9b29bc88112 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -17,8 +17,11 @@ appender.audit_rolling.layout.pattern = {\ %varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\ %varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\ %varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\ + %varsNotEmpty{, "apikey.id":"%enc{%map{apikey.id}}{JSON}"}\ + %varsNotEmpty{, "apikey.name":"%enc{%map{apikey.name}}{JSON}"}\ %varsNotEmpty{, "user.roles":%map{user.roles}}\ %varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\ + %varsNotEmpty{, "authentication.type":"%enc{%map{authentication.type}}{JSON}"}\ %varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\ %varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\ %varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a2b86f4636273..7f599d3bc89f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -34,7 +34,9 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction; import org.elasticsearch.xpack.core.action.XPackUsageAction; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; +import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; +import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; import org.elasticsearch.xpack.core.enrich.EnrichFeatureSetUsage; import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; @@ -125,8 +127,8 @@ import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateCalendarJobAction; -import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateDataFrameAnalyticsAction; +import org.elasticsearch.xpack.core.ml.action.UpdateDatafeedAction; import org.elasticsearch.xpack.core.ml.action.UpdateFilterAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; import org.elasticsearch.xpack.core.ml.action.UpdateModelSnapshotAction; @@ -148,7 +150,6 @@ import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; -import org.elasticsearch.xpack.core.async.DeleteAsyncResultAction; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotFeatureSetUsage; @@ -505,7 +506,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.ENRICH, EnrichFeatureSetUsage::new), // Searchable snapshots new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SEARCHABLE_SNAPSHOTS, - SearchableSnapshotFeatureSetUsage::new) + SearchableSnapshotFeatureSetUsage::new), + // Data Streams + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_STREAMS, DataStreamFeatureSetUsage::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 4a039c0eb9c94..f230f28c2a26c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -61,6 +61,8 @@ public final class XPackField { public static final String ENRICH = "enrich"; /** Name constant for the searchable snapshots feature. */ public static final String SEARCHABLE_SNAPSHOTS = "searchable_snapshots"; + /** Name constant for the data streams feature. */ + public static final String DATA_STREAMS = "data_streams"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 1dc698c72476a..389ace65c3174 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -7,8 +7,11 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -42,11 +45,20 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction ANALYTICS = new XPackInfoFeatureAction(XPackField.ANALYTICS); public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH); public static final XPackInfoFeatureAction SEARCHABLE_SNAPSHOTS = new XPackInfoFeatureAction(XPackField.SEARCHABLE_SNAPSHOTS); + public static final XPackInfoFeatureAction DATA_STREAMS = new XPackInfoFeatureAction(XPackField.DATA_STREAMS); - public static final List ALL = Arrays.asList( - SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH, SEARCHABLE_SNAPSHOTS - ); + public static final List ALL; + static { + final List actions = new ArrayList<>(); + actions.addAll(Arrays.asList( + SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, + TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH, DATA_STREAMS + )); + if (SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { + actions.add(SEARCHABLE_SNAPSHOTS); + } + ALL = Collections.unmodifiableList(actions); + } private XPackInfoFeatureAction(String name) { super(BASE_NAME + name, XPackInfoFeatureResponse::new); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index a3a99a0243917..24c263e2cfbc3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -7,8 +7,11 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -42,11 +45,20 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( - SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, - TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, SEARCHABLE_SNAPSHOTS - ); + public static final List ALL; + static { + final List actions = new ArrayList<>(); + actions.addAll(Arrays.asList( + SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, + TRANSFORM, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, DATA_STREAMS + )); + if (SearchableSnapshotsConstants.SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { + actions.add(SEARCHABLE_SNAPSHOTS); + } + ALL = Collections.unmodifiableList(actions); + } private XPackUsageFeatureAction(String name) { super(BASE_NAME + name, XPackUsageFeatureResponse::new); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java new file mode 100644 index 0000000000000..540a8e3a2160a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/DataStreamFeatureSetUsage.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.datastreams; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Objects; + +public class DataStreamFeatureSetUsage extends XPackFeatureSet.Usage { + private final DataStreamStats streamStats; + + public DataStreamFeatureSetUsage(StreamInput input) throws IOException { + super(input); + this.streamStats = new DataStreamStats(input); + } + + public DataStreamFeatureSetUsage(DataStreamStats stats) { + super(XPackField.DATA_STREAMS, true, true); + this.streamStats = stats; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + streamStats.writeTo(out); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_9_0; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("data_streams", streamStats.totalDataStreamCount); + builder.field("indices_count", streamStats.indicesBehindDataStream); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public int hashCode() { + return streamStats.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + DataStreamFeatureSetUsage other = (DataStreamFeatureSetUsage) obj; + return Objects.equals(streamStats, other.streamStats); + } + + public static class DataStreamStats implements Writeable { + + private final long totalDataStreamCount; + private final long indicesBehindDataStream; + + public DataStreamStats(long totalDataStreamCount, long indicesBehindDataStream) { + this.totalDataStreamCount = totalDataStreamCount; + this.indicesBehindDataStream = indicesBehindDataStream; + } + + public DataStreamStats(StreamInput in) throws IOException { + this.totalDataStreamCount = in.readVLong(); + this.indicesBehindDataStream = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(this.totalDataStreamCount); + out.writeVLong(this.indicesBehindDataStream); + } + + @Override + public int hashCode() { + return Objects.hash(totalDataStreamCount, indicesBehindDataStream); + } + + @Override + public boolean equals(Object obj) { + if (obj.getClass() != getClass()) { + return false; + } + DataStreamStats other = (DataStreamStats) obj; + return totalDataStreamCount == other.totalDataStreamCount && + indicesBehindDataStream == other.indicesBehindDataStream; + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 98b5bcf7d1fb0..a5f13e11d966f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -124,7 +124,7 @@ private static DataFrameAnalysis parseAnalysis(XContentParser parser, boolean ig private final Instant createTime; private final Version version; private final boolean allowLazyStart; - private final Integer maxNumThreads; + private final int maxNumThreads; private DataFrameAnalyticsConfig(String id, String description, DataFrameAnalyticsSource source, DataFrameAnalyticsDest dest, DataFrameAnalysis analysis, Map headers, ByteSizeValue modelMemoryLimit, @@ -141,7 +141,11 @@ private DataFrameAnalyticsConfig(String id, String description, DataFrameAnalyti this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.version = version; this.allowLazyStart = allowLazyStart; - this.maxNumThreads = maxNumThreads; + + if (maxNumThreads != null && maxNumThreads < 1) { + throw ExceptionsHelper.badRequestException("[{}] must be a positive integer", MAX_NUM_THREADS.getPreferredName()); + } + this.maxNumThreads = maxNumThreads == null ? 1 : maxNumThreads; } public DataFrameAnalyticsConfig(StreamInput in) throws IOException { @@ -169,11 +173,7 @@ public DataFrameAnalyticsConfig(StreamInput in) throws IOException { } else { allowLazyStart = false; } - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - maxNumThreads = in.readOptionalVInt(); - } else { - maxNumThreads = null; - } + maxNumThreads = in.readVInt(); } public String getId() { @@ -256,9 +256,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(VERSION.getPreferredName(), version); } builder.field(ALLOW_LAZY_START.getPreferredName(), allowLazyStart); - if (maxNumThreads != null) { - builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); - } + builder.field(MAX_NUM_THREADS.getPreferredName(), maxNumThreads); builder.endObject(); return builder; } @@ -287,9 +285,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(allowLazyStart); } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - out.writeOptionalVInt(maxNumThreads); - } + out.writeVInt(maxNumThreads); } @Override @@ -309,7 +305,7 @@ public boolean equals(Object o) { && Objects.equals(createTime, other.createTime) && Objects.equals(version, other.version) && Objects.equals(allowLazyStart, other.allowLazyStart) - && Objects.equals(maxNumThreads, other.maxNumThreads); + && maxNumThreads == other.maxNumThreads; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdate.java index c2fea3932e368..dd92b76faf8e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdate.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; import java.util.Objects; @@ -33,22 +34,30 @@ public class DataFrameAnalyticsConfigUpdate implements Writeable, ToXContentObje DataFrameAnalyticsConfig.MODEL_MEMORY_LIMIT, VALUE); PARSER.declareBoolean(Builder::setAllowLazyStart, DataFrameAnalyticsConfig.ALLOW_LAZY_START); - + PARSER.declareInt(Builder::setMaxNumThreads, DataFrameAnalyticsConfig.MAX_NUM_THREADS); } private final String id; private final String description; private final ByteSizeValue modelMemoryLimit; private final Boolean allowLazyStart; + private final Integer maxNumThreads; private DataFrameAnalyticsConfigUpdate(String id, @Nullable String description, @Nullable ByteSizeValue modelMemoryLimit, - @Nullable Boolean allowLazyStart) { + @Nullable Boolean allowLazyStart, + @Nullable Integer maxNumThreads) { this.id = id; this.description = description; this.modelMemoryLimit = modelMemoryLimit; this.allowLazyStart = allowLazyStart; + + if (maxNumThreads != null && maxNumThreads < 1) { + throw ExceptionsHelper.badRequestException("[{}] must be a positive integer", + DataFrameAnalyticsConfig.MAX_NUM_THREADS.getPreferredName()); + } + this.maxNumThreads = maxNumThreads; } public DataFrameAnalyticsConfigUpdate(StreamInput in) throws IOException { @@ -56,6 +65,7 @@ public DataFrameAnalyticsConfigUpdate(StreamInput in) throws IOException { this.description = in.readOptionalString(); this.modelMemoryLimit = in.readOptionalWriteable(ByteSizeValue::new); this.allowLazyStart = in.readOptionalBoolean(); + this.maxNumThreads = in.readOptionalVInt(); } @Override @@ -64,6 +74,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(description); out.writeOptionalWriteable(modelMemoryLimit); out.writeOptionalBoolean(allowLazyStart); + out.writeOptionalVInt(maxNumThreads); } public String getId() { @@ -82,6 +93,10 @@ public Boolean isAllowLazyStart() { return allowLazyStart; } + public Integer getMaxNumThreads() { + return maxNumThreads; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -95,6 +110,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (allowLazyStart != null) { builder.field(DataFrameAnalyticsConfig.ALLOW_LAZY_START.getPreferredName(), allowLazyStart); } + if (maxNumThreads != null) { + builder.field(DataFrameAnalyticsConfig.MAX_NUM_THREADS.getPreferredName(), maxNumThreads); + } builder.endObject(); return builder; } @@ -120,6 +138,9 @@ public DataFrameAnalyticsConfig.Builder mergeWithConfig(DataFrameAnalyticsConfig if (allowLazyStart != null) { builder.setAllowLazyStart(allowLazyStart); } + if (maxNumThreads != null) { + builder.setMaxNumThreads(maxNumThreads); + } return builder; } @@ -127,7 +148,8 @@ public DataFrameAnalyticsConfig.Builder mergeWithConfig(DataFrameAnalyticsConfig * Whether this update applied to the given source config requires analytics task restart. */ public boolean requiresRestart(DataFrameAnalyticsConfig source) { - return getModelMemoryLimit() != null && getModelMemoryLimit().equals(source.getModelMemoryLimit()) == false; + return (getModelMemoryLimit() != null && getModelMemoryLimit().equals(source.getModelMemoryLimit()) == false) + || (getMaxNumThreads() != null && getMaxNumThreads().equals(source.getMaxNumThreads()) == false); } @Override @@ -145,12 +167,13 @@ public boolean equals(Object other) { return Objects.equals(this.id, that.id) && Objects.equals(this.description, that.description) && Objects.equals(this.modelMemoryLimit, that.modelMemoryLimit) - && Objects.equals(this.allowLazyStart, that.allowLazyStart); + && Objects.equals(this.allowLazyStart, that.allowLazyStart) + && Objects.equals(this.maxNumThreads, that.maxNumThreads); } @Override public int hashCode() { - return Objects.hash(id, description, modelMemoryLimit, allowLazyStart); + return Objects.hash(id, description, modelMemoryLimit, allowLazyStart, maxNumThreads); } public static class Builder { @@ -159,6 +182,7 @@ public static class Builder { private String description; private ByteSizeValue modelMemoryLimit; private Boolean allowLazyStart; + private Integer maxNumThreads; public Builder(String id) { this.id = id; @@ -188,8 +212,13 @@ public Builder setAllowLazyStart(Boolean allowLazyStart) { return this; } + public Builder setMaxNumThreads(Integer maxNumThreads) { + this.maxNumThreads = maxNumThreads; + return this; + } + public DataFrameAnalyticsConfigUpdate build() { - return new DataFrameAnalyticsConfigUpdate(id, description, modelMemoryLimit, allowLazyStart); + return new DataFrameAnalyticsConfigUpdate(id, description, modelMemoryLimit, allowLazyStart, maxNumThreads); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java index c214ee2421c49..cff761183d70b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java @@ -368,6 +368,11 @@ public InferenceConfig inferenceConfig(FieldInfo fieldInfo) { .build(); } + @Override + public boolean supportsInference() { + return true; + } + public static String extractJobIdFromStateDoc(String stateDocId) { int suffixIndex = stateDocId.lastIndexOf(STATE_DOC_ID_SUFFIX); return suffixIndex <= 0 ? null : stateDocId.substring(0, suffixIndex); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java index bd90eaf0f73a1..f64028c990959 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/DataFrameAnalysis.java @@ -78,6 +78,11 @@ public interface DataFrameAnalysis extends ToXContentObject, NamedWriteable { @Nullable InferenceConfig inferenceConfig(FieldInfo fieldInfo); + /** + * @return {@code true} if this analysis trains a model that can be used for inference + */ + boolean supportsInference(); + /** * Summarizes information about the fields that is necessary for analysis to generate * the parameters needed for the process configuration. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java index d7a6db6503ee7..03451c23085ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java @@ -257,6 +257,11 @@ public InferenceConfig inferenceConfig(FieldInfo fieldInfo) { return null; } + @Override + public boolean supportsInference() { + return false; + } + public enum Method { LOF, LDOF, DISTANCE_KTH_NN, DISTANCE_KNN; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java index 0994e98f678a4..f92e660ba733e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java @@ -269,6 +269,11 @@ public InferenceConfig inferenceConfig(FieldInfo fieldInfo) { .build(); } + @Override + public boolean supportsInference() { + return true; + } + public static String extractJobIdFromStateDoc(String stateDocId) { int suffixIndex = stateDocId.lastIndexOf(STATE_DOC_ID_SUFFIX); return suffixIndex <= 0 ? null : stateDocId.substring(0, suffixIndex); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java index 6d85cd9957608..a7cad70c5794f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/ClassificationInferenceResults.java @@ -17,6 +17,8 @@ import java.io.IOException; import java.util.Collections; +import java.util.Map; +import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -137,18 +139,20 @@ public Object predictedValue() { public void writeResult(IngestDocument document, String parentResultField) { ExceptionsHelper.requireNonNull(document, "document"); ExceptionsHelper.requireNonNull(parentResultField, "parentResultField"); - document.setFieldValue(parentResultField + "." + this.resultsField, - predictionFieldType.transformPredictedValue(value(), valueAsString())); - if (topClasses.size() > 0) { - document.setFieldValue(parentResultField + "." + topNumClassesField, - topClasses.stream().map(TopClassEntry::asValueMap).collect(Collectors.toList())); + document.setFieldValue(parentResultField, asMap()); + } + + @Override + public Map asMap() { + Map map = new LinkedHashMap<>(); + map.put(resultsField, predictionFieldType.transformPredictedValue(value(), valueAsString())); + if (topClasses.isEmpty() == false) { + map.put(topNumClassesField, topClasses.stream().map(TopClassEntry::asValueMap).collect(Collectors.toList())); } - if (getFeatureImportance().size() > 0) { - document.setFieldValue(parentResultField + "." + FEATURE_IMPORTANCE, getFeatureImportance() - .stream() - .map(FeatureImportance::toMap) - .collect(Collectors.toList())); + if (getFeatureImportance().isEmpty() == false) { + map.put(FEATURE_IMPORTANCE, getFeatureImportance().stream().map(FeatureImportance::toMap).collect(Collectors.toList())); } + return map; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResults.java index e9a4e6f1e3e6b..6b83f44cffb04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceResults.java @@ -9,9 +9,13 @@ import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.ingest.IngestDocument; +import java.util.Map; + public interface InferenceResults extends NamedWriteable, ToXContentFragment { void writeResult(IngestDocument document, String parentResultField); + Map asMap(); + Object predictedValue(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java index acd2038b22d68..1f907e8a2f51e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RawInferenceResults.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Map; import java.util.Objects; public class RawInferenceResults implements InferenceResults { @@ -57,6 +58,10 @@ public void writeResult(IngestDocument document, String parentResultField) { throw new UnsupportedOperationException("[raw] does not support writing inference results"); } + @Override + public Map asMap() { + throw new UnsupportedOperationException("[raw] does not support map conversion"); + } @Override public Object predictedValue() { return null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java index 7b9d067ed4cbc..f082633a88f1d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/RegressionInferenceResults.java @@ -15,7 +15,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -85,13 +87,17 @@ public Object predictedValue() { public void writeResult(IngestDocument document, String parentResultField) { ExceptionsHelper.requireNonNull(document, "document"); ExceptionsHelper.requireNonNull(parentResultField, "parentResultField"); - document.setFieldValue(parentResultField + "." + this.resultsField, value()); - if (getFeatureImportance().size() > 0) { - document.setFieldValue(parentResultField + ".feature_importance", getFeatureImportance() - .stream() - .map(FeatureImportance::toMap) - .collect(Collectors.toList())); + document.setFieldValue(parentResultField, asMap()); + } + + @Override + public Map asMap() { + Map map = new LinkedHashMap<>(); + map.put(resultsField, value()); + if (getFeatureImportance().isEmpty() == false) { + map.put(FEATURE_IMPORTANCE, getFeatureImportance().stream().map(FeatureImportance::toMap).collect(Collectors.toList())); } + return map; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResults.java index ccc0743228136..9bff569bc2e49 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/WarningInferenceResults.java @@ -13,6 +13,8 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Objects; public class WarningInferenceResults implements InferenceResults { @@ -56,7 +58,14 @@ public int hashCode() { public void writeResult(IngestDocument document, String parentResultField) { ExceptionsHelper.requireNonNull(document, "document"); ExceptionsHelper.requireNonNull(parentResultField, "resultField"); - document.setFieldValue(parentResultField + "." + NAME, warning); + document.setFieldValue(parentResultField, asMap()); + } + + @Override + public Map asMap() { + Map asMap = new LinkedHashMap<>(); + asMap.put(NAME, warning); + return asMap; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 6bcf179578fd1..8cbb18e5c063a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -145,8 +145,6 @@ public final class Messages { "Adjust the analysis_limits.model_memory_limit setting to ensure all data is analyzed"; public static final String JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT_PRE_7_2 = "Job memory status changed to hard_limit at {0}; adjust the " + "analysis_limits.model_memory_limit setting to ensure all data is analyzed"; - public static final String JOB_AUDIT_CATEGORIZATION_STATUS_WARN = "categorization_status changed to [{0}] after [{1}] buckets." + - " This suggests an inappropriate categorization_field_name has been chosen."; public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES = "categorization_filters contain duplicates"; public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY = diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java index 7ec7d53049f32..ad7a6aaa03125 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/searchablesnapshots/MountSearchableSnapshotRequest.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +24,7 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -102,7 +104,12 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - return null; + ActionRequestValidationException validationException = null; + if (IndexMetadata.INDEX_DATA_PATH_SETTING.exists(indexSettings)) { + validationException = addValidationError( "setting [" + IndexMetadata.SETTING_DATA_PATH + + "] is not permitted on searchable snapshots", validationException); + } + return validationException; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java index 53704147438a6..0a5e4dc32dabd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java @@ -10,9 +10,12 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.node.Node; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; @@ -23,14 +26,21 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import java.util.function.Consumer; import java.util.function.Function; +import static org.elasticsearch.xpack.core.security.authc.Authentication.VERSION_API_KEY_ROLES_AS_BYTES; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY; + /** * A lightweight utility that can find the current user and authentication information for the local thread. */ public class SecurityContext { + private final Logger logger = LogManager.getLogger(SecurityContext.class); private final ThreadContext threadContext; @@ -149,8 +159,27 @@ public void executeAfterRewritingAuthentication(Consumer consumer final Authentication authentication = getAuthentication(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setAuthentication(new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), version, authentication.getAuthenticationType(), authentication.getMetadata())); + authentication.getLookedUpBy(), version, authentication.getAuthenticationType(), + rewriteMetadataForApiKeyRoleDescriptors(version, authentication))); consumer.accept(original); } } + + private Map rewriteMetadataForApiKeyRoleDescriptors(Version streamVersion, Authentication authentication) { + Map metadata = authentication.getMetadata(); + if (authentication.getAuthenticationType() == AuthenticationType.API_KEY + && authentication.getVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES) + && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) { + metadata = new HashMap<>(metadata); + metadata.put( + API_KEY_ROLE_DESCRIPTORS_KEY, + XContentHelper.convertToMap( + (BytesReference) metadata.get(API_KEY_ROLE_DESCRIPTORS_KEY), false, XContentType.JSON).v2()); + metadata.put( + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + XContentHelper.convertToMap( + (BytesReference) metadata.get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), false, XContentType.JSON).v2()); + } + return metadata; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 4aa850e1aa60d..bef92cb9ab3ce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -27,6 +27,8 @@ // That interface can be removed public class Authentication implements ToXContentObject { + public static final Version VERSION_API_KEY_ROLES_AS_BYTES = Version.V_7_9_0; + private final User user; private final RealmRef authenticatedBy; private final RealmRef lookedUpBy; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java index a53a58d637a96..12fab154b8ef7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationField.java @@ -8,6 +8,8 @@ public final class AuthenticationField { public static final String AUTHENTICATION_KEY = "_xpack_security_authentication"; + public static final String API_KEY_ROLE_DESCRIPTORS_KEY = "_security_api_key_role_descriptors"; + public static final String API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY = "_security_api_key_limited_by_role_descriptors"; private AuthenticationField() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java index a987fcac21bd5..97b3247fb643d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java @@ -201,8 +201,6 @@ public static Set names() { public static boolean isClusterAction(String actionName) { return actionName.startsWith("cluster:") || actionName.startsWith("indices:admin/template/") || - // todo: hack until we implement security of data_streams - actionName.startsWith("indices:admin/data_stream/") || actionName.startsWith("indices:admin/index_template/"); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java index 4f4774b811e8b..e95bda670f23f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/IndexPrivilege.java @@ -13,6 +13,9 @@ import org.elasticsearch.action.admin.indices.close.CloseIndexAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsAction; @@ -60,17 +63,19 @@ public final class IndexPrivilege extends Privilege { private static final Automaton MONITOR_AUTOMATON = patterns("indices:monitor/*"); private static final Automaton MANAGE_AUTOMATON = unionAndMinimize(Arrays.asList(MONITOR_AUTOMATON, patterns("indices:admin/*"))); - private static final Automaton CREATE_INDEX_AUTOMATON = patterns(CreateIndexAction.NAME, AutoCreateAction.NAME); - private static final Automaton DELETE_INDEX_AUTOMATON = patterns(DeleteIndexAction.NAME); - private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, - GetIndexAction.NAME, GetFieldMappingsAction.NAME + "*", GetMappingsAction.NAME, - ClusterSearchShardsAction.NAME, ValidateQueryAction.NAME + "*", GetSettingsAction.NAME, ExplainLifecycleAction.NAME); + private static final Automaton CREATE_INDEX_AUTOMATON = patterns(CreateIndexAction.NAME, AutoCreateAction.NAME, + CreateDataStreamAction.NAME); + private static final Automaton DELETE_INDEX_AUTOMATON = patterns(DeleteIndexAction.NAME, DeleteDataStreamAction.NAME); + private static final Automaton VIEW_METADATA_AUTOMATON = patterns(GetAliasesAction.NAME, GetIndexAction.NAME, + GetFieldMappingsAction.NAME + "*", GetMappingsAction.NAME, ClusterSearchShardsAction.NAME, ValidateQueryAction.NAME + "*", + GetSettingsAction.NAME, ExplainLifecycleAction.NAME, GetDataStreamAction.NAME); private static final Automaton MANAGE_FOLLOW_INDEX_AUTOMATON = patterns(PutFollowAction.NAME, UnfollowAction.NAME, CloseIndexAction.NAME + "*"); private static final Automaton MANAGE_LEADER_INDEX_AUTOMATON = patterns(ForgetFollowerAction.NAME + "*"); private static final Automaton MANAGE_ILM_AUTOMATON = patterns("indices:admin/ilm/*"); private static final Automaton MAINTENANCE_AUTOMATON = patterns("indices:admin/refresh*", "indices:admin/flush*", "indices:admin/synced_flush", "indices:admin/forcemerge*"); + private static final Automaton AUTO_CONFIGURE_AUTOMATON = patterns(AutoPutMappingAction.NAME, AutoCreateAction.NAME); public static final IndexPrivilege NONE = new IndexPrivilege("none", Automatons.EMPTY); public static final IndexPrivilege ALL = new IndexPrivilege("all", ALL_AUTOMATON); @@ -88,8 +93,9 @@ public final class IndexPrivilege extends Privilege { public static final IndexPrivilege VIEW_METADATA = new IndexPrivilege("view_index_metadata", VIEW_METADATA_AUTOMATON); public static final IndexPrivilege MANAGE_FOLLOW_INDEX = new IndexPrivilege("manage_follow_index", MANAGE_FOLLOW_INDEX_AUTOMATON); public static final IndexPrivilege MANAGE_LEADER_INDEX = new IndexPrivilege("manage_leader_index", MANAGE_LEADER_INDEX_AUTOMATON); - public static final IndexPrivilege MANAGE_ILM = new IndexPrivilege("manage_ilm", MANAGE_ILM_AUTOMATON); - public static final IndexPrivilege MAINTENANCE = new IndexPrivilege("maintenance", MAINTENANCE_AUTOMATON); + public static final IndexPrivilege MANAGE_ILM = new IndexPrivilege("manage_ilm", MANAGE_ILM_AUTOMATON); + public static final IndexPrivilege MAINTENANCE = new IndexPrivilege("maintenance", MAINTENANCE_AUTOMATON); + public static final IndexPrivilege AUTO_CONFIGURE = new IndexPrivilege("auto_configure", AUTO_CONFIGURE_AUTOMATON); private static final Map VALUES = Map.ofEntries( entry("none", NONE), @@ -109,7 +115,8 @@ public final class IndexPrivilege extends Privilege { entry("manage_follow_index", MANAGE_FOLLOW_INDEX), entry("manage_leader_index", MANAGE_LEADER_INDEX), entry("manage_ilm", MANAGE_ILM), - entry("maintenance", MAINTENANCE)); + entry("maintenance", MAINTENANCE), + entry("auto_configure", AUTO_CONFIGURE)); public static final Predicate ACTION_MATCHER = ALL.predicate(); public static final Predicate CREATE_INDEX_MATCHER = CREATE_INDEX.predicate(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java index 9bfc18a538249..7178dac8e90ca 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; @@ -465,6 +466,14 @@ public Collection> ind .collect(Collectors.toList()); } + @Override + public Collection createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) { + return filterPlugins(ClusterPlugin.class) + .stream() + .flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream()) + .collect(Collectors.toList()); + } + private List filterPlugins(Class type) { return plugins.stream().filter(x -> type.isAssignableFrom(x.getClass())).map(p -> ((T)p)) .collect(Collectors.toList()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java index 331c378522ff7..8068d03456b29 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; @@ -123,9 +124,6 @@ protected DataFrameAnalyticsConfig mutateInstanceForVersion(DataFrameAnalyticsCo builder.setCreateTime(null); builder.setVersion(null); } - if (version.before(Version.V_8_0_0)) { - builder.setMaxNumThreads(null); - } return builder.build(); } @@ -496,6 +494,32 @@ public void testExtractJobIdFromDocId() { assertThat(DataFrameAnalyticsConfig.extractJobIdFromDocId("foo"), is(nullValue())); } + public void testCtor_GivenMaxNumThreadsIsZero() { + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new DataFrameAnalyticsConfig.Builder() + .setId("test_config") + .setSource(new DataFrameAnalyticsSource(new String[] {"source_index"}, null, null)) + .setDest(new DataFrameAnalyticsDest("dest_index", null)) + .setAnalysis(new Regression("foo")) + .setMaxNumThreads(0) + .build()); + + assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); + assertThat(e.getMessage(), equalTo("[max_num_threads] must be a positive integer")); + } + + public void testCtor_GivenMaxNumThreadsIsNegative() { + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new DataFrameAnalyticsConfig.Builder() + .setId("test_config") + .setSource(new DataFrameAnalyticsSource(new String[] {"source_index"}, null, null)) + .setDest(new DataFrameAnalyticsDest("dest_index", null)) + .setAnalysis(new Regression("foo")) + .setMaxNumThreads(randomIntBetween(Integer.MIN_VALUE, 0)) + .build()); + + assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); + assertThat(e.getMessage(), equalTo("[max_num_threads] must be a positive integer")); + } + private static void assertTooSmall(ElasticsearchStatusException e) { assertThat(e.getMessage(), startsWith("model_memory_limit must be at least 1kb.")); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java index ae6bc9952bbe9..6f974fbdc8fb3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigUpdateTests.java @@ -5,9 +5,11 @@ */ package org.elasticsearch.xpack.core.ml.dataframe; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; @@ -47,6 +49,9 @@ public static DataFrameAnalyticsConfigUpdate randomUpdate(String id) { if (randomBoolean()) { builder.setAllowLazyStart(randomBoolean()); } + if (randomBoolean()) { + builder.setMaxNumThreads(randomIntBetween(1, 20)); + } return builder.build(); } @@ -81,6 +86,15 @@ public void testMergeWithConfig_UpdatedAllowLazyStart() { is(equalTo(new DataFrameAnalyticsConfig.Builder(config).setAllowLazyStart(true).build()))); } + public void testMergeWithConfig_UpdatedMaxNumThreads() { + String id = randomValidId(); + DataFrameAnalyticsConfig config = DataFrameAnalyticsConfigTests.createRandomBuilder(id).setMaxNumThreads(3).build(); + DataFrameAnalyticsConfigUpdate update = new DataFrameAnalyticsConfigUpdate.Builder(id).setMaxNumThreads(5).build(); + assertThat( + update.mergeWithConfig(config).build(), + is(equalTo(new DataFrameAnalyticsConfig.Builder(config).setMaxNumThreads(5).build()))); + } + public void testMergeWithConfig_UpdatedAllUpdatableProperties() { String id = randomValidId(); DataFrameAnalyticsConfig config = @@ -88,12 +102,14 @@ public void testMergeWithConfig_UpdatedAllUpdatableProperties() { .setDescription("old description") .setModelMemoryLimit(new ByteSizeValue(1024)) .setAllowLazyStart(false) + .setMaxNumThreads(1) .build(); DataFrameAnalyticsConfigUpdate update = new DataFrameAnalyticsConfigUpdate.Builder(id) .setDescription("new description") .setModelMemoryLimit(new ByteSizeValue(2048)) .setAllowLazyStart(true) + .setMaxNumThreads(4) .build(); assertThat( update.mergeWithConfig(config).build(), @@ -102,6 +118,7 @@ public void testMergeWithConfig_UpdatedAllUpdatableProperties() { .setDescription("new description") .setModelMemoryLimit(new ByteSizeValue(2048)) .setAllowLazyStart(true) + .setMaxNumThreads(4) .build()))); } @@ -155,9 +172,35 @@ public void testRequiresRestart_ModelMemoryLimitUpdateRequiresRestart() { assertThat(update.requiresRestart(config), is(true)); } + public void testRequiresRestart_MaxNumThreadsUpdateRequiresRestart() { + String id = randomValidId(); + DataFrameAnalyticsConfig config = + DataFrameAnalyticsConfigTests.createRandomBuilder(id).setMaxNumThreads(1).build(); + DataFrameAnalyticsConfigUpdate update = new DataFrameAnalyticsConfigUpdate.Builder(id).setMaxNumThreads(8).build(); + + assertThat(update.requiresRestart(config), is(true)); + } + + public void testCtor_GivenMaxNumberThreadsIsZero() { + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> new DataFrameAnalyticsConfigUpdate.Builder("test").setMaxNumThreads(0).build()); + + assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); + assertThat(e.getMessage(), equalTo("[max_num_threads] must be a positive integer")); + } + + public void testCtor_GivenMaxNumberThreadsIsNegative() { + ElasticsearchException e = expectThrows(ElasticsearchException.class, + () -> new DataFrameAnalyticsConfigUpdate.Builder("test").setMaxNumThreads(randomIntBetween(Integer.MIN_VALUE, 0)).build()); + + assertThat(e.status(), equalTo(RestStatus.BAD_REQUEST)); + assertThat(e.getMessage(), equalTo("[max_num_threads] must be a positive integer")); + } + private boolean isNoop(DataFrameAnalyticsConfig config, DataFrameAnalyticsConfigUpdate update) { return (update.getDescription() == null || Objects.equals(config.getDescription(), update.getDescription())) && (update.getModelMemoryLimit() == null || Objects.equals(config.getModelMemoryLimit(), update.getModelMemoryLimit())) - && (update.isAllowLazyStart() == null || Objects.equals(config.isAllowLazyStart(), update.isAllowLazyStart())); + && (update.isAllowLazyStart() == null || Objects.equals(config.isAllowLazyStart(), update.isAllowLazyStart())) + && (update.getMaxNumThreads() == null || Objects.equals(config.getMaxNumThreads(), update.getMaxNumThreads())); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java new file mode 100644 index 0000000000000..d21b8dd0c7ab0 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/searchablesnapshots/DataStreamFeatureSetUsageTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.searchablesnapshots; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; + +import java.io.IOException; + +public class DataStreamFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected DataStreamFeatureSetUsage createTestInstance() { + return new DataStreamFeatureSetUsage(new DataStreamFeatureSetUsage.DataStreamStats(randomNonNegativeLong(), + randomNonNegativeLong())); + } + + @Override + protected DataStreamFeatureSetUsage mutateInstance(DataStreamFeatureSetUsage instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataStreamFeatureSetUsage::new; + } + +} diff --git a/x-pack/plugin/data-streams/build.gradle b/x-pack/plugin/data-streams/build.gradle new file mode 100644 index 0000000000000..a53044439879e --- /dev/null +++ b/x-pack/plugin/data-streams/build.gradle @@ -0,0 +1,34 @@ +import org.elasticsearch.gradle.info.BuildParams + +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.internal-cluster-test' +esplugin { + name 'x-pack-data-streams' + description 'Elasticsearch Expanded Pack Plugin - Data Streams' + classname 'org.elasticsearch.xpack.datastreams.DataStreamsPlugin' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-data-streams' +integTest.enabled = false + +tasks.named('internalClusterTest').configure { + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.datastreams_feature_enabled', 'true' + } +} + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +// add all sub-projects of the qa sub-project +gradle.projectsEvaluated { + project.subprojects + .find { it.path == project.path + ":qa" } + .subprojects + .findAll { it.path.startsWith(project.path + ":qa") } + .each { check.dependsOn it.check } +} diff --git a/x-pack/plugin/data-streams/qa/build.gradle b/x-pack/plugin/data-streams/qa/build.gradle new file mode 100644 index 0000000000000..75d524cc11500 --- /dev/null +++ b/x-pack/plugin/data-streams/qa/build.gradle @@ -0,0 +1,8 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.build' +test.enabled = false + +dependencies { + api project(':test:framework') +} diff --git a/x-pack/plugin/data-streams/qa/multi-node/build.gradle b/x-pack/plugin/data-streams/qa/multi-node/build.gradle new file mode 100644 index 0000000000000..bf386e76c30ed --- /dev/null +++ b/x-pack/plugin/data-streams/qa/multi-node/build.gradle @@ -0,0 +1,30 @@ +import org.elasticsearch.gradle.info.BuildParams + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts') +} + +File repoDir = file("$buildDir/testclusters/repo") + +integTest.runner { + /* To support taking index snapshots, we have to set path.repo setting */ + systemProperty 'tests.path.repo', repoDir +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.searchable_snapshots_feature_enabled', 'true' + } + numberOfNodes = 4 + + setting 'path.repo', repoDir.absolutePath + setting 'xpack.security.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' +} diff --git a/x-pack/plugin/data-streams/qa/multi-node/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamRestIT.java b/x-pack/plugin/data-streams/qa/multi-node/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamRestIT.java new file mode 100644 index 0000000000000..5c7f9d145d9e6 --- /dev/null +++ b/x-pack/plugin/data-streams/qa/multi-node/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamRestIT.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.equalTo; + +public class DataStreamRestIT extends ESRestTestCase { + + @SuppressWarnings("unchecked") + public void testDSXpackInfo() { + Map features = (Map) getLocation("/_xpack").get("features"); + assertNotNull(features); + Map dataStreams = (Map) features.get("data_streams"); + assertNotNull(dataStreams); + assertTrue((boolean) dataStreams.get("available")); + assertTrue((boolean) dataStreams.get("enabled")); + } + + @SuppressWarnings("unchecked") + public void testDSXpackUsage() throws Exception { + Map dataStreams = (Map) getLocation("/_xpack/usage").get("data_streams"); + assertNotNull(dataStreams); + assertTrue((boolean) dataStreams.get("available")); + assertTrue((boolean) dataStreams.get("enabled")); + assertThat(dataStreams.get("data_streams"), anyOf(equalTo(null), equalTo(0))); + + // Create a data stream + Request indexRequest = new Request("POST", "/logs-mysql-default/_doc"); + indexRequest.setJsonEntity("{\"@timestamp\": \"2020-01-01\"}"); + client().performRequest(indexRequest); + + // Roll over the data stream + Request rollover = new Request("POST", "/logs-mysql-default/_rollover"); + client().performRequest(rollover); + + dataStreams = (Map) getLocation("/_xpack/usage").get("data_streams"); + assertNotNull(dataStreams); + assertTrue((boolean) dataStreams.get("available")); + assertTrue((boolean) dataStreams.get("enabled")); + assertThat("got: " + dataStreams, dataStreams.get("data_streams"), equalTo(1)); + assertThat("got: " + dataStreams, dataStreams.get("indices_count"), equalTo(2)); + } + + public Map getLocation(String path) { + try { + Response executeRepsonse = client().performRequest(new Request("GET", path)); + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + EntityUtils.toByteArray(executeRepsonse.getEntity()) + ) + ) { + return parser.map(); + } + } catch (Exception e) { + fail("failed to execute GET request to " + path + " - got: " + e); + throw new RuntimeException(e); + } + } +} diff --git a/x-pack/plugin/data-streams/qa/rest/build.gradle b/x-pack/plugin/data-streams/qa/rest/build.gradle new file mode 100644 index 0000000000000..70ac9548ae42d --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/build.gradle @@ -0,0 +1,20 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +restResources { + restApi { + includeCore 'bulk', 'count', 'search', '_common', 'indices', 'index', 'cluster', 'rank_eval', 'reindex', 'update_by_query', 'delete_by_query' + includeXpack 'enrich' + } +} + +dependencies { + testImplementation project(path: xpackModule('data-streams')) +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'basic' +} diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java b/x-pack/plugin/data-streams/qa/rest/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java new file mode 100644 index 0000000000000..97703509df013 --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsRestIT.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class DataStreamsRestIT extends ESClientYamlSuiteTestCase { + + public DataStreamsRestIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + +} diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/90_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/90_data_streams.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/100_delete_by_query.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml similarity index 99% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml index 6f9f5f5811229..eb6037878e7e9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/10_basic.yml @@ -157,7 +157,7 @@ setup: catch: missing - match: { status: 404 } - - match: { error.root_cause.0.type: "resource_not_found_exception" } + - match: { error.root_cause.0.type: "index_not_found_exception" } - do: indices.get_data_stream: @@ -282,7 +282,7 @@ setup: --- "Indexing a document into a data stream without a timestamp field": - skip: - version: " - 7.9.99" + version: " - 7.99.99" reason: "enable in 7.9+ when backported" features: allowed_warnings diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/90_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/90_data_streams.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/110_update_by_query.yml diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml new file mode 100644 index 0000000000000..fad3bc72b122e --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/20_unsupported_apis.yml @@ -0,0 +1,232 @@ +--- +"Test apis that do not supported data streams": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [logs-*] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: logs-foobar + - is_true: acknowledged + + - do: + index: + index: logs-foobar + refresh: true + body: + '@timestamp': '2020-12-12' + foo: bar + - match: {_index: .ds-logs-foobar-000001} + + - do: + search: + index: logs-foobar + body: { query: { match_all: {} } } + - length: { hits.hits: 1 } + - match: { hits.hits.0._index: .ds-logs-foobar-000001 } + - match: { hits.hits.0._source.foo: 'bar' } + + - do: + catch: missing + indices.delete: + index: logs-foobar + + - do: + indices.delete_data_stream: + name: logs-foobar + - is_true: acknowledged + +--- +"Prohibit clone on data stream's write index": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: bad_request + indices.clone: + index: ".ds-simple-data-stream1-000001" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 2 + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"APIs temporarily muted": + - skip: + version: "all" + reason: "restore to above test after data stream resolution PRs have been merged" + + - do: + catch: bad_request + indices.close: + index: logs-* + +--- +"Prohibit shrink on data stream's write index": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: bad_request + indices.shrink: + index: ".ds-simple-data-stream1-000001" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Close write index for data stream fails": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: bad_request + indices.close: + index: ".ds-simple-data-stream1-000001" + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Prohibit split on data stream's write index": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: bad_request + indices.split: + index: ".ds-simple-data-stream1-000001" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 4 + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/30_auto_create_data_stream.yml diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml new file mode 100644 index 0000000000000..60e2ca59a3c1a --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/40_supported_apis.yml @@ -0,0 +1,333 @@ +--- +setup: + - skip: + features: allowed_warnings + - do: + allowed_warnings: + - "index template [logs_template] has index patterns [logs-foobar] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logs_template] will take precedence during new index creation" + indices.put_index_template: + name: logs_template + body: + index_patterns: logs-foobar + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: logs-foobar + +--- +teardown: + - do: + indices.delete_data_stream: + name: logs-foobar + +--- +"Verify get index api": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + + - do: + indices.get: + index: logs-foobar + - is_true: \.ds-logs-foobar-000001 + - is_false: logs-foobar + - match: { \.ds-logs-foobar-000001.settings.index.number_of_shards: '1' } + +--- +"Verify get mapping api": + - skip: + version: " - 7.8.99" + reason: "data streams only supported in 7.9+" + + - do: + indices.get_mapping: + index: logs-foobar + - is_true: \.ds-logs-foobar-000001.mappings + - is_false: \.ds-logs-foobar.mappings + +--- +"Verify shard stores api": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + settings: + number_of_shards: "1" + number_of_replicas: "0" + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + # rollover data stream to create new backing index + - do: + indices.rollover: + alias: "simple-data-stream1" + + - match: { old_index: .ds-simple-data-stream1-000001 } + - match: { new_index: .ds-simple-data-stream1-000002 } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + cluster.health: + index: simple-data-stream1 + wait_for_status: green + + - do: + indices.shard_stores: + index: simple-data-stream1 + status: "green" + + - match: { indices.\.ds-simple-data-stream1-000001.shards.0.stores.0.allocation: "primary" } + - match: { indices.\.ds-simple-data-stream1-000002.shards.0.stores.0.allocation: "primary" } + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged +--- +"Verify search shards api": + - skip: + features: allowed_warnings + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + search_shards: + index: "simple-data-stream1" + + - match: { shards.0.0.index: ".ds-simple-data-stream1-000001" } + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Verify get field mappings api": + - skip: + features: allowed_warnings + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + indices.get_field_mapping: + index: simple-data-stream1 + fields: foo + + - is_true: \.ds-simple-data-stream1-000001 + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Open write index for data stream opens all backing indices": + - skip: + version: " - 7.99.99" + reason: "change to - 7.8.99 after backport" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + # rollover data stream twice to create new backing indices + - do: + indices.rollover: + alias: "simple-data-stream1" + + - match: { old_index: .ds-simple-data-stream1-000001 } + - match: { new_index: .ds-simple-data-stream1-000002 } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.rollover: + alias: "simple-data-stream1" + + - match: { old_index: .ds-simple-data-stream1-000002 } + - match: { new_index: .ds-simple-data-stream1-000003 } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.close: + index: ".ds-simple-data-stream1-000001,.ds-simple-data-stream1-000002" + - is_true: acknowledged + + - do: + indices.open: + index: simple-data-stream1 + - is_true: acknowledged + + # all closed backing indices should be re-opened and returned + - do: + indices.get: + index: ".ds-simple-data-stream1-*" + + - is_true: \.ds-simple-data-stream1-000001.settings + - is_true: \.ds-simple-data-stream1-000002.settings + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Verify rank eval with data streams": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + features: allowed_warnings + + - do: + index: + index: logs-foobar + id: doc1 + op_type: create + body: { "text": "berlin", "@timestamp": "2020-01-01" } + + - do: + index: + index: logs-foobar + id: doc2 + op_type: create + body: { "text": "amsterdam", "@timestamp": "2020-01-01" } + + # rollover data stream to split documents across multiple backing indices + - do: + indices.rollover: + alias: "logs-foobar" + + - match: { old_index: .ds-logs-foobar-000001 } + - match: { new_index: .ds-logs-foobar-000002 } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + index: + index: logs-foobar + id: doc3 + op_type: create + body: { "text": "amsterdam", "@timestamp": "2020-01-01" } + + - do: + index: + index: logs-foobar + id: doc4 + op_type: create + body: { "text": "something about amsterdam and berlin", "@timestamp": "2020-01-01" } + + - do: + indices.refresh: + index: logs-foobar + + - do: + rank_eval: + index: logs-foobar + search_type: query_then_fetch + body: { + "requests" : [ + { + "id": "amsterdam_query", + "request": { "query": { "match" : {"text" : "amsterdam" }}}, + "ratings": [ + {"_index": ".ds-logs-foobar-000001", "_id": "doc1", "rating": 0}, + {"_index": ".ds-logs-foobar-000001", "_id": "doc2", "rating": 1}, + {"_index": ".ds-logs-foobar-000002", "_id": "doc3", "rating": 1}] + }, + { + "id" : "berlin_query", + "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, + "ratings": [{"_index": ".ds-logs-foobar-000001", "_id": "doc1", "rating": 1}] + } + ], + "metric" : { "precision": { "ignore_unlabeled" : true }} + } + + - match: { metric_score: 1} + - match: { details.amsterdam_query.metric_score: 1.0} + - length: { details.amsterdam_query.hits: 3} + - match: { details.berlin_query.metric_score: 1.0} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/50_delete_backing_indices.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/60_get_backing_indices.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/70_rollover_data_streams.yml diff --git a/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml new file mode 100644 index 0000000000000..baafca5a77093 --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/80_resolve_index_data_streams.yml @@ -0,0 +1,178 @@ +--- +setup: + - skip: + version: "7.8.99 - " + reason: "resolve index api only supported in 7.9+" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [simple-data-stream2] + template: + mappings: + properties: + '@timestamp2': + type: date + data_stream: + timestamp_field: '@timestamp2' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + + - do: + indices.create_data_stream: + name: simple-data-stream2 + + - do: + indices.rollover: + alias: "simple-data-stream2" + + - do: + indices.create: + index: test_index1 + body: + aliases: + test_alias: {} + test_blias: {} + test_clias: {} + + - do: + indices.create: + index: test_index2 + body: + aliases: + test_alias: {} + + - do: + indices.close: + index: test_index2 + + - do: + indices.create: + index: test_index3 + body: + aliases: + test_blias: {} + +--- +"Resolve index with indices, aliases, and data streams": + - skip: + version: " - 7.8.99" + reason: "resolve index api only supported in 7.9+" + + - do: + indices.resolve_index: + name: '*' + + - match: {indices.0.name: test_index1} + - match: {indices.0.aliases.0: test_alias} + - match: {indices.0.aliases.1: test_blias} + - match: {indices.0.aliases.2: test_clias} + - match: {indices.0.attributes.0: open} + - match: {indices.1.name: test_index3} + - match: {indices.1.aliases.0: test_blias} + - match: {indices.1.attributes.0: open} + - match: {aliases.0.name: test_alias} + - match: {aliases.0.indices.0: test_index1} + - match: {aliases.0.indices.1: test_index2} + - match: {aliases.1.name: test_blias} + - match: {aliases.1.indices.0: test_index1} + - match: {aliases.1.indices.1: test_index3} + - match: {aliases.2.name: test_clias} + - match: {aliases.2.indices.0: test_index1} + - match: {data_streams.0.name: simple-data-stream1} + - match: {data_streams.0.backing_indices.0: .ds-simple-data-stream1-000001} + - match: {data_streams.0.timestamp_field: "@timestamp"} + - match: {data_streams.1.name: simple-data-stream2} + - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001} + - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002} + - match: {data_streams.1.timestamp_field: "@timestamp"} + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + indices.delete_data_stream: + name: simple-data-stream2 + - is_true: acknowledged + +--- +"Resolve index with hidden and closed indices": + - skip: + version: " - 7.8.99" + reason: change after backporting + + - do: + indices.resolve_index: + name: '*' + expand_wildcards: [all] + + - match: {indices.0.name: .ds-simple-data-stream1-000001} + - match: {indices.0.attributes.0: hidden} + - match: {indices.0.attributes.1: open} + - match: {indices.0.data_stream: simple-data-stream1} + - match: {indices.1.name: .ds-simple-data-stream2-000001} + - match: {indices.1.attributes.0: hidden} + - match: {indices.1.attributes.1: open} + - match: {indices.1.data_stream: simple-data-stream2} + - match: {indices.2.name: .ds-simple-data-stream2-000002} + - match: {indices.2.attributes.0: hidden} + - match: {indices.2.attributes.1: open} + - match: {indices.2.data_stream: simple-data-stream2} + - match: {indices.3.name: test_index1} + - match: {indices.3.aliases.0: test_alias} + - match: {indices.3.aliases.1: test_blias} + - match: {indices.3.aliases.2: test_clias} + - match: {indices.3.attributes.0: open} + - match: {indices.4.name: test_index2} + - match: {indices.4.aliases.0: test_alias} + - match: {indices.4.attributes.0: closed} + - match: {indices.5.name: test_index3} + - match: {indices.5.aliases.0: test_blias} + - match: {indices.5.attributes.0: open} + - match: {aliases.0.name: test_alias} + - match: {aliases.0.indices.0: test_index1} + - match: {aliases.0.indices.1: test_index2} + - match: {aliases.1.name: test_blias} + - match: {aliases.1.indices.0: test_index1} + - match: {aliases.1.indices.1: test_index3} + - match: {aliases.2.name: test_clias} + - match: {aliases.2.indices.0: test_index1} + - match: {data_streams.0.name: simple-data-stream1} + - match: {data_streams.0.backing_indices.0: .ds-simple-data-stream1-000001} + - match: {data_streams.0.timestamp_field: "@timestamp"} + - match: {data_streams.1.name: simple-data-stream2} + - match: {data_streams.1.backing_indices.0: .ds-simple-data-stream2-000001} + - match: {data_streams.1.backing_indices.1: .ds-simple-data-stream2-000002} + - match: {data_streams.1.timestamp_field: "@timestamp"} + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + indices.delete_data_stream: + name: simple-data-stream2 + - is_true: acknowledged diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml b/x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml similarity index 100% rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/96_data_streams.yml rename to x-pack/plugin/data-streams/qa/rest/src/test/resources/rest-api-spec/test/data-streams/90_reindex.yml diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java similarity index 61% rename from server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java rename to x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 636630d80c485..eee1dd698dd42 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1,22 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.indices; +package org.elasticsearch.datastreams; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionRequestBuilder; @@ -34,6 +21,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -51,9 +39,9 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamServiceTests; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ObjectPath; @@ -61,36 +49,30 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import org.junit.After; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT._flush; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.clearCache; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.getAliases; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.getFieldMapping; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.getMapping; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.getSettings; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.health; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.indicesStats; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.msearch; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.putMapping; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.refreshBuilder; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.search; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.segments; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.updateSettings; -import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.validateQuery; +import static org.elasticsearch.action.DocWriteRequest.OpType.CREATE; +import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItemInArray; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -99,9 +81,14 @@ public class DataStreamIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class); + } + @After public void deleteAllComposableTemplates() { - DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[]{"*"}); + DeleteDataStreamAction.Request deleteDSRequest = new DeleteDataStreamAction.Request(new String[] { "*" }); client().execute(DeleteDataStreamAction.INSTANCE, deleteDSRequest).actionGet(); DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); @@ -116,7 +103,7 @@ public void testBasicScenario() throws Exception { createDataStreamRequest = new CreateDataStreamAction.Request("metrics-bar"); client().admin().indices().createDataStream(createDataStreamRequest).get(); - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*"); + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); getDataStreamResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(2)); @@ -124,18 +111,15 @@ public void testBasicScenario() throws Exception { assertThat(firstDataStream.getName(), equalTo("metrics-bar")); assertThat(firstDataStream.getTimeStampField().getName(), equalTo("@timestamp")); assertThat(firstDataStream.getIndices().size(), equalTo(1)); - assertThat(firstDataStream.getIndices().get(0).getName(), - equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 1))); + assertThat(firstDataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 1))); DataStream dataStream = getDataStreamResponse.getDataStreams().get(1).getDataStream(); assertThat(dataStream.getName(), equalTo("metrics-foo")); assertThat(dataStream.getTimeStampField().getName(), equalTo("@timestamp")); assertThat(dataStream.getIndices().size(), equalTo(1)); - assertThat(dataStream.getIndices().get(0).getName(), - equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 1))); + assertThat(dataStream.getIndices().get(0).getName(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 1))); String backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 1); - GetIndexResponse getIndexResponse = - client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); Map mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); @@ -186,23 +170,39 @@ public void testBasicScenario() throws Exception { verifyDocs("metrics-bar", numDocsBar + numDocsBar2, 1, 2); verifyDocs("metrics-foo", numDocsFoo + numDocsFoo2, 1, 2); - DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[]{"metrics-*"}); + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { "metrics-*" }); client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName("metrics-bar", 1))).actionGet()); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName("metrics-bar", 2))).actionGet()); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName("metrics-foo", 1))).actionGet()); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName("metrics-foo", 2))).actionGet()); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 1))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-bar", 2))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 1))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName("metrics-foo", 2))) + .actionGet() + ); } public void testOtherWriteOps() throws Exception { @@ -212,14 +212,12 @@ public void testOtherWriteOps() throws Exception { client().admin().indices().createDataStream(createDataStreamRequest).get(); { - IndexRequest indexRequest = new IndexRequest(dataStreamName) - .source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON); + IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON); Exception e = expectThrows(IndexNotFoundException.class, () -> client().index(indexRequest).actionGet()); assertThat(e.getMessage(), equalTo("no such index [null]")); } { - UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id") - .doc("{}", XContentType.JSON); + UpdateRequest updateRequest = new UpdateRequest(dataStreamName, "_id").doc("{}", XContentType.JSON); Exception e = expectThrows(IndexNotFoundException.class, () -> client().update(updateRequest).actionGet()); assertThat(e.getMessage(), equalTo("no such index [null]")); } @@ -229,17 +227,17 @@ public void testOtherWriteOps() throws Exception { assertThat(e.getMessage(), equalTo("no such index [null]")); } { - IndexRequest indexRequest = new IndexRequest(dataStreamName) - .source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) .opType(DocWriteRequest.OpType.CREATE); IndexResponse indexResponse = client().index(indexRequest).actionGet(); assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); } { - BulkRequest bulkRequest = new BulkRequest() - .add(new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) - .opType(DocWriteRequest.OpType.CREATE)); - BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); + BulkRequest bulkRequest = new BulkRequest().add( + new IndexRequest(dataStreamName).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) + .opType(DocWriteRequest.OpType.CREATE) + ); + BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet(); assertThat(bulkItemResponses.getItems()[0].getIndex(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1))); } } @@ -252,24 +250,27 @@ public void testOtherWriteOps() throws Exception { public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Exception { String dataStreamName = "logs-foobar"; - String mapping = "{\n" + - " \"properties\": {\n" + - " \"baz_field\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"@timestamp\": {\n" + - " \"type\": \"date\"\n" + - " }\n" + - " }\n" + - " }"; + String mapping = "{\n" + + " \"properties\": {\n" + + " \"baz_field\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + " }"; PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("id_1"); request.indexTemplate( new ComposableIndexTemplate( List.of(dataStreamName), // use no wildcard, so that backing indices don't match just by name - new Template(null, - new CompressedXContent(mapping), null), - null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp")) + new Template(null, new CompressedXContent(mapping), null), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); @@ -278,7 +279,7 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except verifyDocs(dataStreamName, numDocs, 1, 1); String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*"); + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); @@ -286,12 +287,13 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), equalTo(backingIndex)); - GetIndexResponse getIndexResponse = - client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName)).actionGet(); + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - assertThat(ObjectPath.eval("properties.baz_field.type", - getIndexResponse.mappings().get(backingIndex).getSourceAsMap()), equalTo("keyword")); + assertThat( + ObjectPath.eval("properties.baz_field.type", getIndexResponse.mappings().get(backingIndex).getSourceAsMap()), + equalTo("keyword") + ); backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 2); RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).get(); @@ -301,24 +303,34 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); - assertThat(ObjectPath.eval("properties.baz_field.type", - getIndexResponse.mappings().get(backingIndex).getSourceAsMap()), equalTo("keyword")); + assertThat( + ObjectPath.eval("properties.baz_field.type", getIndexResponse.mappings().get(backingIndex).getSourceAsMap()), + equalTo("keyword") + ); int numDocs2 = randomIntBetween(2, 16); indexDocs(dataStreamName, "@timestamp", numDocs2); verifyDocs(dataStreamName, numDocs + numDocs2, 1, 2); - DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[]{dataStreamName}); + DeleteDataStreamAction.Request deleteDataStreamRequest = new DeleteDataStreamAction.Request(new String[] { dataStreamName }); client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet(); getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0)); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName(dataStreamName, 1))).actionGet()); - expectThrows(IndexNotFoundException.class, - () -> client().admin().indices().getIndex(new GetIndexRequest().indices( - DataStream.getDefaultBackingIndexName(dataStreamName, 2))).actionGet()); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName(dataStreamName, 1))) + .actionGet() + ); + expectThrows( + IndexNotFoundException.class, + () -> client().admin() + .indices() + .getIndex(new GetIndexRequest().indices(DataStream.getDefaultBackingIndexName(dataStreamName, 2))) + .actionGet() + ); } public void testTimeStampValidationNoFieldMapping() throws Exception { @@ -328,37 +340,51 @@ public void testTimeStampValidationNoFieldMapping() throws Exception { new ComposableIndexTemplate( List.of("logs-*"), new Template(null, new CompressedXContent("{}"), null), - null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp")) + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + ) ); - Exception e = expectThrows(IllegalArgumentException.class, - () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet()); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet() + ); assertThat(e.getCause().getCause().getMessage(), equalTo("the configured timestamp field [@timestamp] does not exist")); } public void testTimeStampValidationInvalidFieldMapping() throws Exception { // Adding a template with an invalid mapping for timestamp field and expect template creation to fail. - String mapping = "{\n" + - " \"properties\": {\n" + - " \"@timestamp\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - " }"; + String mapping = "{\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }"; PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo"); createTemplateRequest.indexTemplate( new ComposableIndexTemplate( List.of("logs-*"), new Template(null, new CompressedXContent(mapping), null), - null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate("@timestamp")) + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + ) ); - Exception e = expectThrows(IllegalArgumentException.class, - () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet()); - assertThat(e.getCause().getCause().getMessage(), - equalTo("the configured timestamp field [@timestamp] is of type [keyword], but [date,date_nanos] is expected")); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet() + ); + assertThat( + e.getCause().getCause().getMessage(), + equalTo("the configured timestamp field [@timestamp] is of type [keyword], but [date,date_nanos] is expected") + ); } public void testResolvabilityOfDataStreamsInAPIs() throws Exception { @@ -367,31 +393,50 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { CreateDataStreamAction.Request request = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(request).actionGet(); - verifyResolvability(dataStreamName, client().prepareIndex(dataStreamName) + verifyResolvability( + dataStreamName, + client().prepareIndex(dataStreamName) .setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) .setOpType(DocWriteRequest.OpType.CREATE), - false); - verifyResolvability(dataStreamName, refreshBuilder(dataStreamName), false); - verifyResolvability(dataStreamName, search(dataStreamName), false, 1); - verifyResolvability(dataStreamName, msearch(null, dataStreamName), false); - verifyResolvability(dataStreamName, clearCache(dataStreamName), false); - verifyResolvability(dataStreamName, _flush(dataStreamName),false); - verifyResolvability(dataStreamName, segments(dataStreamName), false); - verifyResolvability(dataStreamName, indicesStats(dataStreamName), false); - verifyResolvability(dataStreamName, IndicesOptionsIntegrationIT.forceMerge(dataStreamName), false); - verifyResolvability(dataStreamName, validateQuery(dataStreamName), false); + false + ); + verifyResolvability(dataStreamName, client().admin().indices().prepareRefresh(dataStreamName), false); + verifyResolvability(dataStreamName, client().prepareSearch(dataStreamName), false, 1); + verifyResolvability( + dataStreamName, + client().prepareMultiSearch().add(client().prepareSearch(dataStreamName).setQuery(matchAllQuery())), + false + ); + verifyResolvability(dataStreamName, client().admin().indices().prepareClearCache(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareFlush(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareSegments(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareStats(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareForceMerge(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareValidateQuery(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareUpgrade(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareRecoveries(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().indices().prepareUpgradeStatus(dataStreamName), false); - verifyResolvability(dataStreamName, getAliases(dataStreamName), true); - verifyResolvability(dataStreamName, getFieldMapping(dataStreamName), false); - verifyResolvability(dataStreamName, - putMapping("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", dataStreamName), false); - verifyResolvability(dataStreamName, getMapping(dataStreamName), false); - verifyResolvability(dataStreamName, - updateSettings(Settings.builder().put("index.number_of_replicas", 0), dataStreamName), false); - verifyResolvability(dataStreamName, getSettings(dataStreamName), false); - verifyResolvability(dataStreamName, health(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetAliases("dummy").addIndices(dataStreamName), true); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetFieldMappings(dataStreamName), false); + verifyResolvability( + dataStreamName, + client().admin() + .indices() + .preparePutMapping(dataStreamName) + .setSource("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", XContentType.JSON), + false + ); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetMappings(dataStreamName), false); + verifyResolvability( + dataStreamName, + client().admin() + .indices() + .prepareUpdateSettings(dataStreamName) + .setSettings(Settings.builder().put("index.number_of_replicas", 0)), + false + ); + verifyResolvability(dataStreamName, client().admin().indices().prepareGetSettings(dataStreamName), false); + verifyResolvability(dataStreamName, client().admin().cluster().prepareHealth(dataStreamName), false); verifyResolvability(dataStreamName, client().admin().cluster().prepareState().setIndices(dataStreamName), false); verifyResolvability(dataStreamName, client().prepareFieldCaps(dataStreamName).setFields("*"), false); verifyResolvability(dataStreamName, client().admin().indices().prepareGetIndex().addIndices(dataStreamName), false); @@ -401,33 +446,52 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { request = new CreateDataStreamAction.Request("logs-barbaz"); client().admin().indices().createDataStream(request).actionGet(); - verifyResolvability("logs-barbaz", client().prepareIndex("logs-barbaz") + verifyResolvability( + "logs-barbaz", + client().prepareIndex("logs-barbaz") .setSource("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON) .setOpType(DocWriteRequest.OpType.CREATE), - false); + false + ); String wildcardExpression = "logs*"; - verifyResolvability(wildcardExpression, refreshBuilder(wildcardExpression), false); - verifyResolvability(wildcardExpression, search(wildcardExpression), false, 2); - verifyResolvability(wildcardExpression, msearch(null, wildcardExpression), false); - verifyResolvability(wildcardExpression, clearCache(wildcardExpression), false); - verifyResolvability(wildcardExpression, _flush(wildcardExpression),false); - verifyResolvability(wildcardExpression, segments(wildcardExpression), false); - verifyResolvability(wildcardExpression, indicesStats(wildcardExpression), false); - verifyResolvability(wildcardExpression, IndicesOptionsIntegrationIT.forceMerge(wildcardExpression), false); - verifyResolvability(wildcardExpression, validateQuery(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareRefresh(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().prepareSearch(wildcardExpression), false, 2); + verifyResolvability( + wildcardExpression, + client().prepareMultiSearch().add(client().prepareSearch(wildcardExpression).setQuery(matchAllQuery())), + false + ); + verifyResolvability(wildcardExpression, client().admin().indices().prepareClearCache(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareFlush(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareSegments(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareStats(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareForceMerge(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareValidateQuery(wildcardExpression), false); verifyResolvability(wildcardExpression, client().admin().indices().prepareUpgrade(wildcardExpression), false); verifyResolvability(wildcardExpression, client().admin().indices().prepareRecoveries(wildcardExpression), false); verifyResolvability(wildcardExpression, client().admin().indices().prepareUpgradeStatus(wildcardExpression), false); - verifyResolvability(wildcardExpression, getAliases(wildcardExpression), false); - verifyResolvability(wildcardExpression, getFieldMapping(wildcardExpression), false); - verifyResolvability(wildcardExpression, - putMapping("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", wildcardExpression), false); - verifyResolvability(wildcardExpression, getMapping(wildcardExpression), false); - verifyResolvability(wildcardExpression, getSettings(wildcardExpression), false); - verifyResolvability(wildcardExpression, - updateSettings(Settings.builder().put("index.number_of_replicas", 0), wildcardExpression), false); - verifyResolvability(wildcardExpression, health(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareGetAliases(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareGetFieldMappings(wildcardExpression), false); + verifyResolvability( + wildcardExpression, + client().admin() + .indices() + .preparePutMapping(wildcardExpression) + .setSource("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", XContentType.JSON), + false + ); + verifyResolvability(wildcardExpression, client().admin().indices().prepareGetMappings(wildcardExpression), false); + verifyResolvability(wildcardExpression, client().admin().indices().prepareGetSettings(wildcardExpression), false); + verifyResolvability( + wildcardExpression, + client().admin() + .indices() + .prepareUpdateSettings(wildcardExpression) + .setSettings(Settings.builder().put("index.number_of_replicas", 0)), + false + ); + verifyResolvability(wildcardExpression, client().admin().cluster().prepareHealth(wildcardExpression), false); verifyResolvability(wildcardExpression, client().admin().cluster().prepareState().setIndices(wildcardExpression), false); verifyResolvability(wildcardExpression, client().prepareFieldCaps(wildcardExpression).setFields("*"), false); verifyResolvability(wildcardExpression, client().admin().indices().prepareGetIndex().addIndices(wildcardExpression), false); @@ -446,16 +510,26 @@ public void testCannotDeleteComposableTemplateUsedByDataStream() throws Exceptio DeleteComposableIndexTemplateAction.Request req = new DeleteComposableIndexTemplateAction.Request("id"); Exception e = expectThrows(Exception.class, () -> client().execute(DeleteComposableIndexTemplateAction.INSTANCE, req).get()); - Optional maybeE = ExceptionsHelper.unwrapCausesAndSuppressed(e, err -> - err.getMessage().contains("unable to remove composable templates [id] " + - "as they are in use by a data streams [metrics-foobar-baz, metrics-foobar-baz-eggplant]")); + Optional maybeE = ExceptionsHelper.unwrapCausesAndSuppressed( + e, + err -> err.getMessage() + .contains( + "unable to remove composable templates [id] " + + "as they are in use by a data streams [metrics-foobar-baz, metrics-foobar-baz-eggplant]" + ) + ); assertTrue(maybeE.isPresent()); DeleteComposableIndexTemplateAction.Request req2 = new DeleteComposableIndexTemplateAction.Request("i*"); Exception e2 = expectThrows(Exception.class, () -> client().execute(DeleteComposableIndexTemplateAction.INSTANCE, req2).get()); - maybeE = ExceptionsHelper.unwrapCausesAndSuppressed(e2, err -> - err.getMessage().contains("unable to remove composable templates [id] " + - "as they are in use by a data streams [metrics-foobar-baz, metrics-foobar-baz-eggplant]")); + maybeE = ExceptionsHelper.unwrapCausesAndSuppressed( + e2, + err -> err.getMessage() + .contains( + "unable to remove composable templates [id] " + + "as they are in use by a data streams [metrics-foobar-baz, metrics-foobar-baz-eggplant]" + ) + ); assertTrue(maybeE.isPresent()); } @@ -466,11 +540,12 @@ public void testAliasActionsFailOnDataStreams() throws Exception { client().admin().indices().createDataStream(createDataStreamRequest).get(); IndicesAliasesRequest.AliasActions addAction = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD) - .index(dataStreamName).aliases("foo"); + .index(dataStreamName) + .aliases("foo"); IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest(); aliasesAddRequest.addAliasAction(addAction); Exception e = expectThrows(IndexNotFoundException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet()); - assertThat(e.getMessage(), equalTo("no such index [" + dataStreamName +"]")); + assertThat(e.getMessage(), equalTo("no such index [" + dataStreamName + "]")); } public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { @@ -481,32 +556,41 @@ public void testAliasActionsFailOnDataStreamBackingIndices() throws Exception { String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); IndicesAliasesRequest.AliasActions addAction = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD) - .index(backingIndex).aliases("first_gen"); + .index(backingIndex) + .aliases("first_gen"); IndicesAliasesRequest aliasesAddRequest = new IndicesAliasesRequest(); aliasesAddRequest.addAliasAction(addAction); Exception e = expectThrows(IllegalArgumentException.class, () -> client().admin().indices().aliases(aliasesAddRequest).actionGet()); - assertThat(e.getMessage(), equalTo("The provided expressions [" + backingIndex - + "] match a backing index belonging to data stream [" + dataStreamName + "]. Data streams and their backing indices don't " + - "support aliases.")); + assertThat( + e.getMessage(), + equalTo( + "The provided expressions [" + + backingIndex + + "] match a backing index belonging to data stream [" + + dataStreamName + + "]. Data streams and their backing indices don't " + + "support aliases." + ) + ); } public void testTimestampFieldCustomAttributes() throws Exception { - String mapping = "{\n" + - " \"properties\": {\n" + - " \"@timestamp\": {\n" + - " \"type\": \"date\",\n" + - " \"format\": \"yyyy-MM\",\n" + - " \"meta\": {\n" + - " \"x\": \"y\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }"; + String mapping = "{\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\",\n" + + " \"format\": \"yyyy-MM\",\n" + + " \"meta\": {\n" + + " \"x\": \"y\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; putComposableIndexTemplate("id1", "@timestamp", mapping, List.of("logs-foo*"), null); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar"); client().admin().indices().createDataStream(createDataStreamRequest).get(); - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("logs-foobar"); + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "logs-foobar" }); GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); @@ -527,18 +611,30 @@ public void testUpdateMappingViaDataStream() throws Exception { assertThat(rolloverResponse.getNewIndex(), equalTo(backingIndex2)); assertTrue(rolloverResponse.isRolledOver()); - Map expectedMapping = - Map.of("properties", Map.of("@timestamp", Map.of("type", "date")), "_timestamp", Map.of("path", "@timestamp")); - GetMappingsResponse getMappingsResponse = getMapping("logs-foobar").get(); + Map expectedMapping = Map.of( + "properties", + Map.of("@timestamp", Map.of("type", "date")), + "_data_stream_timestamp", + Map.of("path", "@timestamp") + ); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); - expectedMapping = Map.of("properties", Map.of("@timestamp", Map.of("type", "date"), "my_field", Map.of("type", "keyword")), - "_timestamp", Map.of("path", "@timestamp")); - putMapping("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", "logs-foobar").get(); + expectedMapping = Map.of( + "properties", + Map.of("@timestamp", Map.of("type", "date"), "my_field", Map.of("type", "keyword")), + "_data_stream_timestamp", + Map.of("path", "@timestamp") + ); + client().admin() + .indices() + .preparePutMapping("logs-foobar") + .setSource("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", XContentType.JSON) + .get(); // The mappings of all backing indices should be updated: - getMappingsResponse = getMapping("logs-foobar").get(); + getMappingsResponse = client().admin().indices().prepareGetMappings("logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); @@ -557,13 +653,17 @@ public void testUpdateIndexSettingsViaDataStream() throws Exception { assertTrue(rolloverResponse.isRolledOver()); // The index settings of all backing indices should be updated: - GetSettingsResponse getSettingsResponse = getSettings("logs-foobar").get(); + GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("logs-foobar").get(); assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(2)); assertThat(getSettingsResponse.getSetting(backingIndex1, "index.number_of_replicas"), equalTo("1")); assertThat(getSettingsResponse.getSetting(backingIndex2, "index.number_of_replicas"), equalTo("1")); - updateSettings(Settings.builder().put("index.number_of_replicas", 0), "logs-foobar").get(); - getSettingsResponse = getSettings("logs-foobar").get(); + client().admin() + .indices() + .prepareUpdateSettings("logs-foobar") + .setSettings(Settings.builder().put("index.number_of_replicas", 0)) + .get(); + getSettingsResponse = client().admin().indices().prepareGetSettings("logs-foobar").get(); assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(2)); assertThat(getSettingsResponse.getSetting(backingIndex1, "index.number_of_replicas"), equalTo("0")); assertThat(getSettingsResponse.getSetting(backingIndex2, "index.number_of_replicas"), equalTo("0")); @@ -580,28 +680,41 @@ public void testIndexDocsWithCustomRoutingTargetingDataStreamIsNotAllowed() thro assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName(dataStream, 1))); // Index doc with custom routing that targets the data stream - IndexRequest indexRequestWithRouting = - new IndexRequest(dataStream).source("@timestamp", System.currentTimeMillis()).opType(DocWriteRequest.OpType.CREATE) - .routing("custom"); - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> client().index(indexRequestWithRouting).actionGet()); - assertThat(exception.getMessage(), is("index request targeting data stream [logs-foobar] specifies a custom routing. target the " + - "backing indices directly or remove the custom routing.")); + IndexRequest indexRequestWithRouting = new IndexRequest(dataStream).source("@timestamp", System.currentTimeMillis()) + .opType(DocWriteRequest.OpType.CREATE) + .routing("custom"); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> client().index(indexRequestWithRouting).actionGet() + ); + assertThat( + exception.getMessage(), + is( + "index request targeting data stream [logs-foobar] specifies a custom routing. target the " + + "backing indices directly or remove the custom routing." + ) + ); // Bulk indexing with custom routing targeting the data stream is also prohibited BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < 10; i++) { - bulkRequest.add(new IndexRequest(dataStream) - .opType(DocWriteRequest.OpType.CREATE) - .routing("bulk-request-routing") - .source("{}", XContentType.JSON)); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .routing("bulk-request-routing") + .source("{}", XContentType.JSON) + ); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); for (BulkItemResponse responseItem : bulkResponse.getItems()) { assertThat(responseItem.getFailure(), notNullValue()); - assertThat(responseItem.getFailureMessage(), is("java.lang.IllegalArgumentException: index request targeting data stream " + - "[logs-foobar] specifies a custom routing. target the backing indices directly or remove the custom routing.")); + assertThat( + responseItem.getFailureMessage(), + is( + "java.lang.IllegalArgumentException: index request targeting data stream " + + "[logs-foobar] specifies a custom routing. target the backing indices directly or remove the custom routing." + ) + ); } } @@ -615,9 +728,15 @@ public void testIndexDocsWithCustomRoutingTargetingBackingIndex() throws Excepti assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); // Index doc with custom routing that targets the backing index - IndexRequest indexRequestWithRouting = new IndexRequest(DataStream.getDefaultBackingIndexName("logs-foobar", 1L)) - .source("@timestamp", System.currentTimeMillis()).opType(DocWriteRequest.OpType.INDEX).routing("custom") - .id(indexResponse.getId()).setIfPrimaryTerm(indexResponse.getPrimaryTerm()).setIfSeqNo(indexResponse.getSeqNo()); + IndexRequest indexRequestWithRouting = new IndexRequest(DataStream.getDefaultBackingIndexName("logs-foobar", 1L)).source( + "@timestamp", + System.currentTimeMillis() + ) + .opType(DocWriteRequest.OpType.INDEX) + .routing("custom") + .id(indexResponse.getId()) + .setIfPrimaryTerm(indexResponse.getPrimaryTerm()) + .setIfSeqNo(indexResponse.getSeqNo()); IndexResponse response = client().index(indexRequestWithRouting).actionGet(); assertThat(response.getIndex(), equalTo(DataStream.getDefaultBackingIndexName("logs-foobar", 1))); } @@ -649,16 +768,16 @@ public void testSearchAllResolvesDataStreams() throws Exception { } public void testGetDataStream() throws Exception { - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2) - .build(); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build(); putComposableIndexTemplate("template_for_foo", "@timestamp", List.of("metrics-foo*"), settings); int numDocsFoo = randomIntBetween(2, 16); indexDocs("metrics-foo", "@timestamp", numDocsFoo); - GetDataStreamAction.Response response = - client().admin().indices().getDataStreams(new GetDataStreamAction.Request("metrics-foo")).actionGet(); + GetDataStreamAction.Response response = client().admin() + .indices() + .getDataStreams(new GetDataStreamAction.Request(new String[] { "metrics-foo" })) + .actionGet(); assertThat(response.getDataStreams().size(), is(1)); GetDataStreamAction.Response.DataStreamInfo metricsFooDataStream = response.getDataStreams().get(0); assertThat(metricsFooDataStream.getDataStream().getName(), is("metrics-foo")); @@ -672,8 +791,7 @@ private static void assertBackingIndex(String backingIndex, String timestampFiel } private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping, Map expectedMapping) { - GetIndexResponse getIndexResponse = - client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet(); assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue()); assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true)); Map mappings = getIndexResponse.getMappings().get(backingIndex).getSourceAsMap(); @@ -686,9 +804,7 @@ public void testNoTimestampInDocument() throws Exception { CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); - IndexRequest indexRequest = new IndexRequest(dataStreamName) - .opType("create") - .source("{}", XContentType.JSON); + IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create").source("{}", XContentType.JSON); Exception e = expectThrows(MapperParsingException.class, () -> client().index(indexRequest).actionGet()); assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); } @@ -699,19 +815,109 @@ public void testMultipleTimestampValuesInDocument() throws Exception { CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName); client().admin().indices().createDataStream(createDataStreamRequest).get(); - IndexRequest indexRequest = new IndexRequest(dataStreamName) - .opType("create") + IndexRequest indexRequest = new IndexRequest(dataStreamName).opType("create") .source("{\"@timestamp\": [\"2020-12-12\",\"2022-12-12\"]}", XContentType.JSON); Exception e = expectThrows(MapperParsingException.class, () -> client().index(indexRequest).actionGet()); - assertThat(e.getCause().getMessage(), - equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + } + + public void testMixedAutoCreate() throws Exception { + PutComposableIndexTemplateAction.Request createTemplateRequest = new PutComposableIndexTemplateAction.Request("logs-foo"); + createTemplateRequest.indexTemplate( + new ComposableIndexTemplate( + List.of("logs-foo*"), + new Template(null, new CompressedXContent(generateMapping("@timestamp")), null), + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate("@timestamp") + ) + ); + client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-foobaz3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barbaz").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo2").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkRequest.add(new IndexRequest("logs-barfoo3").opType(CREATE).source("{\"@timestamp\": \"2020-12-12\"}", XContentType.JSON)); + bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); + GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamsResponse.getDataStreams(), hasSize(4)); + getDataStreamsResponse.getDataStreams().sort(Comparator.comparing(dataStreamInfo -> dataStreamInfo.getDataStream().getName())); + assertThat(getDataStreamsResponse.getDataStreams().get(0).getDataStream().getName(), equalTo("logs-foobar")); + assertThat(getDataStreamsResponse.getDataStreams().get(1).getDataStream().getName(), equalTo("logs-foobaz")); + assertThat(getDataStreamsResponse.getDataStreams().get(2).getDataStream().getName(), equalTo("logs-foobaz2")); + assertThat(getDataStreamsResponse.getDataStreams().get(3).getDataStream().getName(), equalTo("logs-foobaz3")); + + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-bar*")).actionGet(); + assertThat(getIndexResponse.getIndices(), arrayWithSize(4)); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barbaz")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo2")); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-barfoo3")); + + DeleteDataStreamAction.Request deleteDSReq = new DeleteDataStreamAction.Request(new String[] { "*" }); + client().execute(DeleteDataStreamAction.INSTANCE, deleteDSReq).actionGet(); + DeleteComposableIndexTemplateAction.Request deleteTemplateRequest = new DeleteComposableIndexTemplateAction.Request("*"); + client().execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteTemplateRequest).actionGet(); + } + + public void testAutoCreateV1TemplateNoDataStream() { + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(); + + PutIndexTemplateRequest v1Request = new PutIndexTemplateRequest("logs-foo"); + v1Request.patterns(List.of("logs-foo*")); + v1Request.settings(settings); + v1Request.order(Integer.MAX_VALUE); // in order to avoid number_of_replicas being overwritten by random_template + client().admin().indices().putTemplate(v1Request).actionGet(); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("logs-foobar").opType(CREATE).source("{}", XContentType.JSON)); + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat("bulk failures: " + Strings.toString(bulkResponse), bulkResponse.hasFailures(), is(false)); + + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); + GetDataStreamAction.Response getDataStreamsResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet(); + assertThat(getDataStreamsResponse.getDataStreams(), hasSize(0)); + + GetIndexResponse getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("logs-foobar")).actionGet(); + assertThat(getIndexResponse.getIndices(), arrayWithSize(1)); + assertThat(getIndexResponse.getIndices(), hasItemInArray("logs-foobar")); + assertThat(getIndexResponse.getSettings().get("logs-foobar").get(IndexMetadata.SETTING_NUMBER_OF_REPLICAS), equalTo("0")); } - private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail) { + private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail) { verifyResolvability(dataStream, requestBuilder, fail, 0); } - private static void verifyResolvability(String dataStream, ActionRequestBuilder requestBuilder, boolean fail, long expectedCount) { + private static void verifyResolvability( + String dataStream, + ActionRequestBuilder requestBuilder, + boolean fail, + long expectedCount + ) { if (fail) { String expectedErrorMessage = "no such index [" + dataStream + "]"; if (requestBuilder instanceof MultiSearchRequestBuilder) { @@ -744,9 +950,10 @@ private static void indexDocs(String dataStream, String timestampField, int numD BulkRequest bulkRequest = new BulkRequest(); for (int i = 0; i < numDocs; i++) { String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add(new IndexRequest(dataStream) - .opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", timestampField, value), XContentType.JSON)); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", timestampField, value), XContentType.JSON) + ); } BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); assertThat(bulkResponse.getItems().length, equalTo(numDocs)); @@ -770,31 +977,38 @@ private static void verifyDocs(String dataStream, long expectedNumHits, long min for (long k = minGeneration; k <= maxGeneration; k++) { expectedIndices.add(DataStream.getDefaultBackingIndexName(dataStream, k)); } - Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { - assertTrue(expectedIndices.contains(hit.getIndex())); - }); + Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> { assertTrue(expectedIndices.contains(hit.getIndex())); }); } public static void putComposableIndexTemplate(String id, String timestampFieldName, List patterns) throws IOException { - String mapping = MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName); + String mapping = generateMapping(timestampFieldName); putComposableIndexTemplate(id, timestampFieldName, mapping, patterns, null); } - static void putComposableIndexTemplate(String id, String timestampFieldName, List patterns, - Settings settings) throws IOException { - String mapping = MetadataCreateDataStreamServiceTests.generateMapping(timestampFieldName); + static void putComposableIndexTemplate(String id, String timestampFieldName, List patterns, Settings settings) + throws IOException { + String mapping = generateMapping(timestampFieldName); putComposableIndexTemplate(id, timestampFieldName, mapping, patterns, settings); } - static void putComposableIndexTemplate(String id, String timestampFieldName, String mapping, List patterns, - @Nullable Settings settings) throws IOException { + static void putComposableIndexTemplate( + String id, + String timestampFieldName, + String mapping, + List patterns, + @Nullable Settings settings + ) throws IOException { PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(id); request.indexTemplate( new ComposableIndexTemplate( patterns, new Template(settings, new CompressedXContent(mapping), null), - null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName)) + null, + null, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(timestampFieldName) + ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java similarity index 72% rename from server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java rename to x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index bafc913e5b88a..8dfb520b30018 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DataStreamsSnapshotsIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -1,23 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ - -package org.elasticsearch.snapshots; +package org.elasticsearch.datastreams; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.DocWriteRequest; @@ -33,12 +19,20 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.indices.DataStreamIT; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotRestoreException; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; +import org.hamcrest.Matchers; import org.junit.Before; import java.nio.file.Path; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -60,6 +54,11 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase { private String id; + @Override + protected Collection> nodePlugins() { + return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class); + } + @Before public void setup() throws Exception { client = client(); @@ -76,16 +75,14 @@ public void setup() throws Exception { response = client.admin().indices().createDataStream(request).get(); assertTrue(response.isAcknowledged()); - IndexResponse indexResponse = client.prepareIndex("ds") - .setOpType(DocWriteRequest.OpType.CREATE) - .setSource(DOCUMENT_SOURCE) - .get(); + IndexResponse indexResponse = client.prepareIndex("ds").setOpType(DocWriteRequest.OpType.CREATE).setSource(DOCUMENT_SOURCE).get(); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); id = indexResponse.getId(); } public void testSnapshotAndRestore() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -100,10 +97,12 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(1, snap.size()); assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), snap.get(0).indices()); - assertTrue(client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"ds"})).get() - .isAcknowledged()); + assertTrue( + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get().isAcknowledged() + ); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() .prepareRestoreSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -116,19 +115,23 @@ public void testSnapshotAndRestore() throws Exception { assertEquals(1, hits.length); assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); - GetDataStreamAction.Response ds = client.admin().indices().getDataStreams(new GetDataStreamAction.Request("ds")).get(); + GetDataStreamAction.Response ds = client.admin() + .indices() + .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds" })) + .get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); } public void testSnapshotAndRestoreAll() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() - .prepareCreateSnapshot(REPO, SNAPSHOT) - .setWaitForCompletion(true) - .setIndices("ds") - .setIncludeGlobalState(false) - .get(); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setIndices("ds") + .setIncludeGlobalState(false) + .get(); RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); @@ -138,14 +141,15 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(1, snap.size()); assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), snap.get(0).indices()); - assertAcked(client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"*"})).get()); + assertAcked(client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "*" })).get()); assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() - .prepareRestoreSnapshot(REPO, SNAPSHOT) - .setWaitForCompletion(true) - .setRestoreGlobalState(true) - .get(); + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() + .prepareRestoreSnapshot(REPO, SNAPSHOT) + .setWaitForCompletion(true) + .setRestoreGlobalState(true) + .get(); assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards()); @@ -154,16 +158,20 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(1, hits.length); assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap()); - GetDataStreamAction.Response ds = client.admin().indices().getDataStreams(new GetDataStreamAction.Request("ds")).get(); + GetDataStreamAction.Response ds = client.admin() + .indices() + .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds" })) + .get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertAcked(client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"ds"})).get()); + assertAcked(client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get()); } public void testRename() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -173,13 +181,13 @@ public void testRename() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - expectThrows(SnapshotRestoreException.class, () -> client.admin().cluster() - .prepareRestoreSnapshot(REPO, SNAPSHOT) - .setWaitForCompletion(true) - .setIndices("ds") - .get()); + expectThrows( + SnapshotRestoreException.class, + () -> client.admin().cluster().prepareRestoreSnapshot(REPO, SNAPSHOT).setWaitForCompletion(true).setIndices("ds").get() + ); - client.admin().cluster() + client.admin() + .cluster() .prepareRestoreSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -187,7 +195,10 @@ public void testRename() throws Exception { .setRenameReplacement("ds2") .get(); - GetDataStreamAction.Response ds = client.admin().indices().getDataStreams(new GetDataStreamAction.Request("ds2")).get(); + GetDataStreamAction.Response ds = client.admin() + .indices() + .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds2" })) + .get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); @@ -196,7 +207,8 @@ public void testRename() throws Exception { } public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -206,17 +218,17 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - expectThrows(SnapshotRestoreException.class, () -> client.admin().cluster() - .prepareRestoreSnapshot(REPO, SNAPSHOT) - .setWaitForCompletion(true) - .setIndices("ds") - .get()); + expectThrows( + SnapshotRestoreException.class, + () -> client.admin().cluster().prepareRestoreSnapshot(REPO, SNAPSHOT).setWaitForCompletion(true).setIndices("ds").get() + ); // delete data stream - client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"ds"})).actionGet(); + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).actionGet(); // restore data stream attempting to rename the backing index - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() .prepareRestoreSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -226,13 +238,14 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() { assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK)); - GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request("ds"); + GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" }); GetDataStreamAction.Response response = client.admin().indices().getDataStreams(getDSRequest).actionGet(); assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME)); } public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -242,14 +255,14 @@ public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - expectThrows(SnapshotRestoreException.class, () -> client.admin().cluster() - .prepareRestoreSnapshot(REPO, SNAPSHOT) - .setWaitForCompletion(true) - .setIndices("ds") - .get()); + expectThrows( + SnapshotRestoreException.class, + () -> client.admin().cluster().prepareRestoreSnapshot(REPO, SNAPSHOT).setWaitForCompletion(true).setIndices("ds").get() + ); // restore data stream attempting to rename the backing index - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() .prepareRestoreSnapshot(REPO, SNAPSHOT) .setWaitForCompletion(true) .setIndices("ds") @@ -260,19 +273,22 @@ public void testDataStreamAndBackingIndidcesAreRenamedUsingRegex() { assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK)); // assert "ds" was restored as "test-ds" and the backing index has a valid name - GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request("test-ds"); + GetDataStreamAction.Request getRenamedDS = new GetDataStreamAction.Request(new String[] { "test-ds" }); GetDataStreamAction.Response response = client.admin().indices().getDataStreams(getRenamedDS).actionGet(); - assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), - is(DataStream.getDefaultBackingIndexName("test-ds", 1L))); + assertThat( + response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), + is(DataStream.getDefaultBackingIndexName("test-ds", 1L)) + ); // data stream "ds" should still exist in the system - GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request("ds"); + GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" }); response = client.admin().indices().getDataStreams(getDSRequest).actionGet(); assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME)); } public void testWildcards() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, "snap2") .setWaitForCompletion(true) .setIndices("d*") @@ -282,7 +298,8 @@ public void testWildcards() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() .prepareRestoreSnapshot(REPO, "snap2") .setWaitForCompletion(true) .setIndices("d*") @@ -292,16 +309,23 @@ public void testWildcards() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); - GetDataStreamAction.Response ds = client.admin().indices().getDataStreams(new GetDataStreamAction.Request("ds2")).get(); + GetDataStreamAction.Response ds = client.admin() + .indices() + .getDataStreams(new GetDataStreamAction.Request(new String[] { "ds2" })) + .get(); assertEquals(1, ds.getDataStreams().size()); assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size()); assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName()); - assertThat("we renamed the restored data stream to one that doesn't match any existing composable template", - ds.getDataStreams().get(0).getIndexTemplate(), is(nullValue())); + assertThat( + "we renamed the restored data stream to one that doesn't match any existing composable template", + ds.getDataStreams().get(0).getIndexTemplate(), + is(nullValue()) + ); } public void testDataStreamNotStoredWhenIndexRequested() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, "snap2") .setWaitForCompletion(true) .setIndices(DS_BACKING_INDEX_NAME) @@ -310,15 +334,15 @@ public void testDataStreamNotStoredWhenIndexRequested() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - expectThrows(Exception.class, () -> client.admin().cluster() - .prepareRestoreSnapshot(REPO, "snap2") - .setWaitForCompletion(true) - .setIndices("ds") - .get()); + expectThrows( + Exception.class, + () -> client.admin().cluster().prepareRestoreSnapshot(REPO, "snap2").setWaitForCompletion(true).setIndices("ds").get() + ); } public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() .prepareCreateSnapshot(REPO, "snap2") .setWaitForCompletion(true) .setIndices("ds") @@ -328,10 +352,12 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { RestStatus status = createSnapshotResponse.getSnapshotInfo().status(); assertEquals(RestStatus.OK, status); - assertTrue(client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"ds"})).get() - .isAcknowledged()); + assertTrue( + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "ds" })).get().isAcknowledged() + ); - RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster() + RestoreSnapshotResponse restoreSnapshotResponse = client.admin() + .cluster() .prepareRestoreSnapshot(REPO, "snap2") .setWaitForCompletion(true) .setIndices(".ds-ds-*") @@ -339,25 +365,28 @@ public void testDataStreamNotRestoredWhenIndexRequested() throws Exception { assertEquals(RestStatus.OK, restoreSnapshotResponse.status()); - GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request("ds"); + GetDataStreamAction.Request getRequest = new GetDataStreamAction.Request(new String[] { "ds" }); expectThrows(ResourceNotFoundException.class, () -> client.admin().indices().getDataStreams(getRequest).actionGet()); } public void testDataStreamNotIncludedInLimitedSnapshot() throws ExecutionException, InterruptedException { final String snapshotName = "test-snap"; - CreateSnapshotResponse createSnapshotResponse = client.admin().cluster() - .prepareCreateSnapshot(REPO, snapshotName) - .setWaitForCompletion(true) - .setIndices("does-not-exist-*") - .setIncludeGlobalState(true) - .get(); - assertThat(createSnapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS)); - - assertThat(client().admin().indices() - .deleteDataStream(new DeleteDataStreamAction.Request(new String[]{"*"})).get().isAcknowledged(), is(true)); - - final RestoreSnapshotResponse restoreSnapshotResponse = - client().admin().cluster().prepareRestoreSnapshot(REPO, snapshotName).get(); + CreateSnapshotResponse createSnapshotResponse = client.admin() + .cluster() + .prepareCreateSnapshot(REPO, snapshotName) + .setWaitForCompletion(true) + .setIndices("does-not-exist-*") + .setIncludeGlobalState(true) + .get(); + assertThat(createSnapshotResponse.getSnapshotInfo().state(), Matchers.is(SnapshotState.SUCCESS)); + + assertThat( + client().admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { "*" })).get().isAcknowledged(), + is(true) + ); + + final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(REPO, snapshotName).get(); assertThat(restoreSnapshotResponse.getRestoreInfo().indices(), empty()); } + } diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java new file mode 100644 index 0000000000000..96a91e47c9197 --- /dev/null +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ShardClusterSnapshotRestoreIT.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.datastreams; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.snapshots.SnapshotInProgressException; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.snapshots.mockstore.MockRepository; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +// The tests in here do a lot of state updates and other writes to disk and are slowed down too much by WindowsFS +@LuceneTestCase.SuppressFileSystems(value = "WindowsFS") +public class ShardClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(MockRepository.Plugin.class, DataStreamsPlugin.class); + } + + public void testDeleteDataStreamDuringSnapshot() throws Exception { + Client client = client(); + + createRepository( + "test-repo", + "mock", + Settings.builder() + .put("location", randomRepoPath()) + .put("compress", randomBoolean()) + .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES) + .put("block_on_data", true) + ); + + String dataStream = "datastream"; + DataStreamIT.putComposableIndexTemplate("dst", "@timestamp", List.of(dataStream)); + + logger.info("--> indexing some data"); + for (int i = 0; i < 100; i++) { + client.prepareIndex(dataStream) + .setOpType(DocWriteRequest.OpType.CREATE) + .setId(Integer.toString(i)) + .setSource(Collections.singletonMap("@timestamp", "2020-12-12")) + .execute() + .actionGet(); + } + refresh(); + assertDocCount(dataStream, 100L); + + logger.info("--> snapshot"); + ActionFuture future = client.admin() + .cluster() + .prepareCreateSnapshot("test-repo", "test-snap") + .setIndices(dataStream) + .setWaitForCompletion(true) + .setPartial(false) + .execute(); + logger.info("--> wait for block to kick in"); + waitForBlockOnAnyDataNode("test-repo", TimeValue.timeValueMinutes(1)); + + // non-partial snapshots do not allow delete operations on data streams where snapshot has not been completed + try { + logger.info("--> delete index while non-partial snapshot is running"); + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { dataStream })).actionGet(); + fail("Expected deleting index to fail during snapshot"); + } catch (SnapshotInProgressException e) { + assertThat(e.getMessage(), containsString("Cannot delete data streams that are being snapshotted: [" + dataStream)); + } finally { + logger.info("--> unblock all data nodes"); + unblockAllDataNodes("test-repo"); + } + logger.info("--> waiting for snapshot to finish"); + CreateSnapshotResponse createSnapshotResponse = future.get(); + + logger.info("Snapshot successfully completed"); + SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.state(), equalTo((SnapshotState.SUCCESS))); + assertThat(snapshotInfo.dataStreams(), contains(dataStream)); + assertThat(snapshotInfo.indices(), contains(DataStream.getDefaultBackingIndexName(dataStream, 1))); + } + +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamInfoTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamInfoTransportAction.java new file mode 100644 index 0000000000000..7e1ba3ae5fc78 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamInfoTransportAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; + +public class DataStreamInfoTransportAction extends XPackInfoFeatureTransportAction { + + @Inject + public DataStreamInfoTransportAction(TransportService transportService, ActionFilters actionFilters) { + super(XPackInfoFeatureAction.DATA_STREAMS.name(), transportService, actionFilters); + } + + @Override + public String name() { + return XPackField.DATA_STREAMS; + } + + @Override + public boolean available() { + return true; + } + + @Override + public boolean enabled() { + return true; + } + +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamUsageTransportAction.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamUsageTransportAction.java new file mode 100644 index 0000000000000..b81686b16e272 --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamUsageTransportAction.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.datastreams.DataStreamFeatureSetUsage; + +import java.util.Map; + +public class DataStreamUsageTransportAction extends XPackUsageFeatureTransportAction { + + @Inject + public DataStreamUsageTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + XPackUsageFeatureAction.DATA_STREAMS.name(), + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver + ); + } + + @Override + protected void masterOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + final Map dataStreams = state.metadata().dataStreams(); + final DataStreamFeatureSetUsage.DataStreamStats stats = new DataStreamFeatureSetUsage.DataStreamStats( + dataStreams.size(), + dataStreams.values().stream().map(ds -> ds.getIndices().size()).reduce(Integer::sum).orElse(0) + ); + final DataStreamFeatureSetUsage usage = new DataStreamFeatureSetUsage(stats); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } +} diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java new file mode 100644 index 0000000000000..d179c87de340b --- /dev/null +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.datastreams.mapper.DataStreamTimestampFieldMapper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.action.ActionModule.DATASTREAMS_FEATURE_ENABLED; + +public class DataStreamsPlugin extends Plugin implements ActionPlugin, MapperPlugin { + + @Override + public Map getMetadataMappers() { + if (DATASTREAMS_FEATURE_ENABLED) { + return Map.of(DataStreamTimestampFieldMapper.NAME, new DataStreamTimestampFieldMapper.TypeParser()); + } else { + return Map.of(); + } + } + + @Override + public List> getActions() { + var dsUsageAction = new ActionHandler<>(XPackUsageFeatureAction.DATA_STREAMS, DataStreamUsageTransportAction.class); + var dsInfoAction = new ActionHandler<>(XPackInfoFeatureAction.DATA_STREAMS, DataStreamInfoTransportAction.class); + List> actions = new ArrayList<>(); + actions.add(dsUsageAction); + actions.add(dsInfoAction); + return actions; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java similarity index 67% rename from server/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java rename to x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index c69f9a27ca35e..e07a323c3e9da 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -1,23 +1,10 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; +package org.elasticsearch.xpack.datastreams.mapper; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; @@ -28,6 +15,15 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentFieldMappers; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -40,11 +36,11 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -public class TimestampFieldMapper extends MetadataFieldMapper { +public class DataStreamTimestampFieldMapper extends MetadataFieldMapper { - public static final String NAME = "_timestamp"; + public static final String NAME = "_data_stream_timestamp"; - public static class Defaults { + public static class Defaults { public static final FieldType TIMESTAMP_FIELD_TYPE = new FieldType(); @@ -62,11 +58,6 @@ public TimestampFieldType() { super(NAME, false, false, TextSearchInfo.NONE, Map.of()); } - @Override - public MappedFieldType clone() { - return new TimestampFieldType(); - } - @Override public String typeName() { return NAME; @@ -98,20 +89,15 @@ public void setPath(String path) { @Override public MetadataFieldMapper build(BuilderContext context) { - return new TimestampFieldMapper( - fieldType, - new TimestampFieldType(), - path - ); + return new DataStreamTimestampFieldMapper(fieldType, new TimestampFieldType(), path); } } public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public MetadataFieldMapper.Builder parse(String name, - Map node, - ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) + throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -127,14 +113,13 @@ public MetadataFieldMapper.Builder parse(String name, @Override public MetadataFieldMapper getDefault(ParserContext parserContext) { - return new TimestampFieldMapper(Defaults.TIMESTAMP_FIELD_TYPE, - new TimestampFieldType(), null); + return new DataStreamTimestampFieldMapper(Defaults.TIMESTAMP_FIELD_TYPE, new TimestampFieldType(), null); } } private final String path; - private TimestampFieldMapper(FieldType fieldType, MappedFieldType mappedFieldType, String path) { + private DataStreamTimestampFieldMapper(FieldType fieldType, MappedFieldType mappedFieldType, String path) { super(fieldType, mappedFieldType); this.path = path; } @@ -150,11 +135,19 @@ public void validate(DocumentFieldMappers lookup) { throw new IllegalArgumentException("the configured timestamp field [" + path + "] does not exist"); } - if (DateFieldMapper.CONTENT_TYPE.equals(mapper.typeName()) == false && - DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(mapper.typeName()) == false) { - throw new IllegalArgumentException("the configured timestamp field [" + path + "] is of type [" + - mapper.typeName() + "], but [" + DateFieldMapper.CONTENT_TYPE + "," + DateFieldMapper.DATE_NANOS_CONTENT_TYPE + - "] is expected"); + if (DateFieldMapper.CONTENT_TYPE.equals(mapper.typeName()) == false + && DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(mapper.typeName()) == false) { + throw new IllegalArgumentException( + "the configured timestamp field [" + + path + + "] is of type [" + + mapper.typeName() + + "], but [" + + DateFieldMapper.CONTENT_TYPE + + "," + + DateFieldMapper.DATE_NANOS_CONTENT_TYPE + + "] is expected" + ); } DateFieldMapper dateFieldMapper = (DateFieldMapper) mapper; @@ -165,22 +158,24 @@ public void validate(DocumentFieldMappers lookup) { throw new IllegalArgumentException("the configured timestamp field [" + path + "] doesn't have doc values"); } if (dateFieldMapper.getNullValue() != null) { - throw new IllegalArgumentException("the configured timestamp field [" + path + - "] has disallowed [null_value] attribute specified"); + throw new IllegalArgumentException( + "the configured timestamp field [" + path + "] has disallowed [null_value] attribute specified" + ); } if (dateFieldMapper.getIgnoreMalformed().explicit()) { - throw new IllegalArgumentException("the configured timestamp field [" + path + - "] has disallowed [ignore_malformed] attribute specified"); + throw new IllegalArgumentException( + "the configured timestamp field [" + path + "] has disallowed [ignore_malformed] attribute specified" + ); } // Catch all validation that validates whether disallowed mapping attributes have been specified // on the field this meta field refers to: try (XContentBuilder builder = jsonBuilder()) { builder.startObject(); - dateFieldMapper.doXContentBody(builder, false, EMPTY_PARAMS); + dateFieldMapper.toXContent(builder, EMPTY_PARAMS); builder.endObject(); - Map configuredSettings = - XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); + Map configuredSettings = XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); + configuredSettings = (Map) configuredSettings.values().iterator().next(); // Only type, meta and format attributes are allowed: configuredSettings.remove("type"); @@ -188,8 +183,9 @@ public void validate(DocumentFieldMappers lookup) { configuredSettings.remove("format"); // All other configured attributes are not allowed: if (configuredSettings.isEmpty() == false) { - throw new IllegalArgumentException("the configured timestamp field [@timestamp] has disallowed attributes: " + - configuredSettings.keySet()); + throw new IllegalArgumentException( + "the configured timestamp field [@timestamp] has disallowed attributes: " + configuredSettings.keySet() + ); } } catch (IOException e) { throw new UncheckedIOException(e); @@ -201,8 +197,7 @@ public String getPath() { } @Override - public void preParse(ParseContext context) throws IOException { - } + public void preParse(ParseContext context) throws IOException {} @Override protected void parseCreateField(ParseContext context) throws IOException { @@ -222,10 +217,9 @@ public void postParse(ParseContext context) throws IOException { throw new IllegalArgumentException("data stream timestamp field [" + path + "] is missing"); } - long numberOfValues = - Arrays.stream(fields) - .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) - .count(); + long numberOfValues = Arrays.stream(fields) + .filter(indexableField -> indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) + .count(); if (numberOfValues > 1) { throw new IllegalArgumentException("data stream timestamp field [" + path + "] encountered multiple values"); } @@ -259,9 +253,9 @@ protected boolean docValuesByDefault() { @Override protected void mergeOptions(FieldMapper other, List conflicts) { - TimestampFieldMapper otherTimestampFieldMapper = (TimestampFieldMapper) other; - if (Objects.equals(path, otherTimestampFieldMapper.path) == false) { - conflicts.add("cannot update path setting for [_timestamp]"); - } + DataStreamTimestampFieldMapper otherTimestampFieldMapper = (DataStreamTimestampFieldMapper) other; + if (Objects.equals(path, otherTimestampFieldMapper.path) == false) { + conflicts.add("cannot update path setting for [_data_stream_timestamp]"); + } } } diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java new file mode 100644 index 0000000000000..020f705425cb4 --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapperTests.java @@ -0,0 +1,357 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.datastreams.mapper; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.index.MapperTestUtils.assertConflicts; +import static org.hamcrest.Matchers.equalTo; + +public class DataStreamTimestampFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(DataStreamsPlugin.class); + } + + public void testPostParse() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", randomBoolean() ? "date" : "date_nanos") + .endObject() + .endObject() + .endObject() + .endObject() + ); + DocumentMapper docMapper = createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + ParsedDocument doc = docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("@timestamp", "2020-12-12").endObject()), + XContentType.JSON + ) + ); + assertThat(doc.rootDoc().getFields("@timestamp").length, equalTo(2)); + + Exception e = expectThrows( + MapperException.class, + () -> docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("@timestamp1", "2020-12-12").endObject()), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); + + e = expectThrows( + MapperException.class, + () -> docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes( + XContentFactory.jsonBuilder().startObject().array("@timestamp", "2020-12-12", "2020-12-13").endObject() + ), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] encountered multiple values")); + } + + public void testValidateNonExistingField() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "non-existing-field") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(e.getMessage(), equalTo("the configured timestamp field [non-existing-field] does not exist")); + } + + public void testValidateInvalidFieldType() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat( + e.getMessage(), + equalTo("the configured timestamp field [@timestamp] is of type [keyword], but [date,date_nanos] is expected") + ); + } + + public void testValidateNotIndexed() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .field("index", "false") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] is not indexed")); + } + + public void testValidateNotDocValues() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .field("doc_values", "false") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] doesn't have doc values")); + } + + public void testValidateNullValue() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .field("null_value", "2020-12-12") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] has disallowed [null_value] attribute specified")); + } + + public void testValidateIgnoreMalformed() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .field("ignore_malformed", "true") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat( + e.getMessage(), + equalTo("the configured timestamp field [@timestamp] has disallowed [ignore_malformed] attribute specified") + ); + } + + public void testValidateNotDisallowedAttribute() throws IOException { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("type") + .startObject("_data_stream_timestamp") + .field("path", "@timestamp") + .endObject() + .startObject("properties") + .startObject("@timestamp") + .field("type", "date") + .field("store", "true") + .endObject() + .endObject() + .endObject() + .endObject() + ); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE) + ); + assertThat(e.getMessage(), equalTo("the configured timestamp field [@timestamp] has disallowed attributes: [store]")); + } + + public void testCannotUpdateTimestampField() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping1 = + "{\"type\":{\"_data_stream_timestamp\":{\"path\":\"@timestamp\"}, \"properties\": {\"@timestamp\": {\"type\": \"date\"}}}}}"; + String mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"path\":\"@timestamp2\"}, \"properties\": {\"@timestamp2\": " + + "{\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; + assertConflicts(mapping1, mapping2, parser, "cannot update path setting for [_data_stream_timestamp]"); + + mapping1 = "{\"type\":{\"properties\":{\"@timestamp\": {\"type\": \"date\"}}}}}"; + mapping2 = "{\"type\":{\"_data_stream_timestamp\":{\"path\":\"@timestamp2\"}, \"properties\": " + + "{\"@timestamp2\": {\"type\": \"date\"},\"@timestamp\": {\"type\": \"date\"}}}})"; + assertConflicts(mapping1, mapping2, parser, "cannot update path setting for [_data_stream_timestamp]"); + } + + public void testDifferentTSField() throws IOException { + String mapping = "{\n" + + " \"_data_stream_timestamp\": {\n" + + " \"path\": \"event.my_timestamp\"\n" + + " },\n" + + " \"properties\": {\n" + + " \"event\": {\n" + + " \"properties\": {\n" + + " \"my_timestamp\": {\n" + + " \"type\": \"date\"" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + DocumentMapper docMapper = createIndex("test").mapperService() + .merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + + ParsedDocument doc = docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("event.my_timestamp", "2020-12-12").endObject()), + XContentType.JSON + ) + ); + assertThat(doc.rootDoc().getFields("event.my_timestamp").length, equalTo(2)); + + Exception e = expectThrows( + MapperException.class, + () -> docMapper.parse( + new SourceToParse( + "test", + "1", + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("event.timestamp", "2020-12-12").endObject()), + XContentType.JSON + ) + ) + ); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [event.my_timestamp] is missing")); + } + +} diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java new file mode 100644 index 0000000000000..21834b0528711 --- /dev/null +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.datastreams.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.cluster.DataStreamTestHelper.generateMapping; +import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; +import static org.hamcrest.Matchers.equalTo; + +public class MetadataCreateDataStreamServiceTests extends ESTestCase { + + public void testValidateTimestampFieldMapping() throws Exception { + String mapping = generateMapping("@timestamp", "date"); + validateTimestampFieldMapping("@timestamp", createMapperService(mapping)); + mapping = generateMapping("@timestamp", "date_nanos"); + validateTimestampFieldMapping("@timestamp", createMapperService(mapping)); + } + + public void testValidateTimestampFieldMappingNoFieldMapping() { + Exception e = expectThrows( + IllegalArgumentException.class, + () -> validateTimestampFieldMapping("@timestamp", createMapperService("{}")) + ); + assertThat( + e.getMessage(), + equalTo("[_data_stream_timestamp] meta field doesn't point to data stream timestamp field [@timestamp]") + ); + + String mapping = generateMapping("@timestamp2", "date"); + e = expectThrows(IllegalArgumentException.class, () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping))); + assertThat( + e.getMessage(), + equalTo("[_data_stream_timestamp] meta field doesn't point to data stream timestamp field [@timestamp]") + ); + } + + public void testValidateTimestampFieldMappingInvalidFieldType() { + String mapping = generateMapping("@timestamp", "keyword"); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> validateTimestampFieldMapping("@timestamp", createMapperService(mapping)) + ); + assertThat( + e.getMessage(), + equalTo("the configured timestamp field [@timestamp] is of type [keyword], " + "but [date,date_nanos] is expected") + ); + } + + MapperService createMapperService(String mapping) throws IOException { + String indexName = "test"; + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + ) + .putMapping(mapping) + .build(); + IndicesModule indicesModule = new IndicesModule(List.of(new DataStreamsPlugin())); + MapperService mapperService = MapperTestUtils.newMapperService( + xContentRegistry(), + createTempDir(), + Settings.EMPTY, + indicesModule, + indexName + ); + mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_UPDATE); + return mapperService; + } + +} diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index b89e1485b3b5c..ec53e67d0f119 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.Processor; @@ -83,6 +84,8 @@ public Processor create(Map processorFactories, Strin case EnrichPolicy.GEO_MATCH_TYPE: String relationStr = ConfigurationUtils.readStringProperty(TYPE, tag, config, "shape_relation", "intersects"); ShapeRelation shapeRelation = ShapeRelation.getRelationByName(relationStr); + String orientationStr = ConfigurationUtils.readStringProperty(TYPE, tag, config, "orientation", "CCW"); + ShapeBuilder.Orientation orientation = ShapeBuilder.Orientation.fromString(orientationStr); return new GeoMatchProcessor( tag, description, @@ -94,7 +97,8 @@ public Processor create(Map processorFactories, Strin ignoreMissing, matchField, maxMatches, - shapeRelation + shapeRelation, + orientation ); default: throw new IllegalArgumentException("unsupported policy type [" + policyType + "]"); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java index e882b05af4d09..c9df978ce2c40 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java @@ -8,23 +8,20 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.script.TemplateScript; -import java.util.ArrayList; -import java.util.List; import java.util.function.BiConsumer; public final class GeoMatchProcessor extends AbstractEnrichProcessor { - private ShapeRelation shapeRelation; + private final ShapeRelation shapeRelation; + private final GeometryParser parser; GeoMatchProcessor( String tag, @@ -37,10 +34,12 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { boolean ignoreMissing, String matchField, int maxMatches, - ShapeRelation shapeRelation + ShapeRelation shapeRelation, + ShapeBuilder.Orientation orientation ) { super(tag, description, client, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); this.shapeRelation = shapeRelation; + parser = new GeometryParser(orientation.getAsBoolean(), true, true); } /** used in tests **/ @@ -55,38 +54,17 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { boolean ignoreMissing, String matchField, int maxMatches, - ShapeRelation shapeRelation + ShapeRelation shapeRelation, + ShapeBuilder.Orientation orientation ) { super(tag, description, searchRunner, policyName, field, targetField, ignoreMissing, overrideEnabled, matchField, maxMatches); this.shapeRelation = shapeRelation; + parser = new GeometryParser(orientation.getAsBoolean(), true, true); } @Override public QueryBuilder getQueryBuilder(Object fieldValue) { - List points = new ArrayList<>(); - if (fieldValue instanceof List) { - List values = (List) fieldValue; - if (values.size() == 2 && values.get(0) instanceof Number) { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(values, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } else { - for (Object value : values) { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(value, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } - } - } else { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(fieldValue, true); - points.add(new Point(geoPoint.lon(), geoPoint.lat())); - } - final Geometry queryGeometry; - if (points.isEmpty()) { - throw new IllegalArgumentException("no geopoints found"); - } else if (points.size() == 1) { - queryGeometry = points.get(0); - } else { - queryGeometry = new MultiPoint(points); - } + final Geometry queryGeometry = parser.parseGeometry(fieldValue); GeoShapeQueryBuilder shapeQuery = new GeoShapeQueryBuilder(matchField, queryGeometry); shapeQuery.relation(shapeRelation); return shapeQuery; diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java index 1e87364c74068..4e46fe8f564ce 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java @@ -14,11 +14,15 @@ import org.elasticsearch.cluster.routing.Preference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.GeoShapeQueryBuilder; @@ -47,16 +51,41 @@ public class GeoMatchProcessorTests extends ESTestCase { public void testBasics() { + // point Point expectedPoint = new Point(-122.084110, 37.386637); testBasicsForFieldValue(Map.of("lat", 37.386637, "lon", -122.084110), expectedPoint); testBasicsForFieldValue("37.386637, -122.084110", expectedPoint); testBasicsForFieldValue("POINT (-122.084110 37.386637)", expectedPoint); testBasicsForFieldValue(List.of(-122.084110, 37.386637), expectedPoint); + testBasicsForFieldValue(Map.of("type", "Point", "coordinates", List.of(-122.084110, 37.386637)), expectedPoint); + // line + Line expectedLine = new Line(new double[] { 0, 1 }, new double[] { 0, 1 }); + testBasicsForFieldValue("LINESTRING(0 0, 1 1)", expectedLine); + testBasicsForFieldValue(Map.of("type", "LineString", "coordinates", List.of(List.of(0, 0), List.of(1, 1))), expectedLine); + // polygon + Polygon expectedPolygon = new Polygon(new LinearRing(new double[] { 0, 1, 1, 0, 0 }, new double[] { 0, 0, 1, 1, 0 })); + testBasicsForFieldValue("POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))", expectedPolygon); testBasicsForFieldValue( - List.of(List.of(-122.084110, 37.386637), "37.386637, -122.084110", "POINT (-122.084110 37.386637)"), - new MultiPoint(List.of(expectedPoint, expectedPoint, expectedPoint)) + Map.of( + "type", + "Polygon", + "coordinates", + List.of(List.of(List.of(0, 0), List.of(1, 0), List.of(1, 1), List.of(0, 1), List.of(0, 0))) + ), + expectedPolygon + ); + // geometry collection + testBasicsForFieldValue( + List.of( + List.of(-122.084110, 37.386637), + "37.386637, -122.084110", + "POINT (-122.084110 37.386637)", + Map.of("type", "Point", "coordinates", List.of(-122.084110, 37.386637)), + Map.of("type", "LineString", "coordinates", List.of(List.of(0, 0), List.of(1, 1))), + "POLYGON((0 0, 1 0, 1 1, 0 1, 0 0))" + ), + new GeometryCollection<>(List.of(expectedPoint, expectedPoint, expectedPoint, expectedPoint, expectedLine, expectedPolygon)) ); - testBasicsForFieldValue("not a point", null); } @@ -74,7 +103,8 @@ private void testBasicsForFieldValue(Object fieldValue, Geometry expectedGeometr false, "shape", maxMatches, - ShapeRelation.INTERSECTS + ShapeRelation.INTERSECTS, + ShapeBuilder.Orientation.CCW ); IngestDocument ingestDocument = new IngestDocument( "_index", diff --git a/x-pack/plugin/eql/build.gradle b/x-pack/plugin/eql/build.gradle index 3b1f5e96b7778..913902752d588 100644 --- a/x-pack/plugin/eql/build.gradle +++ b/x-pack/plugin/eql/build.gradle @@ -21,12 +21,6 @@ archivesBaseName = 'x-pack-eql' // All integration tests live in qa modules integTest.enabled = false -tasks.named('internalClusterTest').configure { - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.eql_feature_flag_registered', 'true' - } -} - dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') compileOnly(project(':modules:lang-painless')) { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java index 5e6450f9ac4ea..8d30b87601c7a 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java @@ -25,8 +25,11 @@ import org.junit.BeforeClass; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static java.util.stream.Collectors.toList; @@ -61,9 +64,10 @@ public void cleanup() throws Exception { public static List readTestSpecs() throws Exception { // Load EQL validation specs - List specs = EqlSpecLoader.load("/test_queries.toml", true); - specs.addAll(EqlSpecLoader.load("/additional_test_queries.toml", true)); - List unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false); + Set uniqueTestNames = new HashSet<>(); + List specs = EqlSpecLoader.load("/test_queries.toml", true, uniqueTestNames); + specs.addAll(EqlSpecLoader.load("/additional_test_queries.toml", true, uniqueTestNames)); + List unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false, uniqueTestNames); // Validate only currently supported specs List filteredSpecs = new ArrayList<>(); @@ -89,7 +93,7 @@ public static List readTestSpecs() throws Exception { private static List asArray(List specs) { AtomicInteger counter = new AtomicInteger(); return specs.stream().map(spec -> { - String name = spec.description(); + String name = spec.name(); if (Strings.isNullOrEmpty(name)) { name = spec.note(); } @@ -168,7 +172,10 @@ private RestHighLevelClient highLevelClient() { protected void assertSearchHits(List events) { assertNotNull(events); - assertArrayEquals("unexpected result for spec: [" + spec.toString() + "]", spec.expectedEventIds(), extractIds(events)); + long[] expected = spec.expectedEventIds(); + long[] actual = extractIds(events); + assertArrayEquals("unexpected result for spec: [" + spec.toString() + "]" + Arrays.toString(expected) + " vs " + Arrays.toString( + actual), expected, actual); } private static long[] extractIds(List events) { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java index 1b6306fcb2a5c..fc851a9f4fedd 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlRestTestCase.java @@ -30,11 +30,8 @@ public abstract class CommonEqlRestTestCase extends ESRestTestCase { {"{\"query\": \"\"}", "query is null or empty"}, {"{\"query\": \"" + validQuery + "\", \"timestamp_field\": \"\"}", "timestamp field is null or empty"}, {"{\"query\": \"" + validQuery + "\", \"event_category_field\": \"\"}", "event category field is null or empty"}, - {"{\"query\": \"" + validQuery + "\", \"implicit_join_key_field\": \"\"}", "implicit join key field is null or empty"}, {"{\"query\": \"" + validQuery + "\", \"size\": 0}", "size must be greater than 0"}, {"{\"query\": \"" + validQuery + "\", \"size\": -1}", "size must be greater than 0"}, - {"{\"query\": \"" + validQuery + "\", \"search_after\": null}", "search_after doesn't support values of type: VALUE_NULL"}, - {"{\"query\": \"" + validQuery + "\", \"search_after\": []}", "must contains at least one value"}, {"{\"query\": \"" + validQuery + "\", \"filter\": null}", "filter doesn't support values of type: VALUE_NULL"}, {"{\"query\": \"" + validQuery + "\", \"filter\": {}}", "query malformed, empty clause found"} }; diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java index bc24227d877d2..810a3cc51c9bf 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java @@ -12,6 +12,7 @@ import java.util.Objects; public class EqlSpec { + private String name; private String description; private String note; private String[] tags; @@ -24,6 +25,14 @@ public class EqlSpec { // FALSE -> case insensitive private Boolean caseSensitive = null; + public String name() { + return name; + } + + public void name(String name) { + this.name = name; + } + public String description() { return description; } @@ -76,6 +85,7 @@ public Boolean caseSensitive() { public String toString() { String str = ""; str = appendWithComma(str, "query", query); + str = appendWithComma(str, "name", name); str = appendWithComma(str, "description", description); str = appendWithComma(str, "note", note); diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java index 2132e631ea585..62877895bd5c6 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java @@ -9,26 +9,40 @@ import io.ous.jtoml.JToml; import io.ous.jtoml.Toml; import io.ous.jtoml.TomlTable; + import org.elasticsearch.common.Strings; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Set; public class EqlSpecLoader { - public static List load(String path, boolean supported) throws Exception { + public static List load(String path, boolean supported, Set uniqueTestNames) throws Exception { try (InputStream is = EqlSpecLoader.class.getResourceAsStream(path)) { - return readFromStream(is, supported); + return readFromStream(is, supported, uniqueTestNames); } } - private static void validateAndAddSpec(List specs, EqlSpec spec, boolean supported) throws Exception { + private static void validateAndAddSpec(List specs, EqlSpec spec, boolean supported, + Set uniqueTestNames) throws Exception { + if (Strings.isNullOrEmpty(spec.name())) { + throw new IllegalArgumentException("Read a test without a name value"); + } + if (Strings.isNullOrEmpty(spec.query())) { throw new IllegalArgumentException("Read a test without a query value"); } - if (supported && spec.expectedEventIds() == null) { - throw new IllegalArgumentException("Read a test without a expected_event_ids value"); + if (supported) { + if (spec.expectedEventIds() == null) { + throw new IllegalArgumentException("Read a test without a expected_event_ids value"); + } + if (uniqueTestNames.contains(spec.name())) { + throw new IllegalArgumentException("Found a test with the same name as another test: " + spec.name()); + } else { + uniqueTestNames.add(spec.name()); + } } specs.add(spec); @@ -42,7 +56,7 @@ private static String getTrimmedString(TomlTable table, String key) { return null; } - private static List readFromStream(InputStream is, boolean supported) throws Exception { + private static List readFromStream(InputStream is, boolean supported, Set uniqueTestNames) throws Exception { List testSpecs = new ArrayList<>(); EqlSpec spec; @@ -52,6 +66,7 @@ private static List readFromStream(InputStream is, boolean supported) t for (TomlTable table : queries) { spec = new EqlSpec(); spec.query(getTrimmedString(table, "query")); + spec.name(getTrimmedString(table, "name")); spec.note(getTrimmedString(table, "note")); spec.description(getTrimmedString(table, "description")); @@ -88,7 +103,7 @@ else if (Boolean.TRUE.equals(caseInsensitive)) { } spec.expectedEventIds(expectedEventIds); } - validateAndAddSpec(testSpecs, spec, supported); + validateAndAddSpec(testSpecs, spec, supported, uniqueTestNames); } return testSpecs; diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml index 879fd59f61b2c..e330d1c84b148 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml @@ -2,48 +2,56 @@ # test_queries.toml file in order to keep the original unchanged and easier to sync with the EQL reference implementation tests. [[queries]] +name = "betweenAdditional1" expected_event_ids = [95] query = ''' file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something" ''' [[queries]] +name = "betweenAdditional2" expected_event_ids = [95] query = ''' file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" ''' [[queries]] +name = "stringCidrMatch1" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "10.6.48.157/8") == true ''' [[queries]] +name = "stringCidrMatch2" expected_event_ids = [75304, 75305] query = ''' network where string(cidrMatch(source_address, "10.6.48.157/8")) == "true" ''' [[queries]] +name = "cidrMatchAdditional2" expected_event_ids = [75304, 75305] query = ''' network where true == cidrMatch(source_address, "10.6.48.157/8") ''' [[queries]] +name = "cidrMatchAdditional3" expected_event_ids = [] query = ''' network where cidrMatch(source_address, "192.168.0.0/16") == true ''' [[queries]] +name = "cidrMatchAdditional4" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") == true ''' [[queries]] +name = "cidrMatchAdditional5" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "0.0.0.0/0") == true @@ -51,6 +59,7 @@ network where cidrMatch(source_address, "0.0.0.0/0") == true [[queries]] +name = "concatEquals1" description = "test string concatenation. update test to avoid case-sensitivity issues" query = ''' process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3' @@ -59,60 +68,72 @@ expected_event_ids = [5] [[queries]] +name = "concatEquals2" query = 'process where concat(serial_event_id) = "1"' expected_event_ids = [1] [[queries]] +name = "concatWithCondition1" query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) != null' expected_event_ids = [2, 3] [[queries]] +name = "concatWithCondition2" query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) == null' expected_event_ids = [1, 4] [[queries]] +name = "concatWithCondition3" query = 'process where serial_event_id < 5 and concat(process_name, null, null) == null' expected_event_ids = [1, 2, 3, 4] [[queries]] +name = "concatWithCondition4" query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null' expected_event_ids = [1, 2, 3, 4] [[queries]] +name = "numberStringConversion1" query = 'process where string(serial_event_id) = "1"' expected_event_ids = [1] [[queries]] +name = "numberStringConversion2" query = 'any where number(string(serial_event_id)) == 17' expected_event_ids = [17] [[queries]] +name = "numberStringConversion3" query = 'any where number(string(serial_event_id), null) == 17' expected_event_ids = [17] [[queries]] +name = "numberStringConversion4" query = 'any where number(string(serial_event_id), 10) == 17' expected_event_ids = [17] [[queries]] +name = "numberStringEquality" query = 'any where number(string(serial_event_id), 13) == number("31", 13)' expected_event_ids = [31] [[queries]] +name = "numberStringConversion5" query = 'any where number(string(serial_event_id), 16) == 17' expected_event_ids = [11] [[queries]] +name = "matchWithCharacterClasses1" expected_event_ids = [98] notes = "regexp doesn't support character classes" query = ''' @@ -122,12 +143,14 @@ process where match(command_line, ?'.*?net1[ ]+localgroup.*?') ''' [[queries]] +name = "matchLiteAdditional" expected_event_ids = [98] query = ''' process where matchLite(command_line, ?'.*?net1.*?') ''' [[queries]] +name = "matchWithCharacterClasses2" expected_event_ids = [98] notes = "regexp doesn't support predefined character classes (like \\s)" query = ''' @@ -138,18 +161,21 @@ process where match(command_line, ?'.*?net1[ ]+[a-z]{4,15}[ ]+.*?') [[queries]] +name = "multiPatternMatch" expected_event_ids = [50, 97, 98] query = ''' process where match(command_line, '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') ''' [[queries]] +name = "matchWithSubstring" expected_event_ids = [50, 98] query = ''' process where match(substring(command_line, 5), '.*?net[1]? localgroup.*?', '.*? myappserver.py .*?') ''' [[queries]] +name = "moduloEqualsField" # Basic test for modulo function query = ''' process where modulo(11, 10) == serial_event_id''' @@ -157,36 +183,44 @@ expected_event_ids = [1] description = "test built-in modulo math functions" [[queries]] +name = "additionalMathWithFields1" # This query give a different result with ES EQL implementation because it doesn't convert to float data types for division expected_event_ids = [82, 83] query = "file where serial_event_id / 2 == 41" # Additional EQL queries with arithmetic operations that were not part of the original EQL implementation [[queries]] +name = "additionalMathWithFields2" expected_event_ids = [82] query = "file where 83 - serial_event_id == 1" [[queries]] +name = "additionalMathWithFields3" expected_event_ids = [82] query = "file where 1 + serial_event_id == 83" [[queries]] +name = "additionalMathWithFields4" expected_event_ids = [82] query = "file where -serial_event_id + 100 == 18" [[queries]] +name = "additionalMathWithFields5" expected_event_ids = [82] query = "file where 2 * serial_event_id == 164" [[queries]] +name = "additionalMathWithFields6" expected_event_ids = [66] query = "file where 66.0 / serial_event_id == 1" [[queries]] +name = "additionalMathWithFields7" expected_event_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 46] query = "process where serial_event_id + ((1 + 3) * 2 / (3 - 1)) * 2 == 54 or 70 + serial_event_id < 100" [[queries]] +name = "twoSequencesAdditional1" query = ''' sequence [process where serial_event_id = 1] @@ -195,6 +229,7 @@ sequence expected_event_ids = [1, 2] [[queries]] +name = "twoSequencesAdditional2" query = ''' sequence [process where serial_event_id=1] by unique_pid @@ -202,6 +237,7 @@ sequence expected_event_ids = [1, 2] [[queries]] +name = "twoSequencesAdditional3" query = ''' sequence [process where serial_event_id<3] by unique_pid @@ -210,6 +246,7 @@ sequence expected_event_ids = [1, 2, 2, 3] [[queries]] +name = "twoSequencesAdditional4" query = ''' sequence [process where false] by unique_pid diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml index c01a2fae3c562..952dbae710e52 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml @@ -1,34 +1,42 @@ [[queries]] +name = "simpleQueryEqual" query = 'process where serial_event_id = 1' expected_event_ids = [1] [[queries]] +name = "simpleQueryLt" query = 'process where serial_event_id < 4' expected_event_ids = [1, 2, 3] [[queries]] +name = "simpleQueryHeadSix" query = 'process where true | head 6' expected_event_ids = [1, 2, 3, 4, 5, 6] [[queries]] +name = "processWhereFalse" query = 'process where false' expected_event_ids = [] [[queries]] +name = "missingField1" expected_event_ids = [] query = 'process where missing_field != null' [[queries]] +name = "equalsNullHead" expected_event_ids = [1, 2, 3, 4, 5] query = 'process where bad_field == null | head 5' [[queries]] +name = "processNameInexistent" query = ''' process where process_name == "impossible name" or (serial_event_id < 4.5 and serial_event_id >= 3.1) ''' expected_event_ids = [4] [[queries]] +name = "lteAndGtWithFilter" tags = ["comparisons", "pipes"] query = ''' process where serial_event_id <= 8 and serial_event_id > 7 @@ -37,6 +45,7 @@ process where serial_event_id <= 8 and serial_event_id > 7 expected_event_ids = [8] [[queries]] +name = "filtersLteAndGt" query = ''' process where true | filter serial_event_id <= 10 @@ -45,6 +54,7 @@ process where true expected_event_ids = [7, 8, 9, 10] [[queries]] +name = "filterLteAndGtWithHead" query = ''' process where true | filter serial_event_id <= 10 @@ -54,6 +64,7 @@ process where true expected_event_ids = [7, 8] [[queries]] +name = "headWithFiltersAndTail" query = ''' process where true | head 1000 @@ -64,42 +75,50 @@ process where true expected_event_ids = [9, 10] [[queries]] +name = "serialEventIdLteAndGt" query = ''' process where serial_event_id<=8 and serial_event_id > 7 ''' expected_event_ids = [8] [[queries]] +name = "exitCodeGteZero" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code >= 0' [[queries]] +name = "zeroLteExitCode" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where 0 <= exit_code' [[queries]] +name = "exitCodeLteZero" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code <= 0' [[queries]] +name = "exitCodeLtOne" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code < 1' [[queries]] +name = "exitCodeGtMinusOne" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code > -1' [[queries]] +name = "minusOneLtExitCode" note = "check that comparisons against null values return null" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where -1 < exit_code' [[queries]] +name = "nullEqualityAndInWithHead" note = "check that comparisons against null values return null" expected_event_ids = [] query = ''' @@ -109,106 +128,129 @@ process where null == (exit_code > -1) ''' [[queries]] +name = "nullEqualityWithGtAndHead" note = "check that comparisons against null values return null" expected_event_ids = [1, 2, 3, 4, 5, 6, 7] query = 'process where null == (exit_code > -1) | head 7' [[queries]] +name = "nullEqualityWithLtAndHead" note = "check that comparisons against null values return null" expected_event_ids = [1, 2, 3, 4, 5, 6, 7] query = 'process where null == (-1 < exit_code) | head 7' [[queries]] +name = "exitCodeGtZero" query = 'process where exit_code > 0' expected_event_ids = [] [[queries]] +name = "exitCodeLtZero" query = 'process where exit_code < 0' expected_event_ids = [] [[queries]] +name = "zeroLtExitCode" query = 'process where 0 < exit_code' expected_event_ids = [] [[queries]] +name = "zeroGtExitCode" query = 'process where 0 > exit_code' expected_event_ids = [] [[queries]] +name = "processWithMultipleConditions1" query = 'process where (serial_event_id<=8 and serial_event_id > 7) and (opcode=3 and opcode>2)' expected_event_ids = [8] [[queries]] +name = "processWithMultipleConditions2" query = 'process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)' expected_event_ids = [7, 8] [[queries]] +name = "processWithStringComparison1" query = 'process where process_name >= "System" and process_name <= "System Idle Process"' expected_event_ids = [1, 2] [[queries]] +name = "compareTwoFields1" # test that it works for comparing field <--> field case_insensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2, 3] [[queries]] +name = "compareTwoFields2" case_sensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2] [[queries]] +name = "processWithStringComparison2" query = 'process where process_name >= "System" and process_name < "System Idle Process"' expected_event_ids = [2] [[queries]] +name = "processWithStringComparison3" query = 'process where process_name > "System" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] +name = "processWithStringComparison4" query = 'process where process_name > "System" and process_name < "System Idle Process"' expected_event_ids = [] [[queries]] +name = "processWithStringComparison5" query = 'process where process_name >= "Syste" and process_name <= "Systez"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringComparison6" query = 'process where process_name > "System" and process_name < "Systez"' expected_event_ids = [1] [[queries]] +name = "processWithStringComparison7" query = 'process where process_name >= "System Idle Process" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] +name = "processWithStringComparisonCaseSensitive1" case_sensitive = true notes = "s == 0x73; S == 0x53" query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [] [[queries]] +name = "processWithStringComparisonCaseInsensitive1" case_insensitive = true query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] +name = "processWithStringComparisonCaseInsensitive2" case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name < "systex"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringComparisonCaseInsensitive3" case_insensitive = true query = 'process where process_name >= "sysT" and process_name < "SYsTeZZZ"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringComparisonCaseInsensitive4" case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name <= "systex"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringEqualityCaseInsensitive1" case_insensitive = true query = ''' process where process_name == "VMACTHLP.exe" and unique_pid == 12 @@ -217,6 +259,7 @@ process where process_name == "VMACTHLP.exe" and unique_pid == 12 expected_event_ids = [12] [[queries]] +name = "processNameIN" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -224,6 +267,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINCaseInsensitive1" case_insensitive = true query = ''' process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") @@ -232,6 +276,7 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINCaseInsensitive2" case_insensitive = true query = ''' process where process_name in ("python.exe", "smss.exe", "Explorer.exe") @@ -240,6 +285,7 @@ process where process_name in ("python.exe", "smss.exe", "Explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINWithUnique1" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique length(process_name) == length("python.exe") @@ -247,6 +293,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 48] [[queries]] +name = "processNameINWithUnique2" case_insensitive = true query = ''' process where process_name in ("Python.exe", "smss.exe", "explorer.exe") @@ -255,6 +302,7 @@ process where process_name in ("Python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 48] [[queries]] +name = "processNameINWithUniqueHeadAndTail1" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -264,6 +312,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [34] [[queries]] +name = "processNameINWithUniqueHeadAndTail2" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -273,6 +322,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [34] [[queries]] +name = "processNameINWithUnique3" query = ''' process where process_name in ("python.exe", "smss.exe") | unique process_name parent_process_name @@ -280,6 +330,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54, 78] [[queries]] +name = "processNameINWithTwoUniqueFields1" query = ''' process where process_name in ("python.exe", "smss.exe") | unique process_name, parent_process_name @@ -287,6 +338,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54, 78] [[queries]] +name = "processNameINWithTwoUniqueFields2" query = ''' process where process_name in ("python.exe", "smss.exe") | head 5 @@ -295,6 +347,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54] [[queries]] +name = "lengthCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -302,18 +355,21 @@ registry where length(bytes_written_string_list) == 2 and bytes_written_string_l ''' [[queries]] +name = "keyPathWildcard" query = ''' registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F" ''' expected_event_ids = [79] [[queries]] +name = "processPathWildcardAndIN" query = ''' process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4) ''' expected_event_ids = [84, 85] [[queries]] +name = "descendant1" query = ''' file where file_name == "csrss.exe" and opcode=0 and descendant of [process where opcode in (1,3) and process_name="cmd.exe"] @@ -321,6 +377,7 @@ file where file_name == "csrss.exe" and opcode=0 expected_event_ids = [72] [[queries]] +name = "descendant2" query = ''' process where opcode=1 and process_name == "csrss.exe" and descendant of [file where file_name == "csrss.exe" and opcode=0] @@ -328,6 +385,7 @@ process where opcode=1 and process_name == "csrss.exe" expected_event_ids = [73] [[queries]] +name = "descendant3" query = ''' process where opcode=1 and process_name == "smss.exe" and descendant of [ @@ -340,6 +398,7 @@ process where opcode=1 and process_name == "smss.exe" expected_event_ids = [78] [[queries]] +name = "wildcardAndMultipleConditions1" query = ''' file where file_path="*\\red_ttp\\winin*.*" and opcode in (0,1,2) and user_name="vagrant" @@ -347,6 +406,7 @@ file where file_path="*\\red_ttp\\winin*.*" expected_event_ids = [83, 86] [[queries]] +name = "wildcardAndMultipleConditions2" query = ''' file where file_path="*\\red_ttp\\winin*.*" and opcode not in (0,1,2) and user_name="vagrant" @@ -354,6 +414,7 @@ file where file_path="*\\red_ttp\\winin*.*" expected_event_ids = [] [[queries]] +name = "wildcardAndMultipleConditions3" query = ''' file where file_path="*\\red_ttp\\winin*.*" and opcode not in (3, 4, 5, 6 ,7) and user_name="vagrant" @@ -362,12 +423,14 @@ expected_event_ids = [83, 86] [[queries]] +name = "INWithCondition" query = ''' file where file_name in ("wininit.exe", "lsass.exe") and opcode == 2 ''' expected_event_ids = [65, 86] [[queries]] +name = "simpleTail" query = ''' file where true | tail 3 @@ -375,12 +438,14 @@ file where true expected_event_ids = [92, 95, 96] [[queries]] +name = "twoINs" query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "System") ''' expected_event_ids = [2, 50, 51] [[queries]] +name = "twoINsCaseInsensitive" case_insensitive = true query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM") @@ -388,6 +453,7 @@ process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM" expected_event_ids = [2, 50, 51] [[queries]] +name = "simpleTailWithSort" case_insensitive = true expected_event_ids = [92, 95, 96, 91] query = ''' @@ -397,6 +463,7 @@ file where true ''' [[queries]] +name = "simpleHeadWithSort1" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -405,6 +472,7 @@ process where true ''' [[queries]] +name = "simpleHeadwithSort2" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -413,6 +481,7 @@ process where true ''' [[queries]] +name = "simpleHeadwithSort3" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -421,6 +490,7 @@ process where true ''' [[queries]] +name = "twoHeadsWithSort" expected_event_ids = [2, 1] query = ''' process where true @@ -430,6 +500,7 @@ process where true ''' [[queries]] +name = "twoSortsWithHead" expected_event_ids = [1, 2, 3, 4, 5] query = ''' process where true @@ -439,7 +510,7 @@ process where true ''' [[queries]] -note = "Sequence: 1-1 match" +name = "sequenceOneOneMatch" query = ''' sequence [process where serial_event_id = 1] @@ -448,7 +519,7 @@ sequence expected_event_ids = [1, 2] [[queries]] -note = "Sequence: many-1 match." +name = "sequenceManyOneMatch" query = ''' sequence [process where serial_event_id < 5] @@ -457,7 +528,7 @@ sequence expected_event_ids = [4, 5] [[queries]] -note = "Sequence: 1-1-1." +name = "sequenceOneOneOne" query = ''' sequence [process where serial_event_id == 1] @@ -468,7 +539,7 @@ expected_event_ids = [1, 2, 3] [[queries]] -note = "Sequence: 1-many-many." +name = "sequenceOneManyMany" query = ''' sequence [process where serial_event_id == 1] @@ -478,6 +549,7 @@ sequence expected_event_ids = [1, 2, 3] [[queries]] +name = "sequenceConditionManyMany" query = ''' sequence [process where serial_event_id <= 3] @@ -488,6 +560,7 @@ expected_event_ids = [1, 2, 3, 2, 3, 4, 3, 4, 5] [[queries]] +name = "sequenceManyConditionMany" query = ''' sequence [process where true] @@ -497,6 +570,7 @@ sequence expected_event_ids = [1, 2, 3, 2, 3, 4] [[queries]] +name = "sequenceManyManyCondition" query = ''' sequence [process where true] @@ -506,6 +580,7 @@ sequence expected_event_ids = [1, 2, 3] [[queries]] +name = "sequenceThreeManyCondition1" query = ''' sequence [process where serial_event_id <= 4] @@ -519,6 +594,7 @@ expected_event_ids = [1, 2, 3, 4, 4, 5, 6, 7] [[queries]] +name = "sequenceThreeManyCondition2" query = ''' sequence [process where true] @@ -531,6 +607,7 @@ expected_event_ids = [1, 2, 3, 4, 3, 4, 5, 6] [[queries]] +name = "sequenceThreeManyCondition3" query = ''' sequence [process where true] @@ -542,6 +619,7 @@ expected_event_ids = [1, 2, 3, 4, 2, 3, 4, 5] [[queries]] +name = "sequenceThreeManyCondition4" query = ''' sequence [process where true] @@ -552,6 +630,7 @@ sequence expected_event_ids = [1, 2, 3, 4] [[queries]] +name = "twoSequencesWithKeys" query = ''' sequence [process where true] by unique_pid @@ -563,6 +642,7 @@ expected_event_ids = [48, 53, 97, 98] [[queries]] +name = "twoSequencesWithTwoKeys" query = ''' sequence [process where true] by unique_pid, process_path @@ -574,6 +654,7 @@ expected_event_ids = [48, 53, 97, 98] [[queries]] +name = "fourSequencesByPidWithUntil1" query = ''' sequence [process where opcode == 1] by unique_pid @@ -586,6 +667,7 @@ until expected_event_ids = [] [[queries]] +name = "fourSequencesByPidWithUntil2" query = ''' sequence [process where opcode == 1] by unique_pid @@ -598,6 +680,7 @@ until expected_event_ids = [54, 55, 61, 67] [[queries]] +name = "fourSequencesByPid" query = ''' sequence [process where opcode == 1] by unique_pid @@ -609,6 +692,7 @@ expected_event_ids = [54, 55, 61, 67] [[queries]] +name = "fourSequencesByPidAndProcessPath1" query = ''' sequence [process where opcode == 1] by unique_pid, process_path @@ -620,6 +704,7 @@ expected_event_ids = [54, 55, 61, 67] [[queries]] +name = "fourSequencesByPidAndProcessPathWithUntil" query = ''' sequence [process where opcode == 1] by unique_pid, process_path @@ -632,7 +717,7 @@ until expected_event_ids = [54, 55, 61, 67] [[queries]] -note = "Sequence: 1-many match with join." +name = "sequenceOneManyWithJoin" query = ''' sequence [process where serial_event_id=1] by unique_pid @@ -642,7 +727,7 @@ expected_event_ids = [1, 2] [[queries]] -note = "Sequence: many-many match with join." +name = "sequenceManyManyWithJoin" query = ''' sequence [process where serial_event_id<3] by unique_pid @@ -652,7 +737,7 @@ expected_event_ids = [1, 2, 2, 3] [[queries]] -note = "Sequence: non-field based join." +name = "sequenceNonFieldBasedJoin" query = ''' sequence [process where serial_event_id<3] by unique_pid * 2 @@ -663,7 +748,7 @@ expected_event_ids = [1, 2, [[queries]] -note = "Sequence: multiple non-field based joins." +name = "sequenceMultipleNonFieldBasedJoins" query = ''' sequence [process where serial_event_id<3] by unique_pid * 2, length(unique_pid), string(unique_pid) @@ -674,6 +759,7 @@ expected_event_ids = [1, 2, [[queries]] +name = "sequencesOnDifferentEventTypes1" query = ''' sequence by unique_pid [process where opcode=1 and process_name == 'MSBuild.exe'] @@ -684,6 +770,7 @@ description = "test that process sequences are working correctly" [[queries]] +name = "sequencesOnDifferentEventTypes2" query = ''' sequence [file where event_subtype_full == "file_create_event"] by file_path @@ -697,6 +784,7 @@ expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] [[queries]] +name = "sequencesOnDifferentEventTypes3" query = ''' sequence with maxspan=1d [file where event_subtype_full == "file_create_event"] by file_path @@ -710,6 +798,7 @@ expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] [[queries]] +name = "sequencesOnDifferentEventTypes4" query = ''' sequence with maxspan=1h [file where event_subtype_full == "file_create_event"] by file_path @@ -723,6 +812,7 @@ expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] [[queries]] +name = "sequencesOnDifferentEventTypes5" query = ''' sequence with maxspan=1m [file where event_subtype_full == "file_create_event"] by file_path @@ -736,6 +826,7 @@ expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] [[queries]] +name = "sequencesOnDifferentEventTypes6" query = ''' sequence with maxspan=10s [file where event_subtype_full == "file_create_event"] by file_path @@ -749,6 +840,7 @@ expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] [[queries]] +name = "sequencesOnDifferentEventTypes7" query = ''' sequence with maxspan=500ms [file where event_subtype_full == "file_create_event"] by file_path @@ -761,6 +853,7 @@ sequence with maxspan=500ms expected_event_ids = [] [[queries]] +name = "doubleSameSequence" query = ''' sequence [process where serial_event_id < 5] @@ -771,6 +864,7 @@ expected_event_ids = [1, 2, 3, 4] [[queries]] +name = "sequencesOnDifferentEventTypesWithBy" query = ''' sequence [file where opcode=0 and file_name="svchost.exe"] by unique_pid @@ -779,6 +873,7 @@ sequence expected_event_ids = [55, 56] [[queries]] +name = "doubleSameSequenceWithBy" query = ''' sequence [file where opcode=0] by unique_pid @@ -788,6 +883,7 @@ sequence expected_event_ids = [55, 61] [[queries]] +name = "doubleSameSequenceWithByAndFilter" query = ''' sequence [file where opcode=0] by unique_pid @@ -797,6 +893,7 @@ sequence expected_event_ids = [87, 92] [[queries]] +name = "doubleSameSequenceWithByUntilAndHead1" query = ''' sequence [file where opcode=0 and file_name="*.exe"] by unique_pid @@ -807,6 +904,7 @@ until [process where opcode=5000] by unique_ppid expected_event_ids = [55, 61] [[queries]] +name = "doubleSameSequenceWithByUntilAndHead2" query = ''' sequence [file where opcode=0 and file_name="*.exe"] by unique_pid @@ -817,6 +915,7 @@ until [process where opcode=1] by unique_ppid expected_event_ids = [] [[queries]] +name = "doubleJoinWithByUntilAndHead" query = ''' join [file where opcode=0 and file_name="*.exe"] by unique_pid @@ -827,6 +926,7 @@ until [process where opcode=1] by unique_ppid expected_event_ids = [61, 59] [[queries]] +name = "twoJoins1" query = ''' join by user_name [process where opcode in (1,3) and process_name="smss.exe"] @@ -835,6 +935,7 @@ join by user_name expected_event_ids = [78, 48] [[queries]] +name = "threeJoins1" query = ''' join by unique_pid [process where opcode=1] @@ -844,6 +945,7 @@ join by unique_pid expected_event_ids = [54, 55, 61] [[queries]] +name = "threeJoins2" query = ''' join by string(unique_pid) [process where opcode=1] @@ -853,6 +955,7 @@ join by string(unique_pid) expected_event_ids = [54, 55, 61] [[queries]] +name = "threeJoinsWithUntil1" query = ''' join by unique_pid [process where opcode=1] @@ -863,6 +966,7 @@ until [file where opcode == 2] expected_event_ids = [] [[queries]] +name = "threeJoinsWithUntil2" query = ''' join by string(unique_pid), unique_pid, unique_pid * 2 [process where opcode=1] @@ -873,6 +977,7 @@ until [file where opcode == 2] expected_event_ids = [] [[queries]] +name = "twoJoins2" query = ''' join [file where opcode=0 and file_name="svchost.exe"] by unique_pid @@ -881,6 +986,7 @@ join expected_event_ids = [55, 56] [[queries]] +name = "twoJoins3" query = ''' join by unique_pid [process where opcode in (1,3) and process_name="python.exe"] @@ -889,6 +995,7 @@ join by unique_pid expected_event_ids = [54, 55] [[queries]] +name = "twoJoins4" query = ''' join by user_name [process where opcode in (1,3) and process_name="python.exe"] @@ -897,6 +1004,7 @@ join by user_name expected_event_ids = [48, 78] [[queries]] +name = "twoJoins5" query = ''' join [process where opcode in (1,3) and process_name="python.exe"] @@ -905,12 +1013,14 @@ join expected_event_ids = [48, 3, 50, 78] [[queries]] +name = "fakeField" expected_event_ids = [] query = ''' process where fake_field == "*" ''' [[queries]] +name = "fakeFieldWithHead1" query = ''' process where fake_field != "*" | head 4 @@ -921,6 +1031,7 @@ expected_event_ids = [] [[queries]] +name = "fakeFieldWithHead2" query = ''' process where not (fake_field == "*") | head 4 @@ -930,18 +1041,21 @@ process where not (fake_field == "*") expected_event_ids = [] [[queries]] +name = "invalidFieldName" expected_event_ids = [] query = ''' registry where invalid_field_name != null ''' [[queries]] +name = "badFieldWithLength" expected_event_ids = [] query = ''' registry where length(bad_field) > 0 ''' [[queries]] +name = "multipleConditions2" query = ''' process where opcode == 1 and process_name in ("net.exe", "net1.exe") @@ -952,6 +1066,7 @@ process where opcode == 1 expected_event_ids = [97] [[queries]] +name = "anyWithUnique" expected_event_ids = [1, 55, 57, 63, 75304] query = ''' any where true @@ -959,6 +1074,7 @@ any where true ''' [[queries]] +name = "multipleConditionsWithDescendant1" query = ''' process where opcode=1 and process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -966,6 +1082,7 @@ process where opcode=1 and process_name in ("services.exe", "smss.exe", "lsass.e expected_event_ids = [62, 68, 78] [[queries]] +name = "INWithDescendant" query = ''' process where process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -973,6 +1090,7 @@ process where process_name in ("services.exe", "smss.exe", "lsass.exe") expected_event_ids = [62, 64, 68, 69, 78, 80] [[queries]] +name = "multipleConditionsWithDescendant2" query = ''' process where opcode=2 and process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -980,6 +1098,7 @@ process where opcode=2 and process_name in ("services.exe", "smss.exe", "lsass.e expected_event_ids = [64, 69, 80] [[queries]] +name = "childOf1" query = ''' process where process_name="svchost.exe" and child of [file where file_name="svchost.exe" and opcode=0] @@ -987,6 +1106,7 @@ process where process_name="svchost.exe" expected_event_ids = [56, 58] [[queries]] +name = "childOf2" query = ''' process where process_name="svchost.exe" and not child of [file where file_name="svchost.exe" and opcode=0] @@ -995,6 +1115,7 @@ process where process_name="svchost.exe" expected_event_ids = [11, 13, 15] [[queries]] +name = "nestedChildOf1" query = ''' process where process_name="lsass.exe" and child of [ @@ -1005,6 +1126,7 @@ process where process_name="lsass.exe" expected_event_ids = [62, 64] [[queries]] +name = "nestedChildOf2" query = ''' file where child of [ process where child of [ @@ -1016,6 +1138,7 @@ file where child of [ expected_event_ids = [91] [[queries]] +name = "fileByUniquePid1" query = ''' file where process_name = "python.exe" | unique unique_pid @@ -1023,6 +1146,7 @@ file where process_name = "python.exe" expected_event_ids = [55, 95] [[queries]] +name = "fileByUniquePid2" query = ''' file where event of [process where process_name = "python.exe" ] | unique unique_pid @@ -1030,16 +1154,19 @@ file where event of [process where process_name = "python.exe" ] expected_event_ids = [55, 95] [[queries]] +name = "simpleStringEquality" query = ''' process where process_name = "python.exe" ''' expected_event_ids = [48, 50, 51, 54, 93] [[queries]] +name = "eventOfProcess" query = 'process where event of [process where process_name = "python.exe" ]' expected_event_ids = [48, 50, 51, 54, 93] [[queries]] +name = "twoSequencesWithKeys2" query = ''' sequence [file where file_name="lsass.exe"] by file_path,process_path @@ -1048,6 +1175,7 @@ sequence expected_event_ids = [61, 62] [[queries]] +name = "twoSequencesWithKeys3" query = ''' sequence by user_name [file where file_name="lsass.exe"] by file_path, process_path @@ -1056,6 +1184,7 @@ sequence by user_name expected_event_ids = [61, 62] [[queries]] +name = "twoSequencesWithKeys4" query = ''' sequence by pid [file where file_name="lsass.exe"] by file_path,process_path @@ -1064,6 +1193,7 @@ sequence by pid expected_event_ids = [] [[queries]] +name = "fourSequencesByMixedFields" query = ''' sequence by user_name [file where opcode=0] by file_path @@ -1075,6 +1205,7 @@ sequence by user_name expected_event_ids = [88, 89, 90, 91] [[queries]] +name = "twoSequencesWithTwoKeysAndUntil" query = ''' sequence by user_name [file where opcode=0] by pid,file_path @@ -1085,6 +1216,7 @@ until expected_event_ids = [] [[queries]] +name = "twoSequencesWithUntil" query = ''' sequence by user_name [file where opcode=0] by pid,file_path @@ -1096,6 +1228,7 @@ until expected_event_ids = [55, 59, 61, 65] [[queries]] +name = "fourSequencesWithTail" query = ''' sequence by pid [file where opcode=0] by file_path @@ -1107,6 +1240,7 @@ sequence by pid expected_event_ids = [] [[queries]] +name = "twoSequencesWithHead" query = ''' join by user_name [file where true] by pid,file_path @@ -1116,6 +1250,7 @@ join by user_name expected_event_ids = [55, 56, 59, 58] [[queries]] +name = "threeSequencesWithHead" query = ''' sequence [process where true] by unique_pid @@ -1126,24 +1261,28 @@ sequence expected_event_ids = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73] [[queries]] +name = "wildcard1" query = ''' process where command_line == "*%*" ''' expected_event_ids = [4, 6, 28] [[queries]] +name = "wildcard2" query = ''' process where command_line == "*%*%*" ''' expected_event_ids = [4, 6, 28] [[queries]] +name = "wildcard3" query = ''' process where command_line == "%*%*" ''' expected_event_ids = [4, 6, 28] [[queries]] +name = "uniqueCount1" expected_event_ids = [11, 60, 63] query = ''' any where process_name == "svchost.exe" @@ -1151,6 +1290,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCount2" expected_event_ids = [63, 60, 11] query = ''' any where process_name == "svchost.exe" @@ -1159,6 +1299,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCount3" expected_event_ids = [60] query = ''' any where process_name == "svchost.exe" @@ -1167,6 +1308,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCountAndFilter" expected_event_ids = [11] query = ''' any where process_name == "svchost.exe" @@ -1176,6 +1318,7 @@ any where process_name == "svchost.exe" [[queries]] +name = "arrayContainsCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1183,6 +1326,7 @@ registry where arrayContains(bytes_written_string_list, 'En-uS') ''' [[queries]] +name = "arrayContainsCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1190,6 +1334,7 @@ registry where arrayContains(bytes_written_string_list, 'En') ''' [[queries]] +name = "lengthCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1197,6 +1342,7 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li ''' [[queries]] +name = "arrayCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1204,6 +1350,7 @@ registry where bytes_written_string_list[0] == 'EN-us' ''' [[queries]] +name = "arrayCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1212,6 +1359,7 @@ registry where bytes_written_string_list[1] == 'EN' [[queries]] +name = "arrayContainsCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1219,6 +1367,7 @@ registry where arrayContains(bytes_written_string_list, 'en-US') ''' [[queries]] +name = "arrayContainsCaseInsensitive4" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1226,6 +1375,7 @@ registry where arrayContains(bytes_written_string_list, 'en') ''' [[queries]] +name = "arrayCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1233,48 +1383,56 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li ''' [[queries]] +name = "arrayEquality1" expected_event_ids = [57] query = ''' registry where bytes_written_string_list[0] == 'en-US' ''' [[queries]] +name = "arrayEquality2" expected_event_ids = [57] query = ''' registry where bytes_written_string_list[1] == 'en' ''' [[queries]] +name = "matchLite1" query = ''' process where matchLite(command_line, ?'.*?net1\s+localgroup\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "matchLite2" query = ''' process where matchLite(command_line, ?'.*?net1\s+\w+\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "matchLite3" query = ''' process where matchLite(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "match1" expected_event_ids = [98] query = ''' process where match(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') ''' [[queries]] +name = "matchLite4" query = ''' process where matchLite(command_line, ?'.*?net1\s+[localgrup]{4,15}\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "stringEqualsCaseInsensitive1" case_insensitive = true query = ''' process where 'net.EXE' == original_file_name @@ -1284,6 +1442,7 @@ expected_event_ids = [97] note = "check that case insensitive comparisons are performed even for lhs strings." [[queries]] +name = "stringEqualsCaseInsensitive2" case_insensitive = true query = ''' process where process_name == original_file_name and process_name='net*.exe' @@ -1292,6 +1451,7 @@ expected_event_ids = [97, 98] note = "check that case insensitive comparisons are performed for fields." [[queries]] +name = "fieldsComparisonCaseInsensitive" case_insensitive = true query = ''' process where original_file_name == process_name and length(original_file_name) > 0 @@ -1300,6 +1460,7 @@ expected_event_ids = [97, 98, 75273, 75303] description = "check that case insensitive comparisons are performed for fields." [[queries]] +name = "startsWithCaseSensitive" case_sensitive = true query = ''' file where opcode=0 and startsWith(file_name, 'explorer.') @@ -1309,6 +1470,7 @@ description = "check built-in string functions" [[queries]] +name = "startsWithCaseInsensitive1" case_insensitive = true query = ''' file where opcode=0 and startsWith(file_name, 'explorer.') @@ -1318,6 +1480,7 @@ description = "check built-in string functions" [[queries]] +name = "startsWithCaseInsensitive2" case_insensitive = true query = ''' file where opcode=0 and startsWith(file_name, 'exploRER.') @@ -1326,6 +1489,7 @@ expected_event_ids = [88, 92] description = "check built-in string functions" [[queries]] +name = "startsWithCaseInsensitive3" case_insensitive = true query = ''' file where opcode=0 and startsWith(file_name, 'expLORER.exe') @@ -1334,6 +1498,7 @@ expected_event_ids = [88, 92] description = "check built-in string functions" [[queries]] +name = "endsWith1" query = ''' file where opcode=0 and endsWith(file_name, 'lorer.exe') ''' @@ -1342,6 +1507,7 @@ description = "check built-in string functions" [[queries]] +name = "endsWithCaseInsensitive" case_insensitive = true query = ''' file where opcode=0 and endsWith(file_name, 'loREr.exe') @@ -1350,6 +1516,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "endsWith2" query = ''' file where opcode=0 and startsWith('explorer.exeaaaaaaaa', file_name) ''' @@ -1357,6 +1524,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "endsWithAndCondition" case_insensitive = true query = ''' file where opcode=0 and serial_event_id = 88 and startsWith('explorer.exeaAAAA', 'EXPLORER.exe') @@ -1365,6 +1533,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "stringContains2" query = ''' file where opcode=0 and stringContains('ABCDEFGHIexplorer.exeJKLMNOP', file_name) ''' @@ -1372,6 +1541,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "indexOfCaseInsensitive" case_insensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plore') == 2 and indexOf(file_name, '.pf') == null @@ -1380,6 +1550,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "indexOf1" query = ''' file where opcode=0 and indexOf(file_name, 'explorer.') > 0 and indexOf(file_name, 'plore', 100) > 0 ''' @@ -1387,6 +1558,7 @@ expected_event_ids = [] description = "check built-in string functions" [[queries]] +name = "indexOf2" case_sensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2 @@ -1395,6 +1567,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "indexOf3" case_insensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 0) == 2 @@ -1403,6 +1576,7 @@ expected_event_ids = [88, 92] description = "check built-in string functions" [[queries]] +name = "indexOf4" case_sensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 2) != null @@ -1411,6 +1585,7 @@ expected_event_ids = [88] description = "check built-in string functions" [[queries]] +name = "indexOf5" case_insensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 2) != null @@ -1419,6 +1594,7 @@ expected_event_ids = [88, 92] description = "check built-in string functions" [[queries]] +name = "indexOf6" query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 4) != null ''' @@ -1426,6 +1602,7 @@ expected_event_ids = [] description = "check built-in string functions" [[queries]] +name = "indexOf7" query = ''' file where opcode=0 and indexOf(file_name, 'thing that never happened') != null ''' @@ -1433,6 +1610,7 @@ expected_event_ids = [] description = "check built-in string functions" [[queries]] +name = "indexOf8" case_insensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2 @@ -1441,6 +1619,7 @@ expected_event_ids = [88, 92] description = "check substring ranges" [[queries]] +name = "indexOf9" case_sensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'plorer.', 2) == 2 @@ -1449,6 +1628,7 @@ expected_event_ids = [88] description = "check substring ranges" [[queries]] +name = "indexOf10" case_sensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0 @@ -1457,6 +1637,7 @@ expected_event_ids = [88] description = "check substring ranges" [[queries]] +name = "indexOf11" case_insensitive = true query = ''' file where opcode=0 and indexOf(file_name, 'explorer.', 0) == 0 @@ -1465,6 +1646,7 @@ expected_event_ids = [88, 92] description = "check substring ranges" [[queries]] +name = "substring1" case_insensitive = true query = ''' file where serial_event_id=88 and substring(file_name, 0, 4) == 'expl' @@ -1473,6 +1655,7 @@ expected_event_ids = [88] description = "check substring ranges" [[queries]] +name = "substring2" case_sensitive = true query = ''' file where substring(file_name, 1, 3) == 'xp' @@ -1481,6 +1664,7 @@ expected_event_ids = [88, 91] description = "check substring ranges" [[queries]] +name = "substring3" case_insensitive = true query = ''' file where substring(file_name, 1, 3) == 'xp' @@ -1489,6 +1673,7 @@ expected_event_ids = [88, 91, 92] description = "check substring ranges" [[queries]] +name = "substring4" query = ''' file where substring(file_name, -4) == '.exe' ''' @@ -1496,6 +1681,7 @@ expected_event_ids = [55, 59, 61, 65, 67, 70, 72, 75, 76, 81, 83, 86, 88, 91] description = "check substring ranges" [[queries]] +name = "substring5" query = ''' file where substring(file_name, -4, -1) == '.ex' ''' @@ -1503,6 +1689,7 @@ expected_event_ids = [55, 59, 61, 65, 67, 70, 72, 75, 76, 81, 83, 86, 88, 91] description = "check substring ranges" [[queries]] +name = "twoAdds" query = ''' process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id ''' @@ -1510,6 +1697,7 @@ expected_event_ids = [1] description = "test built-in math functions" [[queries]] +name = "subtract" query = ''' process where subtract(serial_event_id, -5) == 6 ''' @@ -1517,6 +1705,7 @@ expected_event_ids = [1] description = "test built-in math functions" [[queries]] +name = "multiplyAndDivide" query = ''' process where multiply(6, serial_event_id) == 30 and divide(30, 4.0) == 7.5 ''' @@ -1524,6 +1713,7 @@ expected_event_ids = [5] description = "test built-in math functions" [[queries]] +name = "moduloEquals" query = ''' process where modulo(11, add(serial_event_id, 1)) == serial_event_id ''' @@ -1531,6 +1721,7 @@ expected_event_ids = [1, 2, 3, 5, 11] description = "test built-in math functions" [[queries]] +name = "stringNumberConversion1" query = ''' process where serial_event_id == number('5') ''' @@ -1538,6 +1729,7 @@ expected_event_ids = [5] description = "test string/number conversions" [[queries]] +name = "stringNumberConversion2" expected_event_ids = [50] description = "test string/number conversions" query = ''' @@ -1545,6 +1737,7 @@ process where serial_event_id == number('0x32', 16) ''' [[queries]] +name = "stringNumberConversion3" expected_event_ids = [50] description = "test string/number conversions" query = ''' @@ -1552,6 +1745,7 @@ process where serial_event_id == number('32', 16) ''' [[queries]] +name = "concat1" query = ''' process where concat(serial_event_id, ':', process_name, opcode) == '5:wininit.exe3' ''' @@ -1559,6 +1753,7 @@ expected_event_ids = [5] description = "test string concatenation" [[queries]] +name = "concatCaseInsensitive" case_insensitive = true query = ''' process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3' @@ -1567,6 +1762,7 @@ expected_event_ids = [5] description = "test string concatenation" [[queries]] +name = "fieldComparisonCaseInsensitive" case_insensitive = true query = ''' process where process_name != original_file_name and length(original_file_name) > 0 @@ -1576,6 +1772,7 @@ description = "check that case insensitive comparisons are performed for fields. [[queries]] +name = "arraySearch1" expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -1583,6 +1780,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'en-US') ''' [[queries]] +name = "arraySearchCaseSensitive" case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -1591,6 +1789,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'EN-US') ''' [[queries]] +name = "arraySearchCaseInsensitive1" case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -1599,6 +1798,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'en-us') ''' [[queries]] +name = "arraySearchCaseInsensitive2" case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -1607,6 +1807,7 @@ registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us')) ''' [[queries]] +name = "arraySearchWithMysteriousField1" expected_event_ids = [75305] description = "test arraySearch - true" query = ''' @@ -1615,6 +1816,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField2" expected_event_ids = [] description = "test arraySearch - false" query = ''' @@ -1622,6 +1824,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, fal ''' [[queries]] +name = "arraySearchWithMysteriousField3" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' @@ -1629,6 +1832,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a ''' [[queries]] +name = "arraySearchWithMysteriousField4" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' @@ -1636,6 +1840,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a ''' [[queries]] +name = "arraySearchWithMysteriousField5" expected_event_ids = [75305] description = "test arraySearch - nested" query = ''' @@ -1645,6 +1850,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField6" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -1654,6 +1860,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField7" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -1663,6 +1870,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField8" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -1672,6 +1880,7 @@ network where mysterious_field ''' [[queries]] +name = "safeWrapper" expected_event_ids = [] description = "test 'safe()' wrapper for exception handling" query = ''' @@ -1679,6 +1888,7 @@ network where safe(divide(process_name, process_name)) ''' [[queries]] +name = "nestedSetComparisons" case_insensitive = true query = ''' file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe'))) @@ -1687,6 +1897,7 @@ expected_event_ids = [82] description = "nested set comparisons" [[queries]] +name = "arrayCount1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1694,6 +1905,7 @@ registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1 ''' [[queries]] +name = "arrayCount2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1701,6 +1913,7 @@ registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2 ''' [[queries]] +name = "arrayContainsCaseInsensitive5" case_insensitive = true expected_event_ids = [57] query = ''' @@ -1708,30 +1921,37 @@ registry where arrayContains(bytes_written_string_list, "missing", "en-US") ''' [[queries]] +name = "fieldMath1" expected_event_ids = [82] query = "file where serial_event_id - 1 == 81" [[queries]] +name = "fieldMath2" expected_event_ids = [82] query = "file where serial_event_id + 1 == 83" [[queries]] +name = "fieldMath3" expected_event_ids = [82] query = "file where serial_event_id * 2 == 164" [[queries]] +name = "fieldMath4" expected_event_ids = [82, 83] query = "file where serial_event_id / 2 == 41" [[queries]] +name = "fieldMath5" expected_event_ids = [82] query = "file where serial_event_id / 2.0 == 41" [[queries]] +name = "fieldMath6" expected_event_ids = [82] query = "file where serial_event_id % 40 == 2" [[queries]] +name = "betweenCaseInsensitive1" case_insensitive = true expected_event_ids = [1, 2] query = ''' @@ -1739,6 +1959,7 @@ process where between(process_name, "s", "e") == "yst" ''' [[queries]] +name = "betweenCaseInsensitive2" case_insensitive = true expected_event_ids = [1, 2] query = ''' @@ -1746,6 +1967,7 @@ process where between(process_name, "s", "e", false) == "yst" ''' [[queries]] +name = "betweenCaseSensitive" case_sensitive = true expected_event_ids = [1, 2, 42] query = ''' @@ -1753,12 +1975,14 @@ process where between(process_name, "s", "e", false) == "t" ''' [[queries]] +name = "between1" expected_event_ids = [1] query = ''' process where between(process_name, "S", "e", true) == "ystem Idle Proc" ''' [[queries]] +name = "betweenCaseInsensitive3" case_insensitive = true expected_event_ids = [1] query = ''' @@ -1766,12 +1990,14 @@ process where between(process_name, "s", "e", true) == "ystem Idle Proc" ''' [[queries]] +name = "between2" expected_event_ids = [95] query = ''' file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something" ''' [[queries]] +name = "between3" expected_event_ids = [95] query = ''' file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" @@ -1779,30 +2005,35 @@ file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" [[queries]] +name = "cidrMatch1" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "10.6.48.157/8") ''' [[queries]] +name = "cidrMatch2" expected_event_ids = [] query = ''' network where cidrMatch(source_address, "192.168.0.0/16") ''' [[queries]] +name = "cidrMatch3" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") ''' [[queries]] +name = "cidrMatch4" expected_event_ids = [75304, 75305] query = ''' network where cidrMatch(source_address, "0.0.0.0/0") ''' [[queries]] +name = "lengthCaseSensitive" case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' @@ -1810,6 +2041,7 @@ process where length(between(process_name, 'g', 'e')) > 0 ''' [[queries]] +name = "lengthCaseInsensitiveAndBetween" case_insensitive = true expected_event_ids = [7, 14, 22, 29, 44] query = ''' @@ -1817,6 +2049,7 @@ process where length(between(process_name, 'g', 'e')) > 0 ''' [[queries]] +name = "length1" expected_event_ids = [] query = ''' process where length(between(process_name, 'g', 'z')) > 0 diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index d34d4ff4561e2..f307d8dc4c2d1 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -8,12 +8,8 @@ # in order to allow at least one round-trip test with the test harness. # This will be removed once the EQL implementation is wired and actually supports this query. - -[[queries]] -expected_event_ids = [] -query = 'process where missing_field != null' - [[queries]] +name = "twoSequencesWithKeys" query = ''' sequence [process where true] by unique_pid @@ -25,6 +21,7 @@ expected_event_ids = [48, 53, 97, 98] [[queries]] +name = "twoSequencesWithTwoKeys" query = ''' sequence [process where true] by unique_pid, process_path @@ -40,15 +37,18 @@ expected_event_ids = [48, 53, ############################# [[queries]] +name = "missingField1" expected_event_ids = [] query = 'process where missing_field != null' [[queries]] +name = "equalsNullHead" expected_event_ids = [1, 2, 3, 4, 5] query = 'process where bad_field == null | head 5' [[queries]] +name = "lteAndGtWithFilter" tags = ["comparisons", "pipes"] query = ''' process where serial_event_id <= 8 and serial_event_id > 7 @@ -57,6 +57,7 @@ process where serial_event_id <= 8 and serial_event_id > 7 expected_event_ids = [8] [[queries]] +name = "filtersLteAndGt" query = ''' process where true | filter serial_event_id <= 10 @@ -65,6 +66,7 @@ process where true expected_event_ids = [7, 8, 9, 10] [[queries]] +name = "filterLteAndGtWithHead" query = ''' process where true | filter serial_event_id <= 10 @@ -74,6 +76,7 @@ process where true expected_event_ids = [7, 8] [[queries]] +name = "headWithFiltersAndTail" query = ''' process where true | head 1000 @@ -84,15 +87,18 @@ process where true expected_event_ids = [9, 10] [[queries]] +name = "processWithMultipleConditions2" query = 'process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)' expected_event_ids = [7, 8] [[queries]] +name = "compareTwoFields1" case_insensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2, 3] [[queries]] +name = "compareTwoFields2" case_sensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2] @@ -100,27 +106,32 @@ expected_event_ids = [2] [[queries]] +name = "processWithStringComparisonCaseInsensitive1" case_insensitive = true query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] +name = "processWithStringComparisonCaseInsensitive2" case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name < "systex"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringComparisonCaseInsensitive3" case_insensitive = true query = 'process where process_name >= "sysT" and process_name < "SYsTeZZZ"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringComparisonCaseInsensitive4" case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name <= "systex"' expected_event_ids = [1, 2] [[queries]] +name = "processWithStringEqualityCaseInsensitive1" case_insensitive = true query = ''' process where process_name == "VMACTHLP.exe" and unique_pid == 12 @@ -129,6 +140,7 @@ process where process_name == "VMACTHLP.exe" and unique_pid == 12 expected_event_ids = [12] [[queries]] +name = "processNameIN" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -136,6 +148,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINCaseInsensitive1" case_insensitive = true query = ''' process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") @@ -144,6 +157,7 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINCaseInsensitive2" case_insensitive = true query = ''' process where process_name in ("python.exe", "smss.exe", "Explorer.exe") @@ -152,6 +166,7 @@ process where process_name in ("python.exe", "smss.exe", "Explorer.exe") expected_event_ids = [3, 34, 48] [[queries]] +name = "processNameINWithUnique1" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique length(process_name) == length("python.exe") @@ -159,6 +174,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 48] [[queries]] +name = "processNameINWithUnique2" case_insensitive = true query = ''' process where process_name in ("Python.exe", "smss.exe", "explorer.exe") @@ -167,6 +183,7 @@ process where process_name in ("Python.exe", "smss.exe", "explorer.exe") expected_event_ids = [3, 48] [[queries]] +name = "processNameINWithUniqueHeadAndTail1" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -176,6 +193,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [34] [[queries]] +name = "processNameINWithUniqueHeadAndTail2" query = ''' process where process_name in ("python.exe", "smss.exe", "explorer.exe") | unique process_name @@ -185,6 +203,7 @@ process where process_name in ("python.exe", "smss.exe", "explorer.exe") expected_event_ids = [34] [[queries]] +name = "processNameINWithUnique3" query = ''' process where process_name in ("python.exe", "smss.exe") | unique process_name parent_process_name @@ -192,6 +211,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54, 78] [[queries]] +name = "processNameINWithTwoUniqueFields1" query = ''' process where process_name in ("python.exe", "smss.exe") | unique process_name, parent_process_name @@ -199,6 +219,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54, 78] [[queries]] +name = "processNameINWithTwoUniqueFields2" query = ''' process where process_name in ("python.exe", "smss.exe") | head 5 @@ -207,6 +228,7 @@ process where process_name in ("python.exe", "smss.exe") expected_event_ids = [3, 48, 50, 54] [[queries]] +name = "lengthCaseInsensitive" case_insensitive = true expected_event_ids = [57] query = ''' @@ -214,6 +236,7 @@ registry where length(bytes_written_string_list) == 2 and bytes_written_string_l ''' [[queries]] +name = "descendant1" query = ''' file where file_name == "csrss.exe" and opcode=0 and descendant of [process where opcode in (1,3) and process_name="cmd.exe"] @@ -221,6 +244,7 @@ file where file_name == "csrss.exe" and opcode=0 expected_event_ids = [72] [[queries]] +name = "descendant2" query = ''' process where opcode=1 and process_name == "csrss.exe" and descendant of [file where file_name == "csrss.exe" and opcode=0] @@ -228,6 +252,7 @@ process where opcode=1 and process_name == "csrss.exe" expected_event_ids = [73] [[queries]] +name = "descendant3" query = ''' process where opcode=1 and process_name == "smss.exe" and descendant of [ @@ -241,6 +266,7 @@ expected_event_ids = [78] [[queries]] +name = "simpleTail" query = ''' file where true | tail 3 @@ -248,12 +274,14 @@ file where true expected_event_ids = [92, 95, 96] [[queries]] +name = "twoINs" query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "System") ''' expected_event_ids = [2, 50, 51] [[queries]] +name = "twoINsCaseInsensitive" case_insensitive = true query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM") @@ -261,6 +289,7 @@ process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM" expected_event_ids = [2, 50, 51] [[queries]] +name = "simpleTailWithSort" case_insensitive = true expected_event_ids = [92, 95, 96, 91] query = ''' @@ -270,6 +299,7 @@ file where true ''' [[queries]] +name = "simpleHeadWithSort1" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -278,6 +308,7 @@ process where true ''' [[queries]] +name = "simpleHeadWithSort2" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -286,6 +317,7 @@ process where true ''' [[queries]] +name = "simpleHeadWithSort3" expected_event_ids = [2, 1, 4, 3, 5] query = ''' process where true @@ -294,6 +326,7 @@ process where true ''' [[queries]] +name = "twoHeadsWithSort" expected_event_ids = [2, 1] query = ''' process where true @@ -303,6 +336,7 @@ process where true ''' [[queries]] +name = "twoSortsWithHead" expected_event_ids = [1, 2, 3, 4, 5] query = ''' process where true @@ -311,32 +345,8 @@ process where true | sort serial_event_id ''' - -[[queries]] -query = ''' -sequence - [process where opcode == 1] by unique_pid - [file where opcode == 0] by unique_pid - [file where opcode == 0] by unique_pid - [file where opcode == 0] by unique_pid -until - [file where opcode == 2] by unique_pid -''' -expected_event_ids = [] - -[[queries]] -query = ''' -sequence - [process where opcode == 1] by unique_pid - [file where opcode == 0] by unique_pid - [file where opcode == 0] by unique_pid - [file where opcode == 0] by unique_pid -until - [file where opcode == 200] by unique_pid -''' -expected_event_ids = [54, 55, 61, 67] - [[queries]] +name = "fourSequencesByPidWithUntil1" query = ''' sequence [file where opcode=0] by unique_pid @@ -346,16 +356,7 @@ sequence expected_event_ids = [87, 92] [[queries]] -query = ''' -sequence - [file where opcode=0 and file_name="*.exe"] by unique_pid - [file where opcode=0 and file_name="*.exe"] by unique_pid -until [process where opcode=1] by unique_ppid -| head 1 -''' -expected_event_ids = [] - -[[queries]] +name = "doubleSameSequenceWithByUntilAndHead2" query = ''' join [file where opcode=0 and file_name="*.exe"] by unique_pid @@ -366,6 +367,7 @@ until [process where opcode=1] by unique_ppid expected_event_ids = [61, 59] [[queries]] +name = "twoJoins1" query = ''' join by user_name [process where opcode in (1,3) and process_name="smss.exe"] @@ -374,6 +376,7 @@ join by user_name expected_event_ids = [78, 48] [[queries]] +name = "threeJoins1" query = ''' join by unique_pid [process where opcode=1] @@ -383,6 +386,7 @@ join by unique_pid expected_event_ids = [54, 55, 61] [[queries]] +name = "threeJoins2" query = ''' join by string(unique_pid) [process where opcode=1] @@ -392,6 +396,7 @@ join by string(unique_pid) expected_event_ids = [54, 55, 61] [[queries]] +name = "threeJoinsWithUntil1" query = ''' join by unique_pid [process where opcode=1] @@ -402,6 +407,7 @@ until [file where opcode == 2] expected_event_ids = [] [[queries]] +name = "threeJoinsWithUntil1" query = ''' join by string(unique_pid), unique_pid, unique_pid * 2 [process where opcode=1] @@ -412,6 +418,7 @@ until [file where opcode == 2] expected_event_ids = [] [[queries]] +name = "twoJoins2" query = ''' join [file where opcode=0 and file_name="svchost.exe"] by unique_pid @@ -420,6 +427,7 @@ join expected_event_ids = [55, 56] [[queries]] +name = "twoJoins3" query = ''' join by unique_pid [process where opcode in (1,3) and process_name="python.exe"] @@ -428,6 +436,7 @@ join by unique_pid expected_event_ids = [54, 55] [[queries]] +name = "twoJoins4" query = ''' join by user_name [process where opcode in (1,3) and process_name="python.exe"] @@ -436,6 +445,7 @@ join by user_name expected_event_ids = [48, 78] [[queries]] +name = "twoJoins5" query = ''' join [process where opcode in (1,3) and process_name="python.exe"] @@ -444,12 +454,14 @@ join expected_event_ids = [48, 3, 50, 78] [[queries]] +name = "fakeField" expected_event_ids = [] query = ''' process where fake_field == "*" ''' [[queries]] +name = "fakeFieldWithHead1" # no longer valid, since this returns `null`, and `not null` is still null # expected_event_ids = [1, 2, 3, 4] expected_event_ids = [] @@ -459,6 +471,7 @@ process where fake_field != "*" ''' [[queries]] +name = "fakeFieldWithHead2" # no longer valid, since this returns `null`, and `not null` is still null # expected_event_ids = [1, 2, 3, 4] expected_event_ids = [] @@ -468,52 +481,62 @@ process where not (fake_field == "*") ''' [[queries]] +name = "invalidFieldName" expected_event_ids = [] query = ''' registry where invalid_field_name != null ''' [[queries]] +name = "badFieldWithLength" expected_event_ids = [] query = ''' registry where length(bad_field) > 0 ''' [[queries]] +name = "equalsNullHead" expected_event_ids = [1, 2, 3, 4, 5] query = 'process where bad_field == null | head 5' [[queries]] +name = "exitCodeGtZero" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code >= 0' [[queries]] +name = "zeroLteExitCode" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where 0 <= exit_code' [[queries]] +name = "exitCodeLteZero" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code <= 0' [[queries]] +name = "exitCodeLtOne" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code < 1' [[queries]] +name = "exitCodeGtMinusOne" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where exit_code > -1' [[queries]] +name = "minusOneLtExitCode" note = "check that comparisons against null values return false" expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] query = 'process where -1 < exit_code' [[queries]] +name = "notExitCodeGtWithHead1" note = "check that comparisons against null values return false" expected_event_ids = [] query = ''' @@ -523,16 +546,19 @@ process where not (exit_code > -1) ''' [[queries]] +name = "notExitCodeGtWithHead2" note = "check that comparisons against null values return false" expected_event_ids = [1, 2, 3, 4, 5, 6, 7] query = 'process where not (exit_code > -1) | head 7' [[queries]] +name = "notExitCodeGtWithHead3" note = "check that comparisons against null values return false" expected_event_ids = [1, 2, 3, 4, 5, 6, 7] query = 'process where not (-1 < exit_code) | head 7' [[queries]] +name = "anyWithUnique" expected_event_ids = [1, 55, 57, 63, 75304] query = ''' any where true @@ -540,6 +566,7 @@ any where true ''' [[queries]] +name = "multipleConditionsWithDescendant1" query = ''' process where opcode=1 and process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -547,6 +574,7 @@ process where opcode=1 and process_name in ("services.exe", "smss.exe", "lsass.e expected_event_ids = [62, 68, 78] [[queries]] +name = "INWithDescendant" query = ''' process where process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -554,6 +582,7 @@ process where process_name in ("services.exe", "smss.exe", "lsass.exe") expected_event_ids = [62, 64, 68, 69, 78, 80] [[queries]] +name = "multipleConditionsWithDescendant2" query = ''' process where opcode=2 and process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ] @@ -561,6 +590,7 @@ process where opcode=2 and process_name in ("services.exe", "smss.exe", "lsass.e expected_event_ids = [64, 69, 80] [[queries]] +name = "childOf1" query = ''' process where process_name="svchost.exe" and child of [file where file_name="svchost.exe" and opcode=0] @@ -568,6 +598,7 @@ process where process_name="svchost.exe" expected_event_ids = [56, 58] [[queries]] +name = "childOf2" query = ''' process where process_name="svchost.exe" and not child of [file where file_name="svchost.exe" and opcode=0] @@ -576,6 +607,7 @@ process where process_name="svchost.exe" expected_event_ids = [11, 13, 15] [[queries]] +name = "nestedChildOf1" query = ''' process where process_name="lsass.exe" and child of [ @@ -586,6 +618,7 @@ process where process_name="lsass.exe" expected_event_ids = [62, 64] [[queries]] +name = "nestedChildOf2" query = ''' file where child of [ process where child of [ @@ -597,6 +630,7 @@ file where child of [ expected_event_ids = [91] [[queries]] +name = "fileByUniquePid1" query = ''' file where process_name = "python.exe" | unique unique_pid @@ -604,6 +638,7 @@ file where process_name = "python.exe" expected_event_ids = [55, 95] [[queries]] +name = "fileByUniquePid2" query = ''' file where event of [process where process_name = "python.exe" ] | unique unique_pid @@ -611,37 +646,19 @@ file where event of [process where process_name = "python.exe" ] expected_event_ids = [55, 95] [[queries]] +name = "simpleStringEquality" query = ''' process where process_name = "python.exe" ''' expected_event_ids = [48, 50, 51, 54, 93] [[queries]] +name = "eventOfProcess" query = 'process where event of [process where process_name = "python.exe" ]' expected_event_ids = [48, 50, 51, 54, 93] [[queries]] -query = ''' -sequence by user_name - [file where opcode=0] by pid,file_path - [file where opcode=2] by pid,file_path -until - [process where opcode == 2] by ppid,process_path -''' -expected_event_ids = [] - -[[queries]] -query = ''' -sequence by user_name - [file where opcode=0] by pid,file_path - [file where opcode=2] by pid,file_path -until - [process where opcode == 5] by ppid,process_path -| head 2 -''' -expected_event_ids = [55, 59, 61, 65] - -[[queries]] +name = "twoSequencesWithTwoKeysAndUntil" query = ''' join by user_name [file where true] by pid,file_path @@ -651,6 +668,7 @@ join by user_name expected_event_ids = [55, 56, 59, 58] [[queries]] +name = "threeSequencesWithHead" query = ''' sequence [process where true] by unique_pid @@ -661,6 +679,7 @@ sequence expected_event_ids = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73] [[queries]] +name = "uniqueCount1" expected_event_ids = [11, 60, 63] query = ''' any where process_name == "svchost.exe" @@ -668,6 +687,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCount2" expected_event_ids = [63, 60, 11] query = ''' any where process_name == "svchost.exe" @@ -676,6 +696,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCount3" expected_event_ids = [60] query = ''' any where process_name == "svchost.exe" @@ -684,6 +705,7 @@ any where process_name == "svchost.exe" ''' [[queries]] +name = "uniqueCountAndFilter" expected_event_ids = [11] query = ''' any where process_name == "svchost.exe" @@ -693,6 +715,7 @@ any where process_name == "svchost.exe" [[queries]] +name = "arrayContainsCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -700,6 +723,7 @@ registry where arrayContains(bytes_written_string_list, 'En-uS') ''' [[queries]] +name = "arrayContainsCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -707,6 +731,7 @@ registry where arrayContains(bytes_written_string_list, 'En') ''' [[queries]] +name = "lengthCaseInsensitive" case_insensitive = true expected_event_ids = [57] query = ''' @@ -714,6 +739,7 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li ''' [[queries]] +name = "arrayCaseInsensitive1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -721,6 +747,7 @@ registry where bytes_written_string_list[0] == 'EN-us' ''' [[queries]] +name = "arrayCaseInsensitive2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -729,6 +756,7 @@ registry where bytes_written_string_list[1] == 'EN' [[queries]] +name = "arrayContainsCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' @@ -736,6 +764,7 @@ registry where arrayContains(bytes_written_string_list, 'en-US') ''' [[queries]] +name = "arrayContainsCaseInsensitive4" case_insensitive = true expected_event_ids = [57] query = ''' @@ -743,6 +772,7 @@ registry where arrayContains(bytes_written_string_list, 'en') ''' [[queries]] +name = "arrayCaseInsensitive3" case_insensitive = true expected_event_ids = [57] query = ''' @@ -750,12 +780,14 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li ''' [[queries]] +name = "arrayEquality1" expected_event_ids = [57] query = ''' registry where bytes_written_string_list[0] == 'en-US' ''' [[queries]] +name = "arrayEquality2" expected_event_ids = [57] query = ''' registry where bytes_written_string_list[1] == 'en' @@ -763,36 +795,42 @@ registry where bytes_written_string_list[1] == 'en' # character classes aren't supported. custom tests made in test_queries_supported.toml [[queries]] +name = "matchLite1" query = ''' process where matchLite(command_line, ?'.*?net1\s+localgroup\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "matchLite2" query = ''' process where matchLite(command_line, ?'.*?net1\s+\w+\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "matchLite3" query = ''' process where matchLite(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "match1" expected_event_ids = [98] query = ''' process where match(command_line, ?'.*?net1\s+\w{4,15}\s+.*?') ''' [[queries]] +name = "matchLite4" query = ''' process where matchLite(command_line, ?'.*?net1\s+[localgrup]{4,15}\s+.*?') ''' expected_event_ids = [98] [[queries]] +name = "stringEqualsCaseInsensitive1" case_insensitive = true query = ''' process where 'net.EXE' == original_file_name @@ -802,6 +840,7 @@ expected_event_ids = [97] note = "check that case insensitive comparisons are performed even for lhs strings." [[queries]] +name = "stringEqualsCaseInsensitive2" case_insensitive = true query = ''' process where process_name == original_file_name and process_name='net*.exe' @@ -810,6 +849,7 @@ expected_event_ids = [97, 98] note = "check that case insensitive comparisons are performed for fields." [[queries]] +name = "fieldsComparisonCaseInsensitive" case_insensitive = true query = ''' process where original_file_name == process_name and length(original_file_name) > 0 @@ -818,6 +858,7 @@ expected_event_ids = [97, 98, 75273, 75303] description = "check that case insensitive comparisons are performed for fields." [[queries]] +name = "substring3" case_insensitive = true query = ''' file where substring(file_name, 1, 3) == 'xp' @@ -826,6 +867,7 @@ expected_event_ids = [88, 91, 92] description = "check substring ranges" [[queries]] +name = "moduloEquals" query = ''' process where modulo(11, add(serial_event_id, 1)) == serial_event_id ''' @@ -833,6 +875,7 @@ expected_event_ids = [1, 2, 3, 5, 11] description = "test built-in math functions" [[queries]] +name = "concatCaseInsensitive" case_insensitive = true query = ''' process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3' @@ -841,6 +884,7 @@ expected_event_ids = [5] description = "test string concatenation" [[queries]] +name = "fieldComparisonCaseInsensitive" case_insensitive = true query = ''' process where process_name != original_file_name and length(original_file_name) > 0 @@ -850,6 +894,7 @@ description = "check that case insensitive comparisons are performed for fields. [[queries]] +name = "arraySearch1" expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -857,6 +902,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'en-US') ''' [[queries]] +name = "arraySearchCaseSensitive" case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -865,6 +911,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'EN-US') ''' [[queries]] +name = "arraySearchCaseInsensitive1" case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -873,6 +920,7 @@ registry where arraySearch(bytes_written_string_list, a, a == 'en-us') ''' [[queries]] +name = "arraySearchCaseInsensitive2" case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" @@ -881,6 +929,7 @@ registry where arraySearch(bytes_written_string_list, a, endsWith(a, '-us')) ''' [[queries]] +name = "arraySearchWithMysteriousField1" expected_event_ids = [75305] description = "test arraySearch - true" query = ''' @@ -889,6 +938,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField2" expected_event_ids = [] description = "test arraySearch - false" query = ''' @@ -896,6 +946,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, fal ''' [[queries]] +name = "arraySearchWithMysteriousField3" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' @@ -903,6 +954,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a ''' [[queries]] +name = "arraySearchWithMysteriousField4" expected_event_ids = [75305] description = "test arraySearch - conditional" query = ''' @@ -910,6 +962,7 @@ network where mysterious_field and arraySearch(mysterious_field.subarray, s, s.a ''' [[queries]] +name = "arraySearchWithMysteriousField5" expected_event_ids = [75305] description = "test arraySearch - nested" query = ''' @@ -919,6 +972,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField6" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -928,6 +982,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField7" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -937,6 +992,7 @@ network where mysterious_field ''' [[queries]] +name = "arraySearchWithMysteriousField8" expected_event_ids = [75305] description = "test arraySearch - nested with cross-check pass" query = ''' @@ -946,6 +1002,7 @@ network where mysterious_field ''' [[queries]] +name = "safeWrapper" expected_event_ids = [] description = "test 'safe()' wrapper for exception handling" query = ''' @@ -953,6 +1010,7 @@ network where safe(divide(process_name, process_name)) ''' [[queries]] +name = "nestedSetComparisons" case_insensitive = true query = ''' file where serial_event_id == 82 and (true == (process_name in ('svchost.EXE', 'bad.exe', 'bad2.exe'))) @@ -961,6 +1019,7 @@ expected_event_ids = [82] description = "nested set comparisons" [[queries]] +name = "arrayCount1" case_insensitive = true expected_event_ids = [57] query = ''' @@ -968,6 +1027,7 @@ registry where arrayCount(bytes_written_string_list, s, s == '*-us') == 1 ''' [[queries]] +name = "arrayCount2" case_insensitive = true expected_event_ids = [57] query = ''' @@ -975,6 +1035,7 @@ registry where arrayCount(bytes_written_string_list, s, s == '*en*') == 2 ''' [[queries]] +name = "arrayContainsCaseInsensitive5" case_insensitive = true expected_event_ids = [57] query = ''' @@ -983,6 +1044,7 @@ registry where arrayContains(bytes_written_string_list, "missing", "en-US") [[queries]] +name = "sequenceNonFieldBasedJoin" note = "Sequence: non-field based join." query = ''' sequence @@ -994,6 +1056,7 @@ expected_event_ids = [1, 2, [[queries]] +name = "sequenceMultipleNonFieldBasedJoins" note = "Sequence: multiple non-field based joins." query = ''' sequence @@ -1005,6 +1068,7 @@ expected_event_ids = [1, 2, # TODO: update toggles for this function [[queries]] +name = "betweenCaseSensitive" case_sensitive = true expected_event_ids = [1, 2, 42] query = ''' @@ -1013,6 +1077,7 @@ process where between(process_name, "s", "e", false) == "t" # TODO: add toggles to this function so it's not always insensitive [[queries]] +name = "lengthCaseSensitive" case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' @@ -1021,6 +1086,7 @@ process where length(between(process_name, 'g', 'e')) > 0 # TODO: add toggles to this function so it's not always insensitive #[[queries]] +#name = "" #case_insensitive = true #expected_event_ids = [7, 14, 22, 29, 44] #query = ''' diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle index d5e1dff4464db..d790b6722ff34 100644 --- a/x-pack/plugin/eql/qa/rest/build.gradle +++ b/x-pack/plugin/eql/qa/rest/build.gradle @@ -19,9 +19,7 @@ dependencies { testClusters.integTest { testDistribution = 'DEFAULT' - if (BuildParams.isSnapshotBuild()) { - setting 'xpack.eql.enabled', 'true' - } + setting 'xpack.eql.enabled', 'true' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.monitoring.collection.enabled', 'true' } diff --git a/x-pack/plugin/eql/qa/security/build.gradle b/x-pack/plugin/eql/qa/security/build.gradle index a6048a2ca1447..b91721701df48 100644 --- a/x-pack/plugin/eql/qa/security/build.gradle +++ b/x-pack/plugin/eql/qa/security/build.gradle @@ -11,9 +11,7 @@ dependencies { testClusters.integTest { testDistribution = 'DEFAULT' - if (BuildParams.isSnapshotBuild()) { - setting 'xpack.eql.enabled', 'true' - } + setting 'xpack.eql.enabled', 'true' setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.monitoring.collection.enabled', 'true' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 14afce585a069..fa21ed41f02b6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.searchafter.SearchAfterBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -33,7 +32,6 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_CATEGORY; -import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_IMPLICIT_JOIN_KEY; import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Replaceable, ToXContent { @@ -49,10 +47,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private String timestampField = FIELD_TIMESTAMP; private String tiebreakerField = null; private String eventCategoryField = FIELD_EVENT_CATEGORY; - private String implicitJoinKeyField = FIELD_IMPLICIT_JOIN_KEY; private int size = RequestDefaults.SIZE; private int fetchSize = RequestDefaults.FETCH_SIZE; - private SearchAfterBuilder searchAfterBuilder; private String query; private boolean isCaseSensitive = false; @@ -65,10 +61,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field"; static final String KEY_EVENT_CATEGORY_FIELD = "event_category_field"; - static final String KEY_IMPLICIT_JOIN_KEY_FIELD = "implicit_join_key_field"; static final String KEY_SIZE = "size"; static final String KEY_FETCH_SIZE = "fetch_size"; - static final String KEY_SEARCH_AFTER = "search_after"; static final String KEY_QUERY = "query"; static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; static final String KEY_KEEP_ALIVE = "keep_alive"; @@ -79,10 +73,8 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); static final ParseField TIEBREAKER_FIELD = new ParseField(KEY_TIEBREAKER_FIELD); static final ParseField EVENT_CATEGORY_FIELD = new ParseField(KEY_EVENT_CATEGORY_FIELD); - static final ParseField IMPLICIT_JOIN_KEY_FIELD = new ParseField(KEY_IMPLICIT_JOIN_KEY_FIELD); static final ParseField SIZE = new ParseField(KEY_SIZE); static final ParseField FETCH_SIZE = new ParseField(KEY_FETCH_SIZE); - static final ParseField SEARCH_AFTER = new ParseField(KEY_SEARCH_AFTER); static final ParseField QUERY = new ParseField(KEY_QUERY); static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField(KEY_WAIT_FOR_COMPLETION_TIMEOUT); static final ParseField KEEP_ALIVE = new ParseField(KEY_KEEP_ALIVE); @@ -103,10 +95,8 @@ public EqlSearchRequest(StreamInput in) throws IOException { timestampField = in.readString(); tiebreakerField = in.readOptionalString(); eventCategoryField = in.readString(); - implicitJoinKeyField = in.readString(); size = in.readVInt(); fetchSize = in.readVInt(); - searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); query = in.readString(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: Remove after backport this.waitForCompletionTimeout = in.readOptionalTimeValue(); @@ -147,10 +137,6 @@ public ActionRequestValidationException validate() { validationException = addValidationError("event category field is null or empty", validationException); } - if (implicitJoinKeyField == null || implicitJoinKeyField.isEmpty()) { - validationException = addValidationError("implicit join key field is null or empty", validationException); - } - if (size <= 0) { validationException = addValidationError("size must be greater than 0", validationException); } @@ -177,16 +163,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(KEY_TIEBREAKER_FIELD, tiebreakerField()); } builder.field(KEY_EVENT_CATEGORY_FIELD, eventCategoryField()); - if (implicitJoinKeyField != null) { - builder.field(KEY_IMPLICIT_JOIN_KEY_FIELD, implicitJoinKeyField()); - } builder.field(KEY_SIZE, size()); builder.field(KEY_FETCH_SIZE, fetchSize()); - - if (searchAfterBuilder != null) { - builder.array(SEARCH_AFTER.getPreferredName(), searchAfterBuilder.getSortValues()); - } - builder.field(KEY_QUERY, query); if (waitForCompletionTimeout != null) { builder.field(KEY_WAIT_FOR_COMPLETION_TIMEOUT, waitForCompletionTimeout); @@ -211,11 +189,8 @@ protected static ObjectParser objectParser parser.declareString(EqlSearchRequest::timestampField, TIMESTAMP_FIELD); parser.declareString(EqlSearchRequest::tiebreakerField, TIEBREAKER_FIELD); parser.declareString(EqlSearchRequest::eventCategoryField, EVENT_CATEGORY_FIELD); - parser.declareString(EqlSearchRequest::implicitJoinKeyField, IMPLICIT_JOIN_KEY_FIELD); parser.declareInt(EqlSearchRequest::size, SIZE); parser.declareInt(EqlSearchRequest::fetchSize, FETCH_SIZE); - parser.declareField(EqlSearchRequest::setSearchAfter, SearchAfterBuilder::fromXContent, SEARCH_AFTER, - ObjectParser.ValueType.OBJECT_ARRAY); parser.declareString(EqlSearchRequest::query, QUERY); parser.declareField(EqlSearchRequest::waitForCompletionTimeout, (p, c) -> TimeValue.parseTimeValue(p.text(), KEY_WAIT_FOR_COMPLETION_TIMEOUT), WAIT_FOR_COMPLETION_TIMEOUT, @@ -261,13 +236,6 @@ public EqlSearchRequest eventCategoryField(String eventCategoryField) { return this; } - public String implicitJoinKeyField() { return this.implicitJoinKeyField; } - - public EqlSearchRequest implicitJoinKeyField(String implicitJoinKeyField) { - this.implicitJoinKeyField = implicitJoinKeyField; - return this; - } - public int size() { return this.size; } @@ -286,23 +254,6 @@ public EqlSearchRequest fetchSize(int fetchSize) { return this; } - public Object[] searchAfter() { - if (searchAfterBuilder == null) { - return null; - } - return searchAfterBuilder.getSortValues(); - } - - public EqlSearchRequest searchAfter(Object[] values) { - this.searchAfterBuilder = new SearchAfterBuilder().setSortValues(values); - return this; - } - - private EqlSearchRequest setSearchAfter(SearchAfterBuilder builder) { - this.searchAfterBuilder = builder; - return this; - } - public String query() { return this.query; } public EqlSearchRequest query(String query) { @@ -353,10 +304,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(timestampField); out.writeOptionalString(tiebreakerField); out.writeString(eventCategoryField); - out.writeString(implicitJoinKeyField); out.writeVInt(size); out.writeVInt(fetchSize); - out.writeOptionalWriteable(searchAfterBuilder); out.writeString(query); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO: Remove after backport out.writeOptionalTimeValue(waitForCompletionTimeout); @@ -383,8 +332,6 @@ public boolean equals(Object o) { Objects.equals(timestampField, that.timestampField) && Objects.equals(tiebreakerField, that.tiebreakerField) && Objects.equals(eventCategoryField, that.eventCategoryField) && - Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) && - Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && Objects.equals(query, that.query) && Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && @@ -402,8 +349,6 @@ public int hashCode() { timestampField, tiebreakerField, eventCategoryField, - implicitJoinKeyField, - searchAfterBuilder, query, waitForCompletionTimeout, keepAlive, diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java index 340271bdc5113..6e7d1326a3943 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java @@ -40,11 +40,6 @@ public EqlSearchRequestBuilder eventCategoryField(String eventCategoryField) { return this; } - public EqlSearchRequestBuilder implicitJoinKeyField(String implicitJoinKeyField) { - request.implicitJoinKeyField(implicitJoinKeyField); - return this; - } - public EqlSearchRequestBuilder size(int size) { request.size(size); return this; @@ -55,17 +50,12 @@ public EqlSearchRequestBuilder fetchSize(int fetchSize) { return this; } - public EqlSearchRequestBuilder searchAfter(Object[] values) { - request.searchAfter(values); - return this; - } - public EqlSearchRequestBuilder query(String query) { request.query(query); return this; } - public EqlSearchRequestBuilder query(boolean isCaseSensitive) { + public EqlSearchRequestBuilder isCaseSensitive(boolean isCaseSensitive) { request.isCaseSensitive(isCaseSensitive); return this; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java index 4474962f5c299..c4aebf378992d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/RequestDefaults.java @@ -12,7 +12,6 @@ private RequestDefaults() {} public static final String FIELD_TIMESTAMP = "@timestamp"; public static final String FIELD_EVENT_CATEGORY = "event.category"; - public static final String FIELD_IMPLICIT_JOIN_KEY = "agent.id"; public static int SIZE = 10; public static int FETCH_SIZE = 1000; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/BoxedQueryRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/BoxedQueryRequest.java index 99d9671f16e26..5908e1ec91f48 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/BoxedQueryRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/BoxedQueryRequest.java @@ -75,6 +75,10 @@ public BoxedQueryRequest from(Ordinal begin) { return this; } + public Ordinal after() { + return after; + } + public Ordinal from() { return from; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java index 69d0048679418..4a47705dcdaf3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java @@ -20,6 +20,14 @@ public class KeyAndOrdinal { this.ordinal = ordinal; } + public SequenceKey key() { + return key; + } + + public Ordinal ordinal() { + return ordinal; + } + @Override public int hashCode() { return Objects.hash(key, ordinal); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Matcher.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Matcher.java index 84c929a9ec264..0ab1fec7e189b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Matcher.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Matcher.java @@ -5,11 +5,12 @@ */ package org.elasticsearch.xpack.eql.execution.assembler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.eql.execution.search.Limit; -import org.elasticsearch.xpack.eql.execution.search.Ordinal; import org.elasticsearch.xpack.eql.execution.sequence.Sequence; import org.elasticsearch.xpack.eql.execution.sequence.SequenceStateMachine; import org.elasticsearch.xpack.eql.session.Payload; @@ -21,6 +22,8 @@ */ class Matcher { + private final Logger log = LogManager.getLogger(Matcher.class); + // NB: just like in a list, this represents the total number of stages yet counting starts at 0 private final SequenceStateMachine stateMachine; private final int numberOfStages; @@ -48,27 +51,33 @@ boolean match(int stage, Iterable> hits) { // early skip in case of reaching the limit // check the last stage to avoid calling the state machine in other stages if (stateMachine.reachedLimit()) { + log.trace("Limit reached {}", stateMachine.stats()); return false; } } } + log.trace("{}", stateMachine.stats()); return true; } - boolean until(Iterable markers) { - // no-op so far - - return false; + void until(Iterable markers) { + stateMachine.until(markers); } - public boolean hasCandidates(int stage) { + boolean hasCandidates(int stage) { return stateMachine.hasCandidates(stage); } + void dropUntil() { + stateMachine.dropUntil(); + } + Payload payload(long startTime) { List completed = stateMachine.completeSequences(); TimeValue tookTime = new TimeValue(System.currentTimeMillis() - startTime); - return new SequencePayload(completed, false, tookTime); + Payload p = new SequencePayload(completed, false, tookTime); + stateMachine.clear(); + return p; } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/TumblingWindow.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/TumblingWindow.java index 6832554937600..5d386e168d948 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/TumblingWindow.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/TumblingWindow.java @@ -47,11 +47,10 @@ public class TumblingWindow implements Executable { private static class WindowInfo { private final int baseStage; - private final Ordinal begin, end; + private final Ordinal end; - WindowInfo(int baseStage, Ordinal begin, Ordinal end) { + WindowInfo(int baseStage, Ordinal end) { this.baseStage = baseStage; - this.begin = begin; this.end = end; } } @@ -93,18 +92,29 @@ private void baseCriterion(int baseStage, Payload p, ActionListener lis Criterion base = criteria.get(baseStage); List hits = p.values(); + log.trace("Found [{}] hits", hits.size()); + if (hits.isEmpty() == false) { if (matcher.match(baseStage, wrapValues(base, hits)) == false) { listener.onResponse(payload()); return; } } - // empty or only one result means there aren't going to be any matches + + // only one result means there aren't going to be any matches // so move the window boxing to the next stage if (hits.size() < 2) { // if there are still candidates, advance the window base if (matcher.hasCandidates(baseStage) && baseStage + 1 < maxStages) { - advance(baseStage + 1, listener); + Runnable next = () -> advance(baseStage + 1, listener); + + if (until != null && hits.size() == 1) { + // find "until" ordinals - early on to discard data in-flight to avoid matching + // hits that can occur in other documents + untilCriterion(new WindowInfo(baseStage, base.ordinal(hits.get(0))), listener, next); + } else { + next.run(); + } } // there aren't going to be any matches so cancel search else { @@ -122,37 +132,72 @@ private void baseCriterion(int baseStage, Payload p, ActionListener lis log.trace("Found base [{}] window {}->{}", base.stage(), begin, end); - // find until ordinals - //NB: not currently implemented + WindowInfo info = new WindowInfo(baseStage, end); // no more queries to run if (baseStage + 1 < maxStages) { - secondaryCriterion(new WindowInfo(baseStage, begin, end), baseStage + 1, listener); + Runnable next = () -> secondaryCriterion(info, baseStage + 1, listener); + if (until != null) { + // find "until" ordinals - early on to discard data in-flight to avoid matching + // hits that can occur in other documents + untilCriterion(info, listener, next); + } else { + next.run(); + } } else { advance(baseStage, listener); } } + private void untilCriterion(WindowInfo window, ActionListener listener, Runnable next) { + final BoxedQueryRequest request = until.queryRequest(); + + // before doing a new query, clean all previous until hits + // including dropping any in-flight sequences that were not dropped (because they did not match) + matcher.dropUntil(); + + final boolean reversed = boxQuery(window, until); + + log.trace("Querying until stage {}", request); + + client.query(request, wrap(p -> { + List hits = p.values(); + + log.trace("Found [{}] hits", hits.size()); + // no more results for until - let the other queries run + if (hits.isEmpty()) { + // put the markers in place before the next call + if (reversed) { + request.to(window.end); + } else { + request.from(window.end); + } + } else { + // prepare the query for the next search + request.nextAfter(until.ordinal(hits.get(hits.size() - 1))); + + // if the limit has been reached, return what's available + matcher.until(wrapUntilValues(wrapValues(until, hits))); + } + + // keep running the query runs out of the results (essentially returns less than what we want) + if (hits.size() == windowSize) { + untilCriterion(window, listener, next); + } + // looks like this stage is done, move on + else { + // to the next query + next.run(); + } + + }, listener::onFailure)); + } + private void secondaryCriterion(WindowInfo window, int currentStage, ActionListener listener) { final Criterion criterion = criteria.get(currentStage); - final BoxedQueryRequest request = criterion.queryRequest(); - Criterion base = criteria.get(window.baseStage); - // first box the query - // only the first base can be descending - // all subsequence queries are ascending - if (criterion.reverse() != base.reverse()) { - if (window.end.equals(request.from()) == false) { - // if that's the case, set the starting point - request.from(window.end); - // reposition the pointer - request.nextAfter(window.end); - } - } else { - // otherwise just the upper limit - request.to(window.end); - } + final boolean reversed = boxQuery(window, criterion); log.trace("Querying (secondary) stage [{}] {}", criterion.stage(), request); @@ -160,10 +205,11 @@ private void secondaryCriterion(WindowInfo window, int currentStage, ActionListe List hits = p.values(); log.trace("Found [{}] hits", hits.size()); + // no more results for this query if (hits.isEmpty()) { // put the markers in place before the next call - if (criterion.reverse() != base.reverse()) { + if (reversed) { request.to(window.end); } else { request.from(window.end); @@ -207,6 +253,32 @@ private void secondaryCriterion(WindowInfo window, int currentStage, ActionListe }, listener::onFailure)); } + /** + * Box the query for the given criterion based on the window information. + * Returns a boolean indicating whether reversal has been applied or not. + */ + private boolean boxQuery(WindowInfo window, Criterion criterion) { + final BoxedQueryRequest request = criterion.queryRequest(); + Criterion base = criteria.get(window.baseStage); + + // first box the query + // only the first base can be descending + // all subsequence queries are ascending + if (criterion.reverse() != base.reverse()) { + if (window.end.equals(request.from()) == false) { + // if that's the case, set the starting point + request.from(window.end); + // reposition the pointer + request.nextAfter(window.end); + } + } else { + // otherwise just the upper limit + request.to(window.end); + } + + return criterion.reverse() != base.reverse(); + } + Iterable> wrapValues(Criterion criterion, List hits) { return () -> { final Iterator iter = criterion.reverse() ? new ReversedIterator<>(hits) : hits.iterator(); @@ -229,6 +301,25 @@ public Tuple next() { }; } + Iterable wrapUntilValues(Iterable> iterable) { + return () -> { + final Iterator> iter = iterable.iterator(); + + return new Iterator<>() { + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public KeyAndOrdinal next() { + return iter.next().v1(); + } + }; + }; + } + Payload payload() { return matcher.payload(startTime); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Ordinal.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Ordinal.java index b61111638132b..dc0fc5827b134 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Ordinal.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Ordinal.java @@ -77,6 +77,10 @@ public int compareTo(Ordinal o) { return 1; } + public boolean between(Ordinal left, Ordinal right) { + return (compareTo(left) <= 0 && compareTo(right) >= 0) || (compareTo(right) <= 0 && compareTo(left) >= 0); + } + public Object[] toArray() { return tiebreaker != null ? new Object[] { timestamp, tiebreaker } : new Object[] { timestamp }; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/KeyToSequences.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/KeyToSequences.java index fd0cd7fb2ecf6..8aa5452d6ba4b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/KeyToSequences.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/KeyToSequences.java @@ -6,8 +6,13 @@ package org.elasticsearch.xpack.eql.execution.sequence; +import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.xpack.eql.execution.assembler.KeyAndOrdinal; +import org.elasticsearch.xpack.eql.execution.search.Ordinal; + import java.util.LinkedHashMap; import java.util.Map; +import java.util.Map.Entry; /** Dedicated collection for mapping a key to a list of sequences */ /** The list represents the sequence for each stage (based on its index) and is fixed in size */ @@ -17,19 +22,16 @@ class KeyToSequences { private final int listSize; /** for each key, associate the frame per state (determined by index) */ private final Map keyToSequences; + private final Map keyToUntil; KeyToSequences(int listSize) { this.listSize = listSize; this.keyToSequences = new LinkedHashMap<>(); + this.keyToUntil = new LinkedHashMap<>(); } private SequenceGroup[] group(SequenceKey key) { - SequenceGroup[] groups = keyToSequences.get(key); - if (groups == null) { - groups = new SequenceGroup[listSize]; - keyToSequences.put(key, groups); - } - return groups; + return keyToSequences.computeIfAbsent(key, k -> new SequenceGroup[listSize]); } SequenceGroup groupIfPresent(int stage, SequenceKey key) { @@ -37,6 +39,10 @@ SequenceGroup groupIfPresent(int stage, SequenceKey key) { return groups == null ? null : groups[stage]; } + UntilGroup untilIfPresent(SequenceKey key) { + return keyToUntil.get(key); + } + void add(int stage, Sequence sequence) { SequenceKey key = sequence.key(); SequenceGroup[] groups = group(key); @@ -47,7 +53,64 @@ void add(int stage, Sequence sequence) { groups[stage].add(sequence); } + void until(Iterable until) { + for (KeyAndOrdinal keyAndOrdinal : until) { + // ignore unknown keys + SequenceKey key = keyAndOrdinal.key(); + if (keyToSequences.containsKey(key)) { + UntilGroup group = keyToUntil.computeIfAbsent(key, UntilGroup::new); + group.add(keyAndOrdinal); + } + } + } + + void remove(int stage, SequenceGroup group) { + SequenceKey key = group.key(); + SequenceGroup[] groups = keyToSequences.get(key); + groups[stage] = null; + // clean-up the key if all groups are empty + boolean shouldRemoveKey = true; + for (SequenceGroup gp : groups) { + if (gp != null && gp.isEmpty() == false) { + shouldRemoveKey = false; + break; + } + } + if (shouldRemoveKey) { + keyToSequences.remove(key); + } + } + + void dropUntil() { + // clean-up all candidates that occur before until + for (Entry entry : keyToUntil.entrySet()) { + SequenceGroup[] groups = keyToSequences.get(entry.getKey()); + if (groups != null) { + for (Ordinal o : entry.getValue()) { + for (SequenceGroup group : groups) { + if (group != null) { + group.trimBefore(o); + } + } + } + } + } + + keyToUntil.clear(); + } + + public void clear() { + keyToSequences.clear(); + keyToUntil.clear(); + } + int numberOfKeys() { return keyToSequences.size(); } -} + + @Override + public String toString() { + return LoggerMessageFormat.format(null, "Keys=[{}], Until=[{}]", keyToSequences.size(), keyToUntil.size()); + } + +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/OrdinalGroup.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/OrdinalGroup.java new file mode 100644 index 0000000000000..acde58f0fd8ac --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/OrdinalGroup.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.sequence; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.xpack.eql.execution.search.Ordinal; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; + +/** List of in-flight ordinals for a given key. For fast lookup, typically associated with a stage. */ +abstract class OrdinalGroup implements Iterable { + + private final SequenceKey key; + private final Function extractor; + + // NB: since the size varies significantly, use a LinkedList + // Considering the order it might make sense to use a B-Tree+ for faster lookups which should work well with + // timestamp compression (whose range is known for the current frame). + private final List elements = new LinkedList<>(); + + private int hashCode = 0; + + private Ordinal start, stop; + + protected OrdinalGroup(SequenceKey key, Function extractor) { + this.key = key; + hashCode = key.hashCode(); + + this.extractor = extractor; + } + + public SequenceKey key() { + return key; + } + + public void add(E element) { + elements.add(element); + hashCode = 31 * hashCode + Objects.hashCode(element); + + Ordinal ordinal = extractor.apply(element); + if (start == null) { + start = ordinal; + } else if (stop == null) { + stop = ordinal; + } else { + if (start.compareTo(ordinal) > 0) { + start = ordinal; + } + if (stop.compareTo(ordinal) < 0) { + stop = ordinal; + } + } + } + + /** + * Returns the latest element from the group that has its timestamp + * less than the given argument alongside its position in the list. + * The element and everything before it is removed. + */ + E trimBefore(Ordinal ordinal) { + Tuple match = findBefore(ordinal); + + // trim + if (match != null) { + elements.subList(0, match.v2() + 1).clear(); + + // update min time + if (elements.isEmpty() == false) { + start = extractor.apply(elements.get(0)); + } else { + start = null; + stop = null; + } + } + return match != null ? match.v1() : null; + } + + E before(Ordinal ordinal) { + Tuple match = findBefore(ordinal); + return match != null ? match.v1() : null; + } + + private Tuple findBefore(Ordinal ordinal) { + E match = null; + int matchPos = -1; + int position = -1; + for (E element : elements) { + position++; + Ordinal o = extractor.apply(element); + if (o.compareTo(ordinal) < 0) { + match = element; + matchPos = position; + } else { + break; + } + } + return match != null ? new Tuple<>(match, matchPos) : null; + } + + public boolean isEmpty() { + return elements.isEmpty(); + } + + @Override + public Iterator iterator() { + return new Iterator<>() { + final Iterator iter = elements.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public Ordinal next() { + return extractor.apply(iter.next()); + } + }; + } + + @Override + public int hashCode() { + return key.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + OrdinalGroup other = (OrdinalGroup) obj; + return Objects.equals(key, other.key) + && Objects.equals(hashCode, other.hashCode); + } + + @Override + public String toString() { + return format(null, "[{}][{}-{}]({} seqs)", key, start, stop, elements.size()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/Sequence.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/Sequence.java index b0f41c0aead4b..0298adbad4ab4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/Sequence.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/Sequence.java @@ -47,7 +47,7 @@ public int putMatch(int stage, SearchHit hit, Ordinal ordinal) { matches[currentStage] = new Match(ordinal, hit); return previousStage; } - throw new EqlIllegalArgumentException("Incorrect stage [{}] specified for Sequence[key={}, stage=]", stage, key, currentStage); + throw new EqlIllegalArgumentException("Incorrect stage [{}] specified for Sequence[key={}, stage={}]", stage, key, currentStage); } public SequenceKey key() { @@ -58,8 +58,8 @@ public Ordinal ordinal() { return matches[currentStage].ordinal(); } - public long startTimestamp() { - return matches[0].ordinal().timestamp(); + public Ordinal startOrdinal() { + return matches[0].ordinal(); } public List hits() { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceGroup.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceGroup.java index 1d48e8f6eb646..c7bd5a6d1f683 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceGroup.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceGroup.java @@ -6,122 +6,9 @@ package org.elasticsearch.xpack.eql.execution.sequence; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.xpack.eql.execution.search.Ordinal; - -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.function.Predicate; - -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; - -/** List of in-flight sequences for a given key. For fast lookup, typically associated with a stage. */ -public class SequenceGroup { - - private final SequenceKey key; - - // NB: since the size varies significantly, use a LinkedList - // Considering the order it might make sense to use a B-Tree+ for faster lookups which should work well with - // timestamp compression (whose range is known for the current frame). - private final List sequences = new LinkedList<>(); - - private Ordinal start, stop; +public class SequenceGroup extends OrdinalGroup { SequenceGroup(SequenceKey key) { - this.key = key; - } - - public void add(Sequence sequence) { - sequences.add(sequence); - Ordinal ordinal = sequence.ordinal(); - if (start == null) { - start = ordinal; - } else if (stop == null) { - stop = ordinal; - } else { - if (start.compareTo(ordinal) > 0) { - start = ordinal; - } - if (stop.compareTo(ordinal) < 0) { - stop = ordinal; - } - } - } - - /** - * Returns the latest Sequence from the group that has its timestamp - * less than the given argument alongside its position in the list. - */ - public Tuple before(Ordinal ordinal) { - return find(o -> o.compareTo(ordinal) < 0); - } - - /** - * Returns the first Sequence from the group that has its timestamp - * greater than the given argument alongside its position in the list. - */ - public Tuple after(Ordinal ordinal) { - return find(o -> o.compareTo(ordinal) > 0); - } - - private Tuple find(Predicate predicate) { - Sequence matchSeq = null; - int matchPos = -1; - int position = -1; - for (Sequence sequence : sequences) { - position++; - if (predicate.test(sequence.ordinal())) { - matchSeq = sequence; - matchPos = position; - } else { - break; - } - } - return matchSeq != null ? new Tuple<>(matchSeq, matchPos) : null; - } - - public boolean isEmpty() { - return sequences.isEmpty(); - } - - public void trim(int position) { - sequences.subList(0, position).clear(); - - // update min time - if (sequences.isEmpty() == false) { - start = sequences.get(0).ordinal(); - } else { - start = null; - stop = null; - } - } - - public List sequences() { - return sequences; - } - - @Override - public int hashCode() { - return key.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - SequenceGroup other = (SequenceGroup) obj; - return Objects.equals(key, other.key); - } - - @Override - public String toString() { - return format(null, "[{}][{}-{}]({} seqs)", key, start, stop, sequences.size()); + super(key, Sequence::ordinal); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java index 85a6b9119b6ce..f721efd6d6976 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java @@ -6,10 +6,10 @@ package org.elasticsearch.xpack.eql.execution.sequence; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.eql.execution.assembler.KeyAndOrdinal; import org.elasticsearch.xpack.eql.execution.search.Limit; import org.elasticsearch.xpack.eql.execution.search.Ordinal; @@ -21,6 +21,32 @@ */ public class SequenceStateMachine { + static class Stats { + long seen = 0; + long ignored = 0; + long until = 0; + long rejectionMaxspan = 0; + long rejectionUntil = 0; + + @Override + public String toString() { + return LoggerMessageFormat.format(null, "Stats: Seen [{}]/Ignored [{}]/Until [{}]/Rejected {Maxspan [{}]/Until [{}]}", + seen, + ignored, + until, + rejectionMaxspan, + rejectionUntil); + } + + public void clear() { + seen = 0; + ignored = 0; + until = 0; + rejectionMaxspan = 0; + rejectionUntil = 0; + } + } + /** Current sequences for each key */ /** Note will be multiple sequences for the same key and the same stage with different timestamps */ private final KeyToSequences keyToSequences; @@ -37,6 +63,8 @@ public class SequenceStateMachine { private int limit = -1; private boolean limitReached = false; + private final Stats stats = new Stats(); + @SuppressWarnings("rawtypes") public SequenceStateMachine(int stages, TimeValue maxSpan, Limit limit) { this.completionStage = stages - 1; @@ -61,8 +89,10 @@ public List completeSequences() { public void trackSequence(Sequence sequence) { SequenceKey key = sequence.key(); - stageToKeys.keys(0).add(key); + stageToKeys.add(0, key); keyToSequences.add(0, sequence); + + stats.seen++; } /** @@ -70,30 +100,52 @@ public void trackSequence(Sequence sequence) { * given stage. If that's the case, update the sequence and the rest of the references. */ public void match(int stage, SequenceKey key, Ordinal ordinal, SearchHit hit) { + stats.seen++; + int previousStage = stage - 1; // check key presence to avoid creating a collection SequenceGroup group = keyToSequences.groupIfPresent(previousStage, key); if (group == null || group.isEmpty()) { + stats.ignored++; return; } - Tuple before = group.before(ordinal); - if (before == null) { + + // eliminate the match and all previous values from the group + Sequence sequence = group.trimBefore(ordinal); + if (sequence == null) { + stats.ignored++; return; } - Sequence sequence = before.v1(); - // eliminate the match and all previous values from the frame - group.trim(before.v2() + 1); - // remove the frame and keys early (as the key space is large) + // remove the group early (as the key space is large) if (group.isEmpty()) { - stageToKeys.keys(previousStage).remove(key); + keyToSequences.remove(previousStage, group); + stageToKeys.remove(previousStage, key); } - // check maxspan before continuing the sequence - if (maxSpanInMillis > 0 && (ordinal.timestamp() - sequence.startTimestamp() > maxSpanInMillis)) { + // + // Conditional checks + // + + // maxspan + if (maxSpanInMillis > 0 && (ordinal.timestamp() - sequence.startOrdinal().timestamp() > maxSpanInMillis)) { + stats.rejectionMaxspan++; return; } + // until + UntilGroup until = keyToSequences.untilIfPresent(key); + if (until != null) { + KeyAndOrdinal nearestUntil = until.before(ordinal); + if (nearestUntil != null) { + // check if until matches + if (nearestUntil.ordinal().between(sequence.ordinal(), ordinal)) { + stats.rejectionUntil++; + return; + } + } + } + sequence.putMatch(stage, hit, ordinal); // bump the stages @@ -109,7 +161,7 @@ public void match(int stage, SequenceKey key, Ordinal ordinal, SearchHit hit) { } } } else { - stageToKeys.keys(stage).add(key); + stageToKeys.add(stage, key); keyToSequences.add(stage, sequence); } } @@ -127,17 +179,36 @@ public boolean reachedLimit() { */ public boolean hasCandidates(int stage) { for (int i = stage; i < completionStage; i++) { - if (stageToKeys.keys(i).isEmpty() == false) { + if (stageToKeys.isEmpty(i) == false) { return true; } } return false; } + public void dropUntil() { + keyToSequences.dropUntil(); + } + + public void until(Iterable markers) { + keyToSequences.until(markers); + } + + public Stats stats() { + return stats; + } + + public void clear() { + stats.clear(); + keyToSequences.clear(); + stageToKeys.clear(); + completed.clear(); + } + @Override public String toString() { return LoggerMessageFormat.format(null, "Tracking [{}] keys with [{}] completed and in-flight {}", - keyToSequences.numberOfKeys(), + keyToSequences, completed.size(), stageToKeys); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/StageToKeys.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/StageToKeys.java index 96ad5688c6496..c14d7e511c7c0 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/StageToKeys.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/StageToKeys.java @@ -23,14 +23,34 @@ class StageToKeys { this.stageToKey = Arrays.asList(new Set[stages]); } - Set keys(int stage) { + void add(int stage, SequenceKey key) { Set set = stageToKey.get(stage); if (set == null) { // TODO: could we use an allocation strategy? set = new LinkedHashSet<>(); stageToKey.set(stage, set); } - return set; + set.add(key); + } + + void remove(int stage, SequenceKey key) { + Set set = stageToKey.get(stage); + if (set != null) { + set.remove(key); + } + } + + boolean isEmpty(int stage) { + Set set = stageToKey.get(stage); + return set == null || set.isEmpty(); + } + + void clear() { + for (Set set : stageToKey) { + if (set != null) { + set.clear(); + } + } } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/UntilGroup.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/UntilGroup.java new file mode 100644 index 0000000000000..aa198ab929f64 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/UntilGroup.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.sequence; + +import org.elasticsearch.xpack.eql.execution.assembler.KeyAndOrdinal; + +public class UntilGroup extends OrdinalGroup { + + UntilGroup(SequenceKey key) { + super(key, KeyAndOrdinal::ordinal); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java index 674aac50649e8..ebbfa5f7aedba 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ParserParams.java @@ -12,7 +12,6 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.eql.action.RequestDefaults.FETCH_SIZE; import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_EVENT_CATEGORY; -import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_IMPLICIT_JOIN_KEY; import static org.elasticsearch.xpack.eql.action.RequestDefaults.FIELD_TIMESTAMP; import static org.elasticsearch.xpack.eql.action.RequestDefaults.SIZE; @@ -22,7 +21,6 @@ public class ParserParams { private String fieldEventCategory = FIELD_EVENT_CATEGORY; private String fieldTimestamp = FIELD_TIMESTAMP; private String fieldTiebreaker = null; - private String implicitJoinKey = FIELD_IMPLICIT_JOIN_KEY; private int size = SIZE; private int fetchSize = FETCH_SIZE; private List queryParams = emptyList(); @@ -58,15 +56,6 @@ public ParserParams fieldTiebreaker(String fieldTiebreaker) { return this; } - public String implicitJoinKey() { - return implicitJoinKey; - } - - public ParserParams implicitJoinKey(String implicitJoinKey) { - this.implicitJoinKey = implicitJoinKey; - return this; - } - public int size() { return size; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index e7d89ed4fcd50..92aa548bcf5e0 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -49,27 +49,9 @@ public class EqlPlugin extends Plugin implements ActionPlugin { private final boolean enabled; - private static final boolean EQL_FEATURE_FLAG_REGISTERED; - - static { - final String property = System.getProperty("es.eql_feature_flag_registered"); - if (Build.CURRENT.isSnapshot() && property != null) { - throw new IllegalArgumentException("es.eql_feature_flag_registered is only supported in non-snapshot builds"); - } - if ("true".equals(property)) { - EQL_FEATURE_FLAG_REGISTERED = true; - } else if ("false".equals(property) || property == null) { - EQL_FEATURE_FLAG_REGISTERED = false; - } else { - throw new IllegalArgumentException( - "expected es.eql_feature_flag_registered to be unset or [true|false] but was [" + property + "]" - ); - } - } - public static final Setting EQL_ENABLED_SETTING = Setting.boolSetting( "xpack.eql.enabled", - false, + true, Setting.Property.NodeScope ); @@ -99,10 +81,7 @@ private Collection createComponents(Client client, String clusterName, */ @Override public List> getSettings() { - if (isSnapshot() || EQL_FEATURE_FLAG_REGISTERED) { - return List.of(EQL_ENABLED_SETTING); - } - return List.of(); + return List.of(EQL_ENABLED_SETTING); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index d428cbec5bd1a..3280652f47fe6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -115,7 +115,6 @@ public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlS .fieldEventCategory(request.eventCategoryField()) .fieldTimestamp(request.timestampField()) .fieldTiebreaker(request.tiebreakerField()) - .implicitJoinKey(request.implicitJoinKeyField()) .size(request.size()) .fetchSize(request.fetchSize()); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java index 02d2952c2f4ba..72424aee3cb0a 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java @@ -37,11 +37,6 @@ public void testSearchRequestParser() throws IOException { EqlSearchRequest::fromXContent); assertParsingErrorMessage("{\"event_category_field\" : 123}", "event_category_field doesn't support values of type: VALUE_NUMBER", EqlSearchRequest::fromXContent); - assertParsingErrorMessage("{\"implicit_join_key_field\" : 123}", - "implicit_join_key_field doesn't support values of type: VALUE_NUMBER", - EqlSearchRequest::fromXContent); - assertParsingErrorMessage("{\"search_after\" : 123}", "search_after doesn't support values of type: VALUE_NUMBER", - EqlSearchRequest::fromXContent); assertParsingErrorMessage("{\"size\" : \"foo\"}", "failed to parse field [size]", EqlSearchRequest::fromXContent); assertParsingErrorMessage("{\"query\" : 123}", "query doesn't support values of type: VALUE_NUMBER", EqlSearchRequest::fromXContent); @@ -55,8 +50,6 @@ public void testSearchRequestParser() throws IOException { EqlSearchRequest request = generateRequest("endgame-*", "{\"filter\" : {\"match\" : {\"foo\":\"bar\"}}, " + "\"timestamp_field\" : \"tsf\", " + "\"event_category_field\" : \"etf\"," - + "\"implicit_join_key_field\" : \"imjf\"," - + "\"search_after\" : [ 12345678, \"device-20184\", \"/user/local/foo.exe\", \"2019-11-26T00:45:43.542\" ]," + "\"size\" : \"101\"," + "\"query\" : \"file where user != 'SYSTEM' by file_path\"" + (setIsCaseSensitive ? (",\"case_sensitive\" : " + isCaseSensitive) : "") @@ -69,8 +62,6 @@ public void testSearchRequestParser() throws IOException { assertEquals("bar", filter.value()); assertEquals("tsf", request.timestampField()); assertEquals("etf", request.eventCategoryField()); - assertEquals("imjf", request.implicitJoinKeyField()); - assertArrayEquals(new Object[]{12345678, "device-20184", "/user/local/foo.exe", "2019-11-26T00:45:43.542"}, request.searchAfter()); assertEquals(101, request.size()); assertEquals(1000, request.fetchSize()); assertEquals("file where user != 'SYSTEM' by file_path", request.query()); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index 5cb576c9063b1..17c57beaf9f3d 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -5,19 +5,15 @@ */ package org.elasticsearch.xpack.eql.action; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.searchafter.SearchAfterBuilder; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -65,13 +61,9 @@ protected EqlSearchRequest createTestInstance() { .filter(filter) .timestampField(randomAlphaOfLength(10)) .eventCategoryField(randomAlphaOfLength(10)) - .implicitJoinKeyField(randomAlphaOfLength(10)) .fetchSize(randomIntBetween(1, 50)) .query(randomAlphaOfLength(10)); - if (randomBoolean()) { - request.searchAfter(randomJsonSearchFromBuilder()); - } return request; } catch (IOException ex) { assertNotNull("unexpected IOException " + ex.getCause().getMessage(), ex); @@ -105,24 +97,6 @@ private Object randomValue() { return value.get(); } - private Object[] randomJsonSearchFromBuilder() throws IOException { - int numSearchAfter = randomIntBetween(1, 10); - XContentBuilder jsonBuilder = XContentFactory.jsonBuilder(); - jsonBuilder.startObject(); - jsonBuilder.startArray("search_after"); - for (int i = 0; i < numSearchAfter; i++) { - jsonBuilder.value(randomValue()); - } - jsonBuilder.endArray(); - jsonBuilder.endObject(); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(jsonBuilder))) { - parser.nextToken(); - parser.nextToken(); - parser.nextToken(); - return SearchAfterBuilder.fromXContent(parser).getSortValues(); - } - } - @Override protected Writeable.Reader instanceReader() { return EqlSearchRequest::new; diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/plugin/EqlPluginTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/plugin/EqlPluginTests.java index 03f14247d77f1..e2cfffe71a1f9 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/plugin/EqlPluginTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/plugin/EqlPluginTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.not; public class EqlPluginTests extends ESTestCase { public void testEnabledSettingRegisteredInSnapshotBuilds() { @@ -34,6 +33,6 @@ protected boolean isSnapshot() { } }; - assertThat(plugin.getSettings(), not(hasItem(EqlPlugin.EQL_ENABLED_SETTING))); + assertThat(plugin.getSettings(), hasItem(EqlPlugin.EQL_ENABLED_SETTING)); } } diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java index 51d0eae703ad4..fb7933ca26d45 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SLMSnapshotBlockingIntegTests.java @@ -123,8 +123,8 @@ public void testSnapshotInProgress() throws Exception { SnapshotLifecyclePolicyItem.SnapshotInProgress inProgress = item.getSnapshotInProgress(); assertThat(inProgress.getSnapshotId().getName(), equalTo(snapshotName)); assertThat(inProgress.getStartTime(), greaterThan(0L)); - assertThat(inProgress.getState(), anyOf(equalTo(SnapshotsInProgress.State.INIT), equalTo(SnapshotsInProgress.State.STARTED), - equalTo(SnapshotsInProgress.State.SUCCESS))); + assertThat(inProgress.getState(), + anyOf(equalTo(SnapshotsInProgress.State.STARTED), equalTo(SnapshotsInProgress.State.SUCCESS))); assertNull(inProgress.getFailure()); }); diff --git a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java index d52968bd3b0f2..3c5817646319d 100644 --- a/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java +++ b/x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/slm/SnapshotRetentionTaskTests.java @@ -355,7 +355,8 @@ public void testOkToDeleteSnapshots() { SnapshotDeletionsInProgress delInProgress = SnapshotDeletionsInProgress.of( Collections.singletonList(new SnapshotDeletionsInProgress.Entry( - Collections.singletonList(snapshot.getSnapshotId()), snapshot.getRepository(), 0, 0))); + Collections.singletonList(snapshot.getSnapshotId()), snapshot.getRepository(), 0, 0, + SnapshotDeletionsInProgress.State.STARTED))); state = ClusterState.builder(new ClusterName("cluster")) .putCustom(SnapshotDeletionsInProgress.TYPE, delInProgress) .build(); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 24067a70e5c50..e034e2b4ae268 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -102,7 +102,9 @@ public Mapper.Builder parse(String name, Map node, ParserCont if (value != null) { builder.setValue(value.toString()); } - TypeParsers.parseMeta(builder, name, node); + if (node.containsKey("meta")) { + builder.meta(TypeParsers.parseMeta(name, node.remove("meta"))); + } return builder; } } @@ -120,29 +122,6 @@ public ConstantKeywordFieldType(String name, String value) { this(name, value, Collections.emptyMap()); } - protected ConstantKeywordFieldType(ConstantKeywordFieldType ref) { - super(ref); - this.value = ref.value; - } - - public ConstantKeywordFieldType clone() { - return new ConstantKeywordFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (super.equals(o) == false) { - return false; - } - ConstantKeywordFieldType other = (ConstantKeywordFieldType) o; - return Objects.equals(value, other.value); - } - - @Override - public int hashCode() { - return 31 * super.hashCode() + Objects.hashCode(value); - } - /** Return the value that this field wraps. This may be {@code null} if the field is not configured yet. */ public String value() { return value; @@ -152,11 +131,11 @@ public String value() { public String typeName() { return CONTENT_TYPE; } - + @Override public String familyTypeName() { return KeywordFieldMapper.CONTENT_TYPE; - } + } @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java index 2baf48b802043..3952e33a730f6 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldTypeTests.java @@ -11,19 +11,12 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.xpack.constantkeyword.mapper.ConstantKeywordFieldMapper.ConstantKeywordFieldType; import java.util.Arrays; import java.util.Collections; -import java.util.Map; -public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new ConstantKeywordFieldType(name, "foo", meta); - } +public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase { public void testTermQuery() { ConstantKeywordFieldType ft = new ConstantKeywordFieldType("f", "foo"); diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 2b77d7dbe3721..6fc8183bca421 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -237,34 +237,10 @@ public KeyedFlatObjectFieldType(String name, boolean indexed, boolean hasDocValu this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; } - public KeyedFlatObjectFieldType clone() { - return new KeyedFlatObjectFieldType(this); - } - - private KeyedFlatObjectFieldType(KeyedFlatObjectFieldType ref) { - super(ref); - this.key = ref.key; - this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace; - } - private KeyedFlatObjectFieldType(String name, String key, RootFlatObjectFieldType ref) { this(name, ref.isSearchable(), ref.hasDocValues(), key, ref.splitQueriesOnWhitespace, ref.meta()); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - KeyedFlatObjectFieldType that = (KeyedFlatObjectFieldType) o; - return splitQueriesOnWhitespace == that.splitQueriesOnWhitespace; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), splitQueriesOnWhitespace); - } - @Override public String typeName() { return CONTENT_TYPE; @@ -483,29 +459,6 @@ public RootFlatObjectFieldType(String name, boolean indexed, boolean hasDocValue setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } - private RootFlatObjectFieldType(RootFlatObjectFieldType ref) { - super(ref); - this.splitQueriesOnWhitespace = ref.splitQueriesOnWhitespace; - } - - public RootFlatObjectFieldType clone() { - return new RootFlatObjectFieldType(this); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - RootFlatObjectFieldType that = (RootFlatObjectFieldType) o; - return splitQueriesOnWhitespace == that.splitQueriesOnWhitespace; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), splitQueriesOnWhitespace); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java index 7622b4d2ad3f1..09d6afb6710e6 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlatObjectFieldTypeTests.java @@ -18,31 +18,19 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType; -import org.junit.Before; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; -public class KeyedFlatObjectFieldTypeTests extends FieldTypeTestCase { +public class KeyedFlatObjectFieldTypeTests extends FieldTypeTestCase { - @Before - public void addModifiers() { - addModifier(t -> { - KeyedFlatObjectFieldType copy = t.clone(); - copy.setSplitQueriesOnWhitespace(t.splitQueriesOnWhitespace() == false); - return copy; - }); - } - - @Override - protected KeyedFlatObjectFieldType createDefaultFieldType(String name, Map meta) { - return new KeyedFlatObjectFieldType(name, true, true, "key", false, meta); + protected KeyedFlatObjectFieldType createFieldType() { + return new KeyedFlatObjectFieldType("field", true, true, "key", false, Collections.emptyMap()); } public void testIndexedValueForSearch() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); BytesRef keywordValue = ft.indexedValueForSearch("value"); assertEquals(new BytesRef("key\0value"), keywordValue); @@ -55,7 +43,7 @@ public void testIndexedValueForSearch() { } public void testTermQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); Query expected = new TermQuery(new Term("field", "key\0value")); assertEquals(expected, ft.termQuery("value", null)); @@ -68,7 +56,7 @@ public void testTermQuery() { } public void testTermsQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); Query expected = new TermInSetQuery("field", new BytesRef("key\0value1"), @@ -83,14 +71,14 @@ public void testTermsQuery() { } public void testExistsQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); Query expected = new PrefixQuery(new Term("field", "key\0")); assertEquals(expected, ft.existsQuery(null)); } public void testPrefixQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); Query expected = new PrefixQuery(new Term("field", "key\0val")); assertEquals(expected, ft.prefixQuery("val", MultiTermQuery.CONSTANT_SCORE_REWRITE, MOCK_QSC)); @@ -102,7 +90,7 @@ public void testPrefixQuery() { } public void testFuzzyQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> ft.fuzzyQuery("value", Fuzziness.fromEdits(2), 1, 50, true, randomMockShardContext())); @@ -110,7 +98,7 @@ public void testFuzzyQuery() { } public void testRangeQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); TermRangeQuery expected = new TermRangeQuery("field", new BytesRef("key\0lower"), @@ -139,7 +127,7 @@ public void testRangeQuery() { } public void testRegexpQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> ft.regexpQuery("valu*", 0, 10, null, randomMockShardContext())); @@ -147,7 +135,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - KeyedFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + KeyedFlatObjectFieldType ft = createFieldType(); UnsupportedOperationException e = expectThrows(UnsupportedOperationException.class, () -> ft.wildcardQuery("valu*", null, randomMockShardContext())); diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java index b49fda1ea9fc3..3f049b003ddd6 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlatObjectFieldTypeTests.java @@ -20,29 +20,17 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.RootFlatObjectFieldType; -import org.junit.Before; import java.util.Collections; -import java.util.Map; -public class RootFlatObjectFieldTypeTests extends FieldTypeTestCase { +public class RootFlatObjectFieldTypeTests extends FieldTypeTestCase { - @Before - public void addModifiers() { - addModifier(t -> { - RootFlatObjectFieldType copy = t.clone(); - copy.setSplitQueriesOnWhitespace(t.splitQueriesOnWhitespace() == false); - return copy; - }); - } - - @Override - protected RootFlatObjectFieldType createDefaultFieldType(String name, Map meta) { - return new RootFlatObjectFieldType(name, true, true, meta, false); + protected RootFlatObjectFieldType createDefaultFieldType() { + return new RootFlatObjectFieldType("field", true, true, Collections.emptyMap(), false); } public void testValueForDisplay() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); String fieldValue = "{ \"key\": \"value\" }"; BytesRef storedValue = new BytesRef(fieldValue); @@ -50,7 +38,7 @@ public void testValueForDisplay() { } public void testTermQuery() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); Query expected = new TermQuery(new Term("field", "value")); assertEquals(expected, ft.termQuery("value", null)); @@ -73,7 +61,7 @@ public void testExistsQuery() { } public void testFuzzyQuery() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); Query expected = new FuzzyQuery(new Term("field", "value"), 2, 1, 50, true); Query actual = ft.fuzzyQuery("value", Fuzziness.fromEdits(2), 1, 50, true, MOCK_QSC); @@ -87,7 +75,7 @@ public void testFuzzyQuery() { } public void testRangeQuery() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); TermRangeQuery expected = new TermRangeQuery("field", new BytesRef("lower"), @@ -106,7 +94,7 @@ public void testRangeQuery() { } public void testRegexpQuery() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); Query expected = new RegexpQuery(new Term("field", "val.*")); Query actual = ft.regexpQuery("val.*", 0, 10, null, MOCK_QSC); @@ -119,7 +107,7 @@ public void testRegexpQuery() { } public void testWildcardQuery() { - RootFlatObjectFieldType ft = createDefaultFieldType("field", Collections.emptyMap()); + RootFlatObjectFieldType ft = createDefaultFieldType(); Query expected = new WildcardQuery(new Term("field", new BytesRef("valu*"))); assertEquals(expected, ft.wildcardQuery("valu*", null, MOCK_QSC)); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java index f8ba071e3cf15..ea2de1f8a21c2 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationEvaluationIT.java @@ -646,4 +646,9 @@ private static void indexDistinctAnimals(String indexName, int distinctAnimalCou fail("Failed to index data: " + bulkResponse.buildFailureMessage()); } } + + @Override + boolean supportsInference() { + return true; + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java index 92ec56fec29c4..1769e42c71a0a 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ClassificationIT.java @@ -890,4 +890,9 @@ private String stateDocId() { private String expectedDestIndexAuditMessage() { return (analysisUsesExistingDestIndex ? "Using existing" : "Creating") + " destination index [" + destIndex + "]"; } + + @Override + boolean supportsInference() { + return true; + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java index 28e2d7c183d7c..47a32be0871d2 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ExplainDataFrameAnalyticsIT.java @@ -123,7 +123,12 @@ public void testTrainingPercentageIsApplied() throws IOException { explainResponse = explainDataFrame(config); - assertThat(explainResponse.getMemoryEstimation().getExpectedMemoryWithoutDisk(), + assertThat(explainResponse.getMemoryEstimation().getExpectedMemoryWithoutDisk(), lessThanOrEqualTo(allDataUsedForTraining)); } + + @Override + boolean supportsInference() { + return false; + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java index d17cc272e2b46..c5ad8a30c7059 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java @@ -219,6 +219,8 @@ protected void assertProgressComplete(String id) { progress.stream().allMatch(phaseProgress -> phaseProgress.getProgressPercent() == 100), is(true)); } + abstract boolean supportsInference(); + private List getProgress(String id) { GetDataFrameAnalyticsStatsAction.Response.Stats stats = getAnalyticsStats(id); assertThat(stats.getId(), equalTo(id)); @@ -227,7 +229,12 @@ private List getProgress(String id) { assertThat(progress.size(), greaterThanOrEqualTo(4)); assertThat(progress.get(0).getPhase(), equalTo("reindexing")); assertThat(progress.get(1).getPhase(), equalTo("loading_data")); - assertThat(progress.get(progress.size() - 1).getPhase(), equalTo("writing_results")); + if (supportsInference()) { + assertThat(progress.get(progress.size() - 2).getPhase(), equalTo("writing_results")); + assertThat(progress.get(progress.size() - 1).getPhase(), equalTo("inference")); + } else { + assertThat(progress.get(progress.size() - 1).getPhase(), equalTo("writing_results")); + } return progress; } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java index 9a74aa7719ff9..9f11e61631095 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OutlierDetectionWithMissingFieldsIT.java @@ -111,4 +111,9 @@ public void testMissingFields() throws Exception { assertProgressComplete(id); assertThat(searchStoredProgress(id).getHits().getTotalHits().value, equalTo(1L)); } + + @Override + boolean supportsInference() { + return false; + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java index 35c86207afa64..a947fe66c03cb 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionEvaluationIT.java @@ -148,4 +148,9 @@ private static void indexHousesData(String indexName) { fail("Failed to index data: " + bulkResponse.buildFailureMessage()); } } + + @Override + boolean supportsInference() { + return true; + } } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java index 02094b1fafa77..9b786e3ae43d4 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RegressionIT.java @@ -60,6 +60,7 @@ public void cleanup() { cleanUp(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/59413") public void testSingleNumericFeatureAndMixedTrainingAndNonTrainingRows() throws Exception { initialize("regression_single_numeric_feature_and_mixed_data_set"); String predictedClassField = DEPENDENT_VARIABLE_FIELD + "_prediction"; @@ -538,4 +539,9 @@ private static Map getMlResultsObjectFromDestDoc(Map bucketPaths = new HashMap<>(); + bucketPaths.put("feature1", "avg_feature1"); + InferencePipelineAggregationBuilder inferenceAgg = + new InferencePipelineAggregationBuilder("infer_agg", new SetOnce<>(modelLoading), licenseState, bucketPaths); + inferenceAgg.setModelId(modelId); + + termsAgg.subAggregation(inferenceAgg); + + SearchRequest search = new SearchRequest(index); + search.source().aggregation(termsAgg); + client().search(search).actionGet(); + + // Pick a license that does not allow machine learning + License.OperationMode mode = randomInvalidLicenseType(); + enableLicensing(mode); + assertMLAllowed(false); + + // inferring against a model should now fail + SearchRequest invalidSearch = new SearchRequest(index); + invalidSearch.source().aggregation(termsAgg); + ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, + () -> client().search(invalidSearch).actionGet()); + + assertThat(e.status(), is(RestStatus.FORBIDDEN)); + assertThat(e.getMessage(), containsString("current license is non-compliant for [ml]")); + assertThat(e.getMetadata(LicenseUtils.EXPIRED_FEATURE_METADATA), hasItem(XPackField.MACHINE_LEARNING)); + } + private void putInferenceModel(String modelId) { TrainedModelConfig config = TrainedModelConfig.builder() .setParsedDefinition( @@ -675,13 +735,13 @@ private void putInferenceModel(String modelId) { .setTrainedModel( Tree.builder() .setTargetType(TargetType.REGRESSION) - .setFeatureNames(Arrays.asList("feature1")) + .setFeatureNames(Collections.singletonList("feature1")) .setNodes(TreeNode.builder(0).setLeafValue(1.0)) .build()) .setPreProcessors(Collections.emptyList())) .setModelId(modelId) .setDescription("test model for classification") - .setInput(new TrainedModelInput(Arrays.asList("feature1"))) + .setInput(new TrainedModelInput(Collections.singletonList("feature1"))) .setInferenceConfig(RegressionConfig.EMPTY_PARAMS) .build(); client().execute(PutTrainedModelAction.INSTANCE, new PutTrainedModelAction.Request(config)).actionGet(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index bfe46c467dec7..fed111f22bdf9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -706,8 +706,14 @@ public Collection createComponents(Client client, ClusterService cluster this.modelLoadingService.set(modelLoadingService); // Data frame analytics components - AnalyticsProcessManager analyticsProcessManager = new AnalyticsProcessManager(client, threadPool, analyticsProcessFactory, - dataFrameAnalyticsAuditor, trainedModelProvider, resultsPersisterService, EsExecutors.allocatedProcessors(settings)); + AnalyticsProcessManager analyticsProcessManager = new AnalyticsProcessManager(client, + threadPool, + analyticsProcessFactory, + dataFrameAnalyticsAuditor, + trainedModelProvider, + modelLoadingService, + resultsPersisterService, + EsExecutors.allocatedProcessors(settings)); MemoryUsageEstimationProcessManager memoryEstimationProcessManager = new MemoryUsageEstimationProcessManager( threadPool.generic(), threadPool.executor(MachineLearning.JOB_COMMS_THREAD_POOL_NAME), memoryEstimationProcessFactory); @@ -982,10 +988,9 @@ public Map> getTokenizers() { @Override public List getPipelineAggregations() { PipelineAggregationSpec spec = new PipelineAggregationSpec(InferencePipelineAggregationBuilder.NAME, - in -> new InferencePipelineAggregationBuilder(in, modelLoadingService), + in -> new InferencePipelineAggregationBuilder(in, getLicenseState(), modelLoadingService), (ContextParser) - (parser, name) -> InferencePipelineAggregationBuilder.parse(modelLoadingService, name, parser - )); + (parser, name) -> InferencePipelineAggregationBuilder.parse(modelLoadingService, getLicenseState(), name, parser)); spec.addResultReader(InternalInferenceAggregation::new); return Collections.singletonList(spec); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java index 03eff72478177..06a79c44ec6b4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetDataFrameAnalyticsStatsAction.java @@ -195,7 +195,7 @@ private void searchStats(DataFrameAnalyticsConfig config, ActionListener logger.debug("[{}] Gathering stats for stopped task", config.getId()); RetrievedStatsHolder retrievedStatsHolder = new RetrievedStatsHolder( - ProgressTracker.fromZeroes(config.getAnalysis().getProgressPhases()).report()); + ProgressTracker.fromZeroes(config.getAnalysis().getProgressPhases(), config.getAnalysis().supportsInference()).report()); MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); multiSearchRequest.add(buildStoredProgressSearch(config.getId())); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java index 66593f222e135..c94c668a87bde 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportInternalInferModelAction.java @@ -65,9 +65,14 @@ protected void doExecute(Task task, Request request, ActionListener li model.infer(stringObjectMap, request.getUpdate(), chainedTask))); typedChainTaskExecutor.execute(ActionListener.wrap( - inferenceResultsInterfaces -> - listener.onResponse(responseBuilder.setInferenceResults(inferenceResultsInterfaces).build()), - listener::onFailure + inferenceResultsInterfaces -> { + model.release(); + listener.onResponse(responseBuilder.setInferenceResults(inferenceResultsInterfaces).build()); + }, + e -> { + model.release(); + listener.onFailure(e); + } )); }, listener::onFailure diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index 9c0f81461b4d1..46babd2f1f48c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -84,7 +84,8 @@ public void execute(DataFrameAnalyticsTask task, DataFrameAnalyticsState current // At this point we have the config at hand and we can reset the progress tracker // to use the analyses phases. We preserve reindexing progress as if reindexing was // finished it will not be reset. - task.getStatsHolder().resetProgressTrackerPreservingReindexingProgress(config.getAnalysis().getProgressPhases()); + task.getStatsHolder().resetProgressTrackerPreservingReindexingProgress(config.getAnalysis().getProgressPhases(), + config.getAnalysis().supportsInference()); switch(currentState) { // If we are STARTED, it means the job was started because the start API was called. diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java index 011630d585a9a..fe20ef2cbad09 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java @@ -48,6 +48,11 @@ public final class DestinationIndex { public static final String ID_COPY = "ml__id_copy"; + /** + * The field that indicates whether a doc was used for training or not + */ + public static final String IS_TRAINING = "is_training"; + // Metadata fields static final String CREATION_DATE_MILLIS = "creation_date_in_millis"; static final String VERSION = "version"; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java index 28935782acb1f..9ccaf2ebb9db9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java @@ -19,6 +19,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.CachedSupplier; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -28,6 +29,7 @@ import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis; import org.elasticsearch.xpack.ml.dataframe.DestinationIndex; +import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitter; import org.elasticsearch.xpack.ml.extractor.ExtractedField; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; @@ -64,12 +66,14 @@ public class DataFrameDataExtractor { private boolean isCancelled; private boolean hasNext; private boolean searchHasShardFailure; + private final CachedSupplier crossValidationSplitter; DataFrameDataExtractor(Client client, DataFrameDataExtractorContext context) { this.client = Objects.requireNonNull(client); this.context = Objects.requireNonNull(context); hasNext = true; searchHasShardFailure = false; + this.crossValidationSplitter = new CachedSupplier<>(context.crossValidationSplitterFactory::create); } public Map getHeaders() { @@ -93,6 +97,7 @@ public Optional> next() throws IOException { if (!hasNext()) { throw new NoSuchElementException(); } + Optional> hits = scrollId == null ? Optional.ofNullable(initScroll()) : Optional.ofNullable(continueScroll()); if (!hits.isPresent()) { hasNext = false; @@ -162,7 +167,7 @@ private void setFetchSource(SearchRequestBuilder searchRequestBuilder) { } } - private List processSearchResponse(SearchResponse searchResponse) throws IOException { + private List processSearchResponse(SearchResponse searchResponse) { scrollId = searchResponse.getScrollId(); if (searchResponse.getHits().getHits().length == 0) { hasNext = false; @@ -202,7 +207,8 @@ private Row createRow(SearchHit hit) { } } } - return new Row(extractedValues, hit); + boolean isTraining = extractedValues == null ? false : crossValidationSplitter.get().isTraining(extractedValues); + return new Row(extractedValues, hit, isTraining); } private List continueScroll() throws IOException { @@ -307,14 +313,17 @@ public DataSummary(long rows, int cols) { public static class Row { - private SearchHit hit; + private final SearchHit hit; @Nullable - private String[] values; + private final String[] values; + + private final boolean isTraining; - private Row(String[] values, SearchHit hit) { + private Row(String[] values, SearchHit hit, boolean isTraining) { this.values = values; this.hit = hit; + this.isTraining = isTraining; } @Nullable @@ -330,6 +339,10 @@ public boolean shouldSkip() { return values == null; } + public boolean isTraining() { + return isTraining; + } + public int getChecksum() { return Arrays.hashCode(values); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java index 64ad4bed452e7..55d2afc65d60d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorContext.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml.dataframe.extractor; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; import java.util.List; @@ -22,9 +23,11 @@ public class DataFrameDataExtractorContext { final Map headers; final boolean includeSource; final boolean supportsRowsWithMissingValues; + final CrossValidationSplitterFactory crossValidationSplitterFactory; DataFrameDataExtractorContext(String jobId, ExtractedFields extractedFields, List indices, QueryBuilder query, int scrollSize, - Map headers, boolean includeSource, boolean supportsRowsWithMissingValues) { + Map headers, boolean includeSource, boolean supportsRowsWithMissingValues, + CrossValidationSplitterFactory crossValidationSplitterFactory) { this.jobId = Objects.requireNonNull(jobId); this.extractedFields = Objects.requireNonNull(extractedFields); this.indices = indices.toArray(new String[indices.size()]); @@ -33,5 +36,6 @@ public class DataFrameDataExtractorContext { this.headers = headers; this.includeSource = includeSource; this.supportsRowsWithMissingValues = supportsRowsWithMissingValues; + this.crossValidationSplitterFactory = Objects.requireNonNull(crossValidationSplitterFactory); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java index a699e16a7d602..693c52647f3e4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorFactory.java @@ -7,9 +7,13 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField; +import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory; +import org.elasticsearch.xpack.ml.extractor.ExtractedField; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; import java.util.Arrays; @@ -17,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class DataFrameDataExtractorFactory { @@ -25,19 +30,24 @@ public class DataFrameDataExtractorFactory { private final List indices; private final QueryBuilder sourceQuery; private final ExtractedFields extractedFields; + private final List requiredFields; private final Map headers; private final boolean supportsRowsWithMissingValues; + private final CrossValidationSplitterFactory crossValidationSplitterFactory; private DataFrameDataExtractorFactory(Client client, String analyticsId, List indices, QueryBuilder sourceQuery, - ExtractedFields extractedFields, Map headers, - boolean supportsRowsWithMissingValues) { + ExtractedFields extractedFields, List requiredFields, Map headers, + boolean supportsRowsWithMissingValues, + CrossValidationSplitterFactory crossValidationSplitterFactory) { this.client = Objects.requireNonNull(client); this.analyticsId = Objects.requireNonNull(analyticsId); this.indices = Objects.requireNonNull(indices); this.sourceQuery = Objects.requireNonNull(sourceQuery); this.extractedFields = Objects.requireNonNull(extractedFields); + this.requiredFields = Objects.requireNonNull(requiredFields); this.headers = headers; this.supportsRowsWithMissingValues = supportsRowsWithMissingValues; + this.crossValidationSplitterFactory = Objects.requireNonNull(crossValidationSplitterFactory); } public DataFrameDataExtractor newExtractor(boolean includeSource) { @@ -45,15 +55,22 @@ public DataFrameDataExtractor newExtractor(boolean includeSource) { analyticsId, extractedFields, indices, - QueryBuilders.boolQuery().filter(sourceQuery), + buildQuery(), 1000, headers, includeSource, - supportsRowsWithMissingValues + supportsRowsWithMissingValues, + crossValidationSplitterFactory ); return new DataFrameDataExtractor(client, context); } + private QueryBuilder buildQuery() { + BoolQueryBuilder query = QueryBuilders.boolQuery().filter(sourceQuery); + requiredFields.forEach(requiredField -> query.filter(QueryBuilders.existsQuery(requiredField.getName()))); + return query; + } + public ExtractedFields getExtractedFields() { return extractedFields; } @@ -71,7 +88,14 @@ public ExtractedFields getExtractedFields() { public static DataFrameDataExtractorFactory createForSourceIndices(Client client, String taskId, DataFrameAnalyticsConfig config, ExtractedFields extractedFields) { return new DataFrameDataExtractorFactory(client, taskId, Arrays.asList(config.getSource().getIndex()), - config.getSource().getParsedQuery(), extractedFields, config.getHeaders(), config.getAnalysis().supportsMissingValues()); + config.getSource().getParsedQuery(), extractedFields, config.getAnalysis().getRequiredFields(), config.getHeaders(), + config.getAnalysis().supportsMissingValues(), createCrossValidationSplitterFactory(client, config, extractedFields)); + } + + private static CrossValidationSplitterFactory createCrossValidationSplitterFactory(Client client, DataFrameAnalyticsConfig config, + ExtractedFields extractedFields) { + return new CrossValidationSplitterFactory(client, config, + extractedFields.getAllFields().stream().map(ExtractedField::getName).collect(Collectors.toList())); } /** @@ -93,7 +117,8 @@ public static void createForDestinationIndex(Client client, DataFrameDataExtractorFactory extractorFactory = new DataFrameDataExtractorFactory(client, config.getId(), Collections.singletonList(config.getDest().getIndex()), config.getSource().getParsedQuery(), extractedFields, - config.getHeaders(), config.getAnalysis().supportsMissingValues()); + config.getAnalysis().getRequiredFields(), config.getHeaders(), config.getAnalysis().supportsMissingValues(), + createCrossValidationSplitterFactory(client, config, extractedFields)); listener.onResponse(extractorFactory); }, listener::onFailure diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java index 827ade391aa3f..d56d3838b3994 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.BooleanFieldMapper; -import org.elasticsearch.index.mapper.TimestampFieldMapper; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; @@ -54,17 +53,15 @@ public class ExtractedFieldsDetector { */ private static final List IGNORE_FIELDS = Arrays.asList("_id", "_field_names", "_index", "_parent", "_routing", "_seq_no", "_source", "_type", "_uid", "_version", "_feature", "_ignored", "_nested_path", DestinationIndex.ID_COPY, - TimestampFieldMapper.NAME); + "_data_stream_timestamp"); - private final String[] index; private final DataFrameAnalyticsConfig config; private final int docValueFieldsLimit; private final FieldCapabilitiesResponse fieldCapabilitiesResponse; private final Map cardinalitiesForFieldsWithConstraints; - ExtractedFieldsDetector(String[] index, DataFrameAnalyticsConfig config, int docValueFieldsLimit, - FieldCapabilitiesResponse fieldCapabilitiesResponse, Map cardinalitiesForFieldsWithConstraints) { - this.index = Objects.requireNonNull(index); + ExtractedFieldsDetector(DataFrameAnalyticsConfig config, int docValueFieldsLimit, FieldCapabilitiesResponse fieldCapabilitiesResponse, + Map cardinalitiesForFieldsWithConstraints) { this.config = Objects.requireNonNull(config); this.docValueFieldsLimit = docValueFieldsLimit; this.fieldCapabilitiesResponse = Objects.requireNonNull(fieldCapabilitiesResponse); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java index 918999014d06c..2e46e5d5757c0 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorFactory.java @@ -68,8 +68,7 @@ private void create(String[] index, DataFrameAnalyticsConfig config, ActionListe ActionListener> fieldCardinalitiesHandler = ActionListener.wrap( fieldCardinalities -> { ExtractedFieldsDetector detector = - new ExtractedFieldsDetector( - index, config, docValueFieldsLimitHolder.get(), fieldCapsResponseHolder.get(), fieldCardinalities); + new ExtractedFieldsDetector(config, docValueFieldsLimitHolder.get(), fieldCapsResponseHolder.get(), fieldCardinalities); listener.onResponse(detector); }, listener::onFailure diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java new file mode 100644 index 0000000000000..5322484db6058 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.dataframe.inference; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.dataframe.DestinationIndex; +import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker; +import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker; +import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel; +import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; +import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService; + +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +public class InferenceRunner { + + private static final Logger LOGGER = LogManager.getLogger(InferenceRunner.class); + + private static final int MAX_PROGRESS_BEFORE_COMPLETION = 98; + private static final int RESULTS_BATCH_SIZE = 1000; + + private final Client client; + private final ModelLoadingService modelLoadingService; + private final ResultsPersisterService resultsPersisterService; + private final TaskId parentTaskId; + private final DataFrameAnalyticsConfig config; + private final ProgressTracker progressTracker; + private final DataCountsTracker dataCountsTracker; + private volatile boolean isCancelled; + + public InferenceRunner(Client client, ModelLoadingService modelLoadingService, ResultsPersisterService resultsPersisterService, + TaskId parentTaskId, DataFrameAnalyticsConfig config, ProgressTracker progressTracker, + DataCountsTracker dataCountsTracker) { + this.client = Objects.requireNonNull(client); + this.modelLoadingService = Objects.requireNonNull(modelLoadingService); + this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService); + this.parentTaskId = Objects.requireNonNull(parentTaskId); + this.config = Objects.requireNonNull(config); + this.progressTracker = Objects.requireNonNull(progressTracker); + this.dataCountsTracker = Objects.requireNonNull(dataCountsTracker); + } + + public void cancel() { + isCancelled = true; + } + + public void run(String modelId) { + if (isCancelled) { + return; + } + + LOGGER.info("[{}] Started inference on test data against model [{}]", config.getId(), modelId); + try { + PlainActionFuture localModelPlainActionFuture = new PlainActionFuture<>(); + modelLoadingService.getModelForSearch(modelId, localModelPlainActionFuture); + LocalModel localModel = localModelPlainActionFuture.actionGet(); + TestDocsIterator testDocsIterator = new TestDocsIterator(new OriginSettingClient(client, ClientHelper.ML_ORIGIN), config); + inferTestDocs(localModel, testDocsIterator); + } catch (Exception e) { + throw ExceptionsHelper.serverError("[{}] failed running inference on model [{}]", e, config.getId(), modelId); + } + } + + // Visible for testing + void inferTestDocs(LocalModel model, TestDocsIterator testDocsIterator) { + long totalDocCount = 0; + long processedDocCount = 0; + BulkRequest bulkRequest = new BulkRequest(); + + while (testDocsIterator.hasNext()) { + if (isCancelled) { + break; + } + + Deque batch = testDocsIterator.next(); + + if (totalDocCount == 0) { + totalDocCount = testDocsIterator.getTotalHits(); + } + + for (SearchHit doc : batch) { + dataCountsTracker.incrementTestDocsCount(); + InferenceResults inferenceResults = model.inferNoStats(new HashMap<>(doc.getSourceAsMap())); + bulkRequest.add(createIndexRequest(doc, inferenceResults, config.getDest().getResultsField())); + + processedDocCount++; + int progressPercent = Math.min((int) (processedDocCount * 100.0 / totalDocCount), MAX_PROGRESS_BEFORE_COMPLETION); + progressTracker.updateInferenceProgress(progressPercent); + } + + if (bulkRequest.numberOfActions() == RESULTS_BATCH_SIZE) { + executeBulkRequest(bulkRequest); + bulkRequest = new BulkRequest(); + } + } + if (bulkRequest.numberOfActions() > 0 && isCancelled == false) { + executeBulkRequest(bulkRequest); + } + + if (isCancelled == false) { + progressTracker.updateInferenceProgress(100); + } + } + + private IndexRequest createIndexRequest(SearchHit hit, InferenceResults results, String resultField) { + Map resultsMap = new LinkedHashMap<>(results.asMap()); + resultsMap.put(DestinationIndex.IS_TRAINING, false); + + Map source = new LinkedHashMap<>(hit.getSourceAsMap()); + source.put(resultField, resultsMap); + IndexRequest indexRequest = new IndexRequest(hit.getIndex()); + indexRequest.id(hit.getId()); + indexRequest.source(source); + indexRequest.opType(DocWriteRequest.OpType.INDEX); + indexRequest.setParentTask(parentTaskId); + return indexRequest; + } + + private void executeBulkRequest(BulkRequest bulkRequest) { + resultsPersisterService.bulkIndexWithHeadersWithRetry( + config.getHeaders(), + bulkRequest, + config.getId(), + () -> isCancelled == false, + errorMsg -> {}); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java new file mode 100644 index 0000000000000..05695fcb7d6ea --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/TestDocsIterator.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.dataframe.inference; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.OriginSettingClient; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.ml.dataframe.DestinationIndex; +import org.elasticsearch.xpack.ml.utils.persistence.SearchAfterDocumentsIterator; + +import java.util.Objects; + +public class TestDocsIterator extends SearchAfterDocumentsIterator { + + private final DataFrameAnalyticsConfig config; + private String lastDocId; + + TestDocsIterator(OriginSettingClient client, DataFrameAnalyticsConfig config) { + super(client, config.getDest().getIndex(), true); + this.config = Objects.requireNonNull(config); + } + + @Override + protected QueryBuilder getQuery() { + return QueryBuilders.boolQuery().mustNot( + QueryBuilders.termQuery(config.getDest().getResultsField() + "." + DestinationIndex.IS_TRAINING, true)); + } + + @Override + protected FieldSortBuilder sortField() { + return SortBuilders.fieldSort(DestinationIndex.ID_COPY).order(SortOrder.ASC); + } + + @Override + protected SearchHit map(SearchHit hit) { + return hit; + } + + @Override + protected Object[] searchAfterFields() { + return lastDocId == null ? null : new Object[] {lastDocId}; + } + + @Override + protected void extractSearchAfterFields(SearchHit lastSearchHit) { + lastDocId = lastSearchHit.getId(); + } + + @Override + protected SearchResponse executeSearchRequest(SearchRequest searchRequest) { + return ClientHelper.executeWithHeaders(config.getHeaders(), ClientHelper.ML_ORIGIN, client(), + () -> client().search(searchRequest).actionGet()); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java index 4366a19d5fe05..1991f93540984 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManager.java @@ -33,13 +33,13 @@ import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsTask; import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractor; import org.elasticsearch.xpack.ml.dataframe.extractor.DataFrameDataExtractorFactory; -import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitter; -import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory; +import org.elasticsearch.xpack.ml.dataframe.inference.InferenceRunner; import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult; import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker; import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker; import org.elasticsearch.xpack.ml.dataframe.stats.StatsPersister; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; +import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor; import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService; @@ -68,6 +68,7 @@ public class AnalyticsProcessManager { private final ConcurrentMap processContextByAllocation = new ConcurrentHashMap<>(); private final DataFrameAnalyticsAuditor auditor; private final TrainedModelProvider trainedModelProvider; + private final ModelLoadingService modelLoadingService; private final ResultsPersisterService resultsPersisterService; private final int numAllocatedProcessors; @@ -76,6 +77,7 @@ public AnalyticsProcessManager(Client client, AnalyticsProcessFactory analyticsProcessFactory, DataFrameAnalyticsAuditor auditor, TrainedModelProvider trainedModelProvider, + ModelLoadingService modelLoadingService, ResultsPersisterService resultsPersisterService, int numAllocatedProcessors) { this( @@ -85,6 +87,7 @@ public AnalyticsProcessManager(Client client, analyticsProcessFactory, auditor, trainedModelProvider, + modelLoadingService, resultsPersisterService, numAllocatedProcessors); } @@ -96,6 +99,7 @@ public AnalyticsProcessManager(Client client, AnalyticsProcessFactory analyticsProcessFactory, DataFrameAnalyticsAuditor auditor, TrainedModelProvider trainedModelProvider, + ModelLoadingService modelLoadingService, ResultsPersisterService resultsPersisterService, int numAllocatedProcessors) { this.client = Objects.requireNonNull(client); @@ -104,6 +108,7 @@ public AnalyticsProcessManager(Client client, this.processFactory = Objects.requireNonNull(analyticsProcessFactory); this.auditor = Objects.requireNonNull(auditor); this.trainedModelProvider = Objects.requireNonNull(trainedModelProvider); + this.modelLoadingService = Objects.requireNonNull(modelLoadingService); this.resultsPersisterService = Objects.requireNonNull(resultsPersisterService); this.numAllocatedProcessors = numAllocatedProcessors; } @@ -168,9 +173,7 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont AnalyticsResultProcessor resultProcessor = processContext.resultProcessor.get(); try { writeHeaderRecord(dataExtractor, process); - writeDataRows(dataExtractor, process, config, task); - processContext.statsPersister.persistWithRetry(task.getStatsHolder().getDataCountsTracker().report(config.getId()), - DataCounts::documentId); + writeDataRows(dataExtractor, process, task); process.writeEndOfDataMessage(); process.flushStream(); @@ -182,10 +185,15 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont LOGGER.info("[{}] Waiting for result processor to complete", config.getId()); resultProcessor.awaitForCompletion(); processContext.setFailureReason(resultProcessor.getFailure()); + LOGGER.info("[{}] Result processor has completed", config.getId()); + + runInference(parentTaskClient, task, processContext); + + processContext.statsPersister.persistWithRetry(task.getStatsHolder().getDataCountsTracker().report(config.getId()), + DataCounts::documentId); refreshDest(parentTaskClient, config); refreshIndices(parentTaskClient, config.getId()); - LOGGER.info("[{}] Result processor has completed", config.getId()); } catch (Exception e) { if (task.isStopping()) { // Errors during task stopping are expected but we still want to log them just in case. @@ -220,12 +228,9 @@ private void processData(DataFrameAnalyticsTask task, ProcessContext processCont } private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProcess process, - DataFrameAnalyticsConfig config, DataFrameAnalyticsTask task) throws IOException { + DataFrameAnalyticsTask task) throws IOException { ProgressTracker progressTracker = task.getStatsHolder().getProgressTracker(); DataCountsTracker dataCountsTracker = task.getStatsHolder().getDataCountsTracker(); - CrossValidationSplitter crossValidationSplitter = new CrossValidationSplitterFactory( - new ParentTaskAssigningClient(client, task.getParentTaskId()), config, dataExtractor.getFieldNames()) - .create(); // The extra fields are for the doc hash and the control field (should be an empty string) String[] record = new String[dataExtractor.getFieldNames().size() + 2]; @@ -245,9 +250,10 @@ private void writeDataRows(DataFrameDataExtractor dataExtractor, AnalyticsProces String[] rowValues = row.getValues(); System.arraycopy(rowValues, 0, record, 0, rowValues.length); record[record.length - 2] = String.valueOf(row.getChecksum()); - crossValidationSplitter.process(record, dataCountsTracker::incrementTrainingDocsCount, - dataCountsTracker::incrementTestDocsCount); - process.writeRecord(record); + if (row.isTraining()) { + dataCountsTracker.incrementTrainingDocsCount(); + process.writeRecord(record); + } } } rowsProcessed += rows.get().size(); @@ -315,6 +321,22 @@ private Consumer onProcessCrash(DataFrameAnalyticsTask task) { }; } + private void runInference(ParentTaskAssigningClient parentTaskClient, DataFrameAnalyticsTask task, ProcessContext processContext) { + if (processContext.failureReason.get() != null) { + // If there has been an error thus far let's not run inference at all + return; + } + + if (processContext.config.getAnalysis().supportsInference()) { + refreshDest(parentTaskClient, processContext.config); + InferenceRunner inferenceRunner = new InferenceRunner(parentTaskClient, modelLoadingService, resultsPersisterService, + task.getParentTaskId(), processContext.config, task.getStatsHolder().getProgressTracker(), + task.getStatsHolder().getDataCountsTracker()); + processContext.setInferenceRunner(inferenceRunner); + inferenceRunner.run(processContext.resultProcessor.get().getLatestModelId()); + } + } + private void refreshDest(ParentTaskAssigningClient parentTaskClient, DataFrameAnalyticsConfig config) { ClientHelper.executeWithHeaders(config.getHeaders(), ClientHelper.ML_ORIGIN, parentTaskClient, () -> parentTaskClient.execute(RefreshAction.INSTANCE, new RefreshRequest(config.getDest().getIndex())).actionGet()); @@ -375,6 +397,7 @@ class ProcessContext { private final SetOnce> process = new SetOnce<>(); private final SetOnce dataExtractor = new SetOnce<>(); private final SetOnce resultProcessor = new SetOnce<>(); + private final SetOnce inferenceRunner = new SetOnce<>(); private final SetOnce failureReason = new SetOnce<>(); private final StatsPersister statsPersister; @@ -395,6 +418,10 @@ void setFailureReason(String failureReason) { this.failureReason.trySet(failureReason); } + void setInferenceRunner(InferenceRunner inferenceRunner) { + this.inferenceRunner.set(inferenceRunner); + } + synchronized void stop() { LOGGER.debug("[{}] Stopping process", config.getId()); if (dataExtractor.get() != null) { @@ -403,6 +430,9 @@ synchronized void stop() { if (resultProcessor.get() != null) { resultProcessor.get().cancel(); } + if (inferenceRunner.get() != null) { + inferenceRunner.get().cancel(); + } statsPersister.cancel(); if (process.get() != null) { try { @@ -443,7 +473,7 @@ private AnalyticsProcessConfig createProcessConfig(DataFrameDataExtractor dataEx ExtractedFields extractedFields) { DataFrameDataExtractor.DataSummary dataSummary = dataExtractor.collectDataSummary(); Set categoricalFields = dataExtractor.getCategoricalFields(config.getAnalysis()); - int threads = config.getMaxNumThreads() == null ? 1 : Math.min(config.getMaxNumThreads(), numAllocatedProcessors); + int threads = Math.min(config.getMaxNumThreads(), numAllocatedProcessors); return new AnalyticsProcessConfig( config.getId(), dataSummary.rows, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java index 815540ac958e4..32af30a93da4a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java @@ -31,7 +31,6 @@ import java.util.Objects; import java.util.concurrent.CountDownLatch; - public class AnalyticsResultProcessor { private static final Logger LOGGER = LogManager.getLogger(AnalyticsResultProcessor.class); @@ -58,6 +57,8 @@ public class AnalyticsResultProcessor { private volatile String failure; private volatile boolean isCancelled; + private volatile String latestModelId; + public AnalyticsResultProcessor(DataFrameAnalyticsConfig analytics, DataFrameRowsJoiner dataFrameRowsJoiner, StatsHolder statsHolder, TrainedModelProvider trainedModelProvider, DataFrameAnalyticsAuditor auditor, StatsPersister statsPersister, ExtractedFields extractedFields) { @@ -153,7 +154,7 @@ private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJo } ModelSizeInfo modelSize = result.getModelSizeInfo(); if (modelSize != null) { - chunkedTrainedModelPersister.createAndIndexInferenceModelMetadata(modelSize); + latestModelId = chunkedTrainedModelPersister.createAndIndexInferenceModelMetadata(modelSize); } TrainedModelDefinitionChunk trainedModelDefinitionChunk = result.getTrainedModelDefinitionChunk(); if (trainedModelDefinitionChunk != null) { @@ -190,4 +191,9 @@ private void processMemoryUsage(MemoryUsage memoryUsage) { statsHolder.setMemoryUsage(memoryUsage); statsPersister.persistWithRetry(memoryUsage, memoryUsage::documentId); } + + @Nullable + public String getLatestModelId() { + return latestModelId; + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java index 58e2227f4dad0..725627ab21cf8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java @@ -95,12 +95,12 @@ public void createAndIndexInferenceModelDoc(TrainedModelDefinitionChunk trainedM } } - public void createAndIndexInferenceModelMetadata(ModelSizeInfo inferenceModelSize) { + public String createAndIndexInferenceModelMetadata(ModelSizeInfo inferenceModelSize) { if (readyToStoreNewModel.compareAndSet(true, false) == false) { failureHandler.accept(ExceptionsHelper.serverError( "new inference model is attempting to be stored before completion previous model storage" )); - return; + return null; } TrainedModelConfig trainedModelConfig = createTrainedModelConfig(inferenceModelSize); CountDownLatch latch = storeTrainedModelMetadata(trainedModelConfig); @@ -113,6 +113,7 @@ public void createAndIndexInferenceModelMetadata(ModelSizeInfo inferenceModelSiz this.readyToStoreNewModel.set(true); failureHandler.accept(ExceptionsHelper.serverError("interrupted waiting for inference model metadata to be stored")); } + return trainedModelConfig.getModelId(); } private CountDownLatch storeTrainedModelDoc(TrainedModelDefinitionDoc trainedModelDefinitionDoc) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java index 1bab306bab10e..8ec023ec573d9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoiner.java @@ -162,17 +162,21 @@ public boolean hasNext() { @Override public DataFrameDataExtractor.Row next() { DataFrameDataExtractor.Row row = null; - while ((row == null || row.shouldSkip()) && hasNext()) { + while (hasNoMatch(row) && hasNext()) { advanceToNextBatchIfNecessary(); row = currentDataFrameRows.get(currentDataFrameRowsIndex++); } - if (row == null || row.shouldSkip()) { + if (hasNoMatch(row)) { throw ExceptionsHelper.serverError("no more data frame rows could be found while joining results"); } return row; } + private boolean hasNoMatch(DataFrameDataExtractor.Row row) { + return row == null || row.shouldSkip() || row.isTraining() == false; + } + private void advanceToNextBatchIfNecessary() { if (currentDataFrameRowsIndex >= currentDataFrameRows.size()) { currentDataFrameRows = getNextDataRowsBatch().orElse(Collections.emptyList()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/AbstractReservoirCrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/AbstractReservoirCrossValidationSplitter.java index 33837c0f622a5..f2ac127e5b765 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/AbstractReservoirCrossValidationSplitter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/AbstractReservoirCrossValidationSplitter.java @@ -40,11 +40,10 @@ private static int findDependentVariableIndex(List fieldNames, String de } @Override - public void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs) { + public boolean isTraining(String[] row) { if (canBeUsedForTraining(row) == false) { - incrementTestDocs.run(); - return; + return false; } SampleInfo sample = getSampleInfo(row); @@ -62,11 +61,10 @@ public void process(String[] row, Runnable incrementTrainingDocs, Runnable incre sample.observed++; if (isTraining) { sample.training++; - incrementTrainingDocs.run(); - } else { - row[dependentVariableIndex] = DataFrameDataExtractor.NULL_VALUE; - incrementTestDocs.run(); + return true; } + + return false; } private boolean canBeUsedForTraining(String[] row) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java index fce602b28e2c5..5bd6a53e9843b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitter.java @@ -10,5 +10,5 @@ */ public interface CrossValidationSplitter { - void process(String[] row, Runnable incrementTrainingDocs, Runnable incrementTestDocs); + boolean isTraining(String[] row); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java index 6701de1bf346f..3408a6428040a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/CrossValidationSplitterFactory.java @@ -47,7 +47,7 @@ public CrossValidationSplitter create() { if (config.getAnalysis() instanceof Classification) { return createStratifiedSplitter((Classification) config.getAnalysis()); } - return (row, incrementTrainingDocs, incrementTestDocs) -> incrementTrainingDocs.run(); + return row -> true; } private CrossValidationSplitter createSingleClassSplitter(Regression regression) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTracker.java index a4bf3cf66c4ab..691836d26a438 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTracker.java @@ -24,16 +24,20 @@ public class ProgressTracker { public static final String REINDEXING = "reindexing"; public static final String LOADING_DATA = "loading_data"; public static final String WRITING_RESULTS = "writing_results"; + public static final String INFERENCE = "inference"; private final String[] phasesInOrder; private final Map progressPercentPerPhase; - public static ProgressTracker fromZeroes(List analysisProgressPhases) { - List phases = new ArrayList<>(3 + analysisProgressPhases.size()); + public static ProgressTracker fromZeroes(List analysisProgressPhases, boolean hasInferencePhase) { + List phases = new ArrayList<>(3 + analysisProgressPhases.size() + (hasInferencePhase ? 1 : 0)); phases.add(new PhaseProgress(REINDEXING, 0)); phases.add(new PhaseProgress(LOADING_DATA, 0)); analysisProgressPhases.forEach(analysisPhase -> phases.add(new PhaseProgress(analysisPhase, 0))); phases.add(new PhaseProgress(WRITING_RESULTS, 0)); + if (hasInferencePhase) { + phases.add(new PhaseProgress(INFERENCE, 0)); + } return new ProgressTracker(phases); } @@ -76,6 +80,14 @@ public int getWritingResultsProgressPercent() { return progressPercentPerPhase.get(WRITING_RESULTS); } + public void updateInferenceProgress(int progressPercent) { + updatePhase(INFERENCE, progressPercent); + } + + public int getInferenceProgressPercent() { + return progressPercentPerPhase.getOrDefault(INFERENCE, 0); + } + public void updatePhase(PhaseProgress phase) { updatePhase(phase.getPhase(), phase.getProgressPercent()); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java index d1724bd2f1123..2adb206a5d040 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolder.java @@ -30,9 +30,9 @@ public StatsHolder(List progressOnStart) { dataCountsTracker = new DataCountsTracker(); } - public void resetProgressTrackerPreservingReindexingProgress(List analysisPhases) { + public void resetProgressTrackerPreservingReindexingProgress(List analysisPhases, boolean hasInferencePhase) { int reindexingProgressPercent = progressTracker.getReindexingProgressPercent(); - progressTracker = ProgressTracker.fromZeroes(analysisPhases); + progressTracker = ProgressTracker.fromZeroes(analysisPhases, hasInferencePhase); progressTracker.updateReindexingProgress(reindexingProgressPercent); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilder.java index 9ca041c6b73ee..940b6a7178aef 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilder.java @@ -10,15 +10,17 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate; @@ -44,10 +46,10 @@ public class InferencePipelineAggregationBuilder extends AbstractPipelineAggrega static final String AGGREGATIONS_RESULTS_FIELD = "value"; @SuppressWarnings("unchecked") - private static final ConstructingObjectParser, String>> PARSER = new ConstructingObjectParser<>( - NAME, false, - (args, context) -> new InferencePipelineAggregationBuilder(context.v2(), context.v1(), (Map) args[0]) + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(NAME, false, + (args, context) -> new InferencePipelineAggregationBuilder(context.name, context.modelLoadingService, + context.licenseState, (Map) args[0]) ); static { @@ -60,34 +62,52 @@ public class InferencePipelineAggregationBuilder extends AbstractPipelineAggrega private final Map bucketPathMap; private String modelId; private InferenceConfigUpdate inferenceConfig; + private final XPackLicenseState licenseState; private final SetOnce modelLoadingService; /** * The model. Set to a non-null value during the rewrite phase. */ private final Supplier model; + private static class ParserSupplement { + final XPackLicenseState licenseState; + final SetOnce modelLoadingService; + final String name; + + ParserSupplement(String name, XPackLicenseState licenseState, SetOnce modelLoadingService) { + this.name = name; + this.licenseState = licenseState; + this.modelLoadingService = modelLoadingService; + } + } public static InferencePipelineAggregationBuilder parse(SetOnce modelLoadingService, + XPackLicenseState licenseState, String pipelineAggregatorName, XContentParser parser) { - Tuple, String> context = new Tuple<>(modelLoadingService, pipelineAggregatorName); - return PARSER.apply(parser, context); + return PARSER.apply(parser, new ParserSupplement(pipelineAggregatorName, licenseState, modelLoadingService)); } - public InferencePipelineAggregationBuilder(String name, SetOnce modelLoadingService, + public InferencePipelineAggregationBuilder(String name, + SetOnce modelLoadingService, + XPackLicenseState licenseState, Map bucketsPath) { super(name, NAME, new TreeMap<>(bucketsPath).values().toArray(new String[] {})); this.modelLoadingService = modelLoadingService; this.bucketPathMap = bucketsPath; this.model = null; + this.licenseState = licenseState; } - public InferencePipelineAggregationBuilder(StreamInput in, SetOnce modelLoadingService) throws IOException { + public InferencePipelineAggregationBuilder(StreamInput in, + XPackLicenseState licenseState, + SetOnce modelLoadingService) throws IOException { super(in, NAME); modelId = in.readString(); bucketPathMap = in.readMap(StreamInput::readString, StreamInput::readString); inferenceConfig = in.readOptionalNamedWriteable(InferenceConfigUpdate.class); this.modelLoadingService = modelLoadingService; this.model = null; + this.licenseState = licenseState; } /** @@ -98,7 +118,8 @@ private InferencePipelineAggregationBuilder( Map bucketsPath, Supplier model, String modelId, - InferenceConfigUpdate inferenceConfig + InferenceConfigUpdate inferenceConfig, + XPackLicenseState licenseState ) { super(name, NAME, new TreeMap<>(bucketsPath).values().toArray(new String[] {})); modelLoadingService = null; @@ -113,13 +134,14 @@ private InferencePipelineAggregationBuilder( */ this.modelId = modelId; this.inferenceConfig = inferenceConfig; + this.licenseState = licenseState; } - void setModelId(String modelId) { + public void setModelId(String modelId) { this.modelId = modelId; } - void setInferenceConfig(InferenceConfigUpdate inferenceConfig) { + public void setInferenceConfig(InferenceConfigUpdate inferenceConfig) { this.inferenceConfig = inferenceConfig; } @@ -160,7 +182,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { } @Override - public InferencePipelineAggregationBuilder rewrite(QueryRewriteContext context) throws IOException { + public InferencePipelineAggregationBuilder rewrite(QueryRewriteContext context) { if (model != null) { return this; } @@ -168,10 +190,17 @@ public InferencePipelineAggregationBuilder rewrite(QueryRewriteContext context) context.registerAsyncAction((client, listener) -> { modelLoadingService.get().getModelForSearch(modelId, ActionListener.delegateFailure(listener, (delegate, model) -> { loadedModel.set(model); - delegate.onResponse(null); + + boolean isLicensed = licenseState.checkFeature(XPackLicenseState.Feature.MACHINE_LEARNING) || + licenseState.isAllowedByLicense(model.getLicenseLevel()); + if (isLicensed) { + delegate.onResponse(null); + } else { + delegate.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); + } })); }); - return new InferencePipelineAggregationBuilder(name, bucketPathMap, loadedModel::get, modelId, inferenceConfig); + return new InferencePipelineAggregationBuilder(name, bucketPathMap, loadedModel::get, modelId, inferenceConfig, licenseState); } @Override diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregator.java index 8a0eb04df3447..8d4aa5c478538 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregator.java @@ -49,69 +49,74 @@ public InferencePipelineAggregator(String name, Map originalAgg = - (InternalMultiBucketAggregation) aggregation; - List buckets = originalAgg.getBuckets(); - - List newBuckets = new ArrayList<>(); - for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { - Map inputFields = new HashMap<>(); - - if (bucket.getDocCount() == 0) { - // ignore this empty bucket unless the doc count is used - if (bucketPathMap.containsKey("_count") == false) { - newBuckets.add(bucket); - continue; + try (model) { + InternalMultiBucketAggregation originalAgg = + (InternalMultiBucketAggregation) aggregation; + List buckets = originalAgg.getBuckets(); + + List newBuckets = new ArrayList<>(); + for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { + Map inputFields = new HashMap<>(); + + if (bucket.getDocCount() == 0) { + // ignore this empty bucket unless the doc count is used + if (bucketPathMap.containsKey("_count") == false) { + newBuckets.add(bucket); + continue; + } } - } - for (Map.Entry entry : bucketPathMap.entrySet()) { - String aggName = entry.getKey(); - String bucketPath = entry.getValue(); - Object propertyValue = resolveBucketValue(originalAgg, bucket, bucketPath); - - if (propertyValue instanceof Number) { - double doubleVal = ((Number) propertyValue).doubleValue(); - // NaN or infinite values indicate a missing value or a - // valid result of an invalid calculation. Either way only - // a valid number will do - if (Double.isFinite(doubleVal)) { - inputFields.put(aggName, doubleVal); + for (Map.Entry entry : bucketPathMap.entrySet()) { + String aggName = entry.getKey(); + String bucketPath = entry.getValue(); + Object propertyValue = resolveBucketValue(originalAgg, bucket, bucketPath); + + if (propertyValue instanceof Number) { + double doubleVal = ((Number) propertyValue).doubleValue(); + // NaN or infinite values indicate a missing value or a + // valid result of an invalid calculation. Either way only + // a valid number will do + if (Double.isFinite(doubleVal)) { + inputFields.put(aggName, doubleVal); + } + } else if (propertyValue instanceof InternalNumericMetricsAggregation.SingleValue) { + double doubleVal = ((InternalNumericMetricsAggregation.SingleValue) propertyValue).value(); + if (Double.isFinite(doubleVal)) { + inputFields.put(aggName, doubleVal); + } + } else if (propertyValue instanceof StringTerms.Bucket) { + StringTerms.Bucket b = (StringTerms.Bucket) propertyValue; + inputFields.put(aggName, b.getKeyAsString()); + } else if (propertyValue instanceof String) { + inputFields.put(aggName, propertyValue); + } else if (propertyValue != null) { + // Doubles, String terms or null are valid, any other type is an error + throw invalidAggTypeError(bucketPath, propertyValue); } - } else if (propertyValue instanceof InternalNumericMetricsAggregation.SingleValue) { - double doubleVal = ((InternalNumericMetricsAggregation.SingleValue) propertyValue).value(); - if (Double.isFinite(doubleVal)) { - inputFields.put(aggName, doubleVal); - } - } else if (propertyValue instanceof StringTerms.Bucket) { - StringTerms.Bucket b = (StringTerms.Bucket) propertyValue; - inputFields.put(aggName, b.getKeyAsString()); - } else if (propertyValue instanceof String) { - inputFields.put(aggName, propertyValue); - } else if (propertyValue != null) { - // Doubles, String terms or null are valid, any other type is an error - throw invalidAggTypeError(bucketPath, propertyValue); } - } - InferenceResults inference; - try { - inference = model.infer(inputFields, configUpdate); - } catch (Exception e) { - inference = new WarningInferenceResults(e.getMessage()); + InferenceResults inference; + try { + inference = model.infer(inputFields, configUpdate); + } catch (Exception e) { + inference = new WarningInferenceResults(e.getMessage()); + } + + final List aggs = bucket.getAggregations().asList().stream().map( + (p) -> (InternalAggregation) p).collect(Collectors.toList()); + + InternalInferenceAggregation aggResult = new InternalInferenceAggregation(name(), metadata(), inference); + aggs.add(aggResult); + InternalMultiBucketAggregation.InternalBucket newBucket = originalAgg.createBucket(InternalAggregations.from(aggs), bucket); + newBuckets.add(newBucket); } - final List aggs = bucket.getAggregations().asList().stream().map( - (p) -> (InternalAggregation) p).collect(Collectors.toList()); + // the model is released at the end of this block. + assert model.getReferenceCount() > 0; - InternalInferenceAggregation aggResult = new InternalInferenceAggregation(name(), metadata(), inference); - aggs.add(aggResult); - InternalMultiBucketAggregation.InternalBucket newBucket = originalAgg.createBucket(InternalAggregations.from(aggs), bucket); - newBuckets.add(newBucket); + return originalAgg.create(newBuckets); } - - return originalAgg.create(newBuckets); } public static Object resolveBucketValue(MultiBucketsAggregation agg, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java index cc830ab16fba3..ce4591a7abfe8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModel.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.ml.inference.loadingservice; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.license.License; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults; @@ -18,16 +20,29 @@ import org.elasticsearch.xpack.core.ml.utils.MapHelper; import org.elasticsearch.xpack.ml.inference.TrainedModelStatsService; +import java.io.Closeable; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import static org.elasticsearch.xpack.core.ml.job.messages.Messages.INFERENCE_WARNING_ALL_FIELDS_MISSING; -public class LocalModel { +/** + * LocalModels implement reference counting for proper accounting in + * the {@link CircuitBreaker}. When the model is not longer used {@link #release()} + * must be called and if the reference count == 0 then the model's bytes + * will be removed from the circuit breaker. + * + * The class is constructed with an initial reference count of 1 and its + * bytes must have been added to the circuit breaker before construction. + * New references must call {@link #acquire()} and {@link #release()} as the model + * is used. + */ +public class LocalModel implements Closeable { private final InferenceDefinition trainedModelDefinition; private final String modelId; @@ -38,14 +53,19 @@ public class LocalModel { private volatile long persistenceQuotient = 100; private final LongAdder currentInferenceCount; private final InferenceConfig inferenceConfig; + private final License.OperationMode licenseLevel; + private final CircuitBreaker trainedModelCircuitBreaker; + private final AtomicLong referenceCount; - public LocalModel(String modelId, + LocalModel(String modelId, String nodeId, InferenceDefinition trainedModelDefinition, TrainedModelInput input, Map defaultFieldMap, InferenceConfig modelInferenceConfig, - TrainedModelStatsService trainedModelStatsService) { + License.OperationMode licenseLevel, + TrainedModelStatsService trainedModelStatsService, + CircuitBreaker trainedModelCircuitBreaker) { this.trainedModelDefinition = trainedModelDefinition; this.modelId = modelId; this.fieldNames = new HashSet<>(input.getFieldNames()); @@ -56,6 +76,9 @@ public LocalModel(String modelId, this.defaultFieldMap = defaultFieldMap == null ? null : new HashMap<>(defaultFieldMap); this.currentInferenceCount = new LongAdder(); this.inferenceConfig = modelInferenceConfig; + this.licenseLevel = licenseLevel; + this.trainedModelCircuitBreaker = trainedModelCircuitBreaker; + this.referenceCount = new AtomicLong(1); } long ramBytesUsed() { @@ -66,6 +89,10 @@ public String getModelId() { return modelId; } + public License.OperationMode getLicenseLevel() { + return licenseLevel; + } + public InferenceStats getLatestStatsAndReset() { return statsAccumulator.currentStatsAndReset(); } @@ -80,6 +107,20 @@ void persistStats(boolean flush) { } } + /** + * Infers without updating the stats. + * This is mainly for usage by data frame analytics jobs + * when they do inference against test data. + */ + public InferenceResults inferNoStats(Map fields) { + LocalModel.mapFieldsIfNecessary(fields, defaultFieldMap); + Map flattenedFields = MapHelper.dotCollapse(fields, fieldNames); + if (flattenedFields.isEmpty()) { + new WarningInferenceResults(Messages.getMessage(INFERENCE_WARNING_ALL_FIELDS_MISSING, modelId)); + } + return trainedModelDefinition.infer(flattenedFields, inferenceConfig); + } + public void infer(Map fields, InferenceConfigUpdate update, ActionListener listener) { if (update.isSupported(this.inferenceConfig) == false) { listener.onFailure(ExceptionsHelper.badRequestException( @@ -155,4 +196,36 @@ public static void mapFieldsIfNecessary(Map fields, Map= 0; + if (count == 0) { + // no references to this model, it no longer needs to be accounted for + trainedModelCircuitBreaker.addWithoutBreaking(-ramBytesUsed()); + } + return referenceCount.get(); + } + + /** + * Convenience method so the class can be used in try-with-resource + * constructs to invoke {@link #release()}. + */ + public void close() { + release(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java index 590240556cebc..89b99ab98308a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingService.java @@ -60,6 +60,12 @@ * mode the model is not cached. All other uses will cache the model * * If more than one processor references the same model, that model will only be cached once. + * + * LocalModels are created with a reference count of 1 accounting for the reference the + * cache holds. When models are evicted from the cache the reference count is decremented. + * The {@code getModelForX} methods automatically increment the model's reference count + * it is up to the consumer to call {@link LocalModel#release()} when the model is no + * longer used. */ public class ModelLoadingService implements ClusterStateListener { @@ -93,8 +99,8 @@ public enum Consumer { } private static class ModelAndConsumer { - private LocalModel model; - private EnumSet consumers; + private final LocalModel model; + private final EnumSet consumers; private ModelAndConsumer(LocalModel model, Consumer consumer) { this.model = model; @@ -197,6 +203,12 @@ private void getModel(String modelId, Consumer consumer, ActionListener new ParameterizedMessage("[{}] loaded from cache", modelId)); return; @@ -223,6 +235,12 @@ private boolean loadModelIfNecessary(String modelId, Consumer consumer, ActionLi ModelAndConsumer cachedModel = localModelCache.get(modelId); if (cachedModel != null) { cachedModel.consumers.add(consumer); + try { + cachedModel.model.acquire(); + } catch (CircuitBreakingException e) { + modelActionListener.onFailure(e); + return true; + } modelActionListener.onResponse(cachedModel.model); return true; } @@ -256,20 +274,16 @@ private void loadModel(String modelId, Consumer consumer) { trainedModelCircuitBreaker.addEstimateBytesAndMaybeBreak(trainedModelConfig.getEstimatedHeapMemory(), modelId); provider.getTrainedModelForInference(modelId, ActionListener.wrap( inferenceDefinition -> { - // Since we have used the previously stored estimate to help guard against OOM we need to adjust the memory - // So that the memory this model uses in the circuit breaker is the most accurate estimate. - long estimateDiff = inferenceDefinition.ramBytesUsed() - trainedModelConfig.getEstimatedHeapMemory(); - if (estimateDiff < 0) { - trainedModelCircuitBreaker.addWithoutBreaking(estimateDiff); - } else if (estimateDiff > 0) { // rare case where estimate is now HIGHER - try { - trainedModelCircuitBreaker.addEstimateBytesAndMaybeBreak(estimateDiff, modelId); - } catch (CircuitBreakingException ex) { // if we failed here, we should remove the initial estimate as well - trainedModelCircuitBreaker.addWithoutBreaking(-trainedModelConfig.getEstimatedHeapMemory()); - handleLoadFailure(modelId, ex); - return; - } + try { + // Since we have used the previously stored estimate to help guard against OOM we need + // to adjust the memory so that the memory this model uses in the circuit breaker + // is the most accurate estimate. + updateCircuitBreakerEstimate(modelId, inferenceDefinition, trainedModelConfig); + } catch (CircuitBreakingException ex) { + handleLoadFailure(modelId, ex); + return; } + handleLoadSuccess(modelId, consumer, trainedModelConfig, inferenceDefinition); }, failure -> { @@ -300,8 +314,14 @@ private void loadWithoutCaching(String modelId, ActionListener model InferenceConfig inferenceConfig = trainedModelConfig.getInferenceConfig() == null ? inferenceConfigFromTargetType(inferenceDefinition.getTargetType()) : trainedModelConfig.getInferenceConfig(); - // Remove the bytes as we cannot control how long the caller will keep the model in memory - trainedModelCircuitBreaker.addWithoutBreaking(-trainedModelConfig.getEstimatedHeapMemory()); + + try { + updateCircuitBreakerEstimate(modelId, inferenceDefinition, trainedModelConfig); + } catch (CircuitBreakingException ex) { + modelActionListener.onFailure(ex); + return; + } + modelActionListener.onResponse(new LocalModel( trainedModelConfig.getModelId(), localNode, @@ -309,7 +329,9 @@ private void loadWithoutCaching(String modelId, ActionListener model trainedModelConfig.getInput(), trainedModelConfig.getDefaultFieldMap(), inferenceConfig, - modelStatsService)); + trainedModelConfig.getLicenseLevel(), + modelStatsService, + trainedModelCircuitBreaker)); }, // Failure getting the definition, remove the initial estimation value e -> { @@ -322,6 +344,21 @@ private void loadWithoutCaching(String modelId, ActionListener model )); } + private void updateCircuitBreakerEstimate(String modelId, InferenceDefinition inferenceDefinition, + TrainedModelConfig trainedModelConfig) throws CircuitBreakingException { + long estimateDiff = inferenceDefinition.ramBytesUsed() - trainedModelConfig.getEstimatedHeapMemory(); + if (estimateDiff < 0) { + trainedModelCircuitBreaker.addWithoutBreaking(estimateDiff); + } else if (estimateDiff > 0) { // rare case where estimate is now HIGHER + try { + trainedModelCircuitBreaker.addEstimateBytesAndMaybeBreak(estimateDiff, modelId); + } catch (CircuitBreakingException ex) { // if we failed here, we should remove the initial estimate as well + trainedModelCircuitBreaker.addWithoutBreaking(-trainedModelConfig.getEstimatedHeapMemory()); + throw ex; + } + } + } + private void handleLoadSuccess(String modelId, Consumer consumer, TrainedModelConfig trainedModelConfig, @@ -337,21 +374,31 @@ private void handleLoadSuccess(String modelId, trainedModelConfig.getInput(), trainedModelConfig.getDefaultFieldMap(), inferenceConfig, - modelStatsService); + trainedModelConfig.getLicenseLevel(), + modelStatsService, + trainedModelCircuitBreaker); synchronized (loadingListeners) { listeners = loadingListeners.remove(modelId); // If there is no loadingListener that means the loading was canceled and the listener was already notified as such // Consequently, we should not store the retrieved model if (listeners == null) { - trainedModelCircuitBreaker.addWithoutBreaking(-inferenceDefinition.ramBytesUsed()); + loadedModel.release(); return; } + + // temporarily increase the reference count before adding to + // the cache in case the model is evicted before the listeners + // are called in which case acquire() would throw. + loadedModel.acquire(); localModelCache.put(modelId, new ModelAndConsumer(loadedModel, consumer)); shouldNotAudit.remove(modelId); } // synchronized (loadingListeners) for (ActionListener listener = listeners.poll(); listener != null; listener = listeners.poll()) { + loadedModel.acquire(); listener.onResponse(loadedModel); } + // account for the acquire in the synchronized block above + loadedModel.release(); } private void handleLoadFailure(String modelId, Exception failure) { @@ -386,7 +433,7 @@ private void cacheEvictionListener(RemovalNotification // If the model is no longer referenced, flush the stats to persist as soon as possible notification.getValue().model.persistStats(referencedModels.contains(notification.getKey()) == false); } finally { - trainedModelCircuitBreaker.addWithoutBreaking(-notification.getValue().model.ramBytesUsed()); + notification.getValue().model.release(); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java index 43b6927b17a2a..f6ffc74e569ed 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; import org.elasticsearch.xpack.core.ml.annotations.Annotation; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizationStatus; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerStats; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; @@ -281,6 +280,7 @@ void processResult(AutodetectResult result) { Annotation annotation = result.getAnnotation(); if (annotation != null) { bulkAnnotationsPersister.persistAnnotation(annotation); + notifyCategorizationStatusChange(annotation); } Forecast forecast = result.getForecast(); if (forecast != null) { @@ -395,7 +395,6 @@ private void processModelSizeStats(ModelSizeStats modelSizeStats) { persister.persistModelSizeStats(modelSizeStats, this::isAlive); notifyModelMemoryStatusChange(modelSizeStats); - notifyCategorizationStatusChange(modelSizeStats); latestModelSizeStats = modelSizeStats; } @@ -418,13 +417,11 @@ private void notifyModelMemoryStatusChange(ModelSizeStats modelSizeStats) { } } - private void notifyCategorizationStatusChange(ModelSizeStats modelSizeStats) { - CategorizationStatus categorizationStatus = modelSizeStats.getCategorizationStatus(); - if (categorizationStatus != latestModelSizeStats.getCategorizationStatus()) { - if (categorizationStatus == CategorizationStatus.WARN) { - auditor.warning(jobId, Messages.getMessage(Messages.JOB_AUDIT_CATEGORIZATION_STATUS_WARN, categorizationStatus, - priorRunsBucketCount + currentRunBucketCount)); - } + private void notifyCategorizationStatusChange(Annotation annotation) { + if (annotation.getEvent() == Annotation.Event.CATEGORIZATION_STATUS_CHANGE) { + long bucketCount = priorRunsBucketCount + currentRunBucketCount; + auditor.warning(jobId, annotation.getAnnotation() + " after " + + bucketCount + ((bucketCount == 1) ? " bucket" : " buckets")); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java index 2d0f4944a6a8f..bd1cd208e1f62 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/SearchAfterDocumentsIterator.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.client.OriginSettingClient; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchHit; @@ -20,6 +21,7 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * An iterator useful to fetch a large number of documents of type T @@ -44,12 +46,27 @@ public abstract class SearchAfterDocumentsIterator implements BatchedIterator private final OriginSettingClient client; private final String index; + private final boolean trackTotalHits; + private final AtomicLong totalHits = new AtomicLong(); private final AtomicBoolean lastSearchReturnedResults; private int batchSize = BATCH_SIZE; protected SearchAfterDocumentsIterator(OriginSettingClient client, String index) { + this(client, index, false); + } + + /** + * Constructs an iterator that searches docs using search after + * + * @param client the client + * @param index the index + * @param trackTotalHits whether to track total hits. Note this is only done in the first search + * and the result will only be accurate if the index is not changed between searches. + */ + protected SearchAfterDocumentsIterator(OriginSettingClient client, String index, boolean trackTotalHits) { this.client = Objects.requireNonNull(client); this.index = Objects.requireNonNull(index); + this.trackTotalHits = trackTotalHits; this.lastSearchReturnedResults = new AtomicBoolean(true); } @@ -88,6 +105,9 @@ public Deque next() { } SearchResponse searchResponse = doSearch(searchAfterFields()); + if (trackTotalHits && totalHits.get() == 0) { + totalHits.set(searchResponse.getHits().getTotalHits().value); + } return mapHits(searchResponse); } @@ -100,11 +120,19 @@ private SearchResponse doSearch(Object [] searchAfterValues) { .fetchSource(shouldFetchSource()) .sort(sortField())); + if (trackTotalHits && totalHits.get() == 0L) { + sourceBuilder.trackTotalHits(true); + } + if (searchAfterValues != null) { sourceBuilder.searchAfter(searchAfterValues); } searchRequest.source(sourceBuilder); + return executeSearchRequest(searchRequest); + } + + protected SearchResponse executeSearchRequest(SearchRequest searchRequest) { return client.search(searchRequest).actionGet(); } @@ -177,4 +205,15 @@ protected boolean shouldFetchSource() { void setBatchSize(int batchSize) { this.batchSize = batchSize; } + + protected Client client() { + return client; + } + + public long getTotalHits() { + if (trackTotalHits == false) { + throw new IllegalStateException("cannot return total hits because tracking was not enabled"); + } + return totalHits.get(); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java index 01661f6ec82a6..f3a7325552f56 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractorTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification; import org.elasticsearch.xpack.core.ml.dataframe.analyses.OutlierDetectionTests; import org.elasticsearch.xpack.core.ml.dataframe.analyses.Regression; +import org.elasticsearch.xpack.ml.dataframe.process.crossvalidation.CrossValidationSplitterFactory; import org.elasticsearch.xpack.ml.extractor.DocValueField; import org.elasticsearch.xpack.ml.extractor.ExtractedField; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; @@ -66,6 +67,7 @@ public class DataFrameDataExtractorTests extends ESTestCase { private QueryBuilder query; private int scrollSize; private Map headers; + private CrossValidationSplitterFactory crossValidationSplitterFactory; private ArgumentCaptor capturedClearScrollRequests; private ActionFuture clearScrollFuture; @@ -85,6 +87,9 @@ public void setUpTests() { scrollSize = 1000; headers = Collections.emptyMap(); + crossValidationSplitterFactory = mock(CrossValidationSplitterFactory.class); + when(crossValidationSplitterFactory.create()).thenReturn(row -> true); + clearScrollFuture = mock(ActionFuture.class); capturedClearScrollRequests = ArgumentCaptor.forClass(ClearScrollRequest.class); when(client.execute(same(ClearScrollAction.INSTANCE), capturedClearScrollRequests.capture())).thenReturn(clearScrollFuture); @@ -462,8 +467,8 @@ public void testGetCategoricalFields() { } private TestExtractor createExtractor(boolean includeSource, boolean supportsRowsWithMissingValues) { - DataFrameDataExtractorContext context = new DataFrameDataExtractorContext( - JOB_ID, extractedFields, indices, query, scrollSize, headers, includeSource, supportsRowsWithMissingValues); + DataFrameDataExtractorContext context = new DataFrameDataExtractorContext(JOB_ID, extractedFields, indices, query, scrollSize, + headers, includeSource, supportsRowsWithMissingValues, crossValidationSplitterFactory); return new TestExtractor(client, context); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index cb785ba2d71ad..6b2a9db266731 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -55,7 +55,7 @@ public void testDetect_GivenFloatField() { .addAggregatableField("some_float", "float").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -73,7 +73,7 @@ public void testDetect_GivenNumericFieldWithMultipleTypes() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -91,7 +91,7 @@ public void testDetect_GivenOutlierDetectionAndNonNumericField() { .addAggregatableField("some_keyword", "keyword").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields().isEmpty(), is(true)); @@ -107,7 +107,7 @@ public void testDetect_GivenOutlierDetectionAndFieldWithNumericAndNonNumericType .addAggregatableField("indecisive_field", "float", "keyword").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields().isEmpty(), is(true)); @@ -127,7 +127,7 @@ public void testDetect_GivenOutlierDetectionAndMultipleFields() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -156,7 +156,7 @@ public void testDetect_GivenRegressionAndMultipleFields() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -183,7 +183,7 @@ public void testDetect_GivenRegressionAndRequiredFieldMissing() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("required field [foo] is missing; analysis requires fields [foo]")); @@ -199,7 +199,7 @@ public void testDetect_GivenRegressionAndRequiredFieldExcluded() { analyzedFields = new FetchSourceContext(true, new String[0], new String[] {"foo"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("required field [foo] is missing; analysis requires fields [foo]")); @@ -215,7 +215,7 @@ public void testDetect_GivenRegressionAndRequiredFieldNotIncluded() { analyzedFields = new FetchSourceContext(true, new String[] {"some_float", "some_keyword"}, new String[0]); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("required field [foo] is missing; analysis requires fields [foo]")); @@ -229,7 +229,7 @@ public void testDetect_GivenFieldIsBothIncludedAndExcluded() { analyzedFields = new FetchSourceContext(true, new String[] {"foo", "bar"}, new String[] {"foo"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -250,7 +250,7 @@ public void testDetect_GivenFieldIsNotIncludedAndIsExcluded() { analyzedFields = new FetchSourceContext(true, new String[] {"foo"}, new String[] {"bar"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -272,7 +272,7 @@ public void testDetect_GivenRegressionAndRequiredFieldHasInvalidType() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("foo"), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("invalid types [keyword] for required field [foo]; " + @@ -288,7 +288,7 @@ public void testDetect_GivenClassificationAndRequiredFieldHasInvalidType() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildClassificationConfig("some_float"), 100, fieldCapabilities, Collections.emptyMap()); + buildClassificationConfig("some_float"), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("invalid types [float] for required field [some_float]; " + @@ -302,7 +302,7 @@ public void testDetect_GivenClassificationAndDependentVariableHasInvalidCardinal .addAggregatableField("foo", "keyword") .build(); - ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector(SOURCE_INDEX, + ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( buildClassificationConfig("some_keyword"), 100, fieldCapabilities, Collections.singletonMap("some_keyword", 31L)); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); @@ -314,7 +314,7 @@ public void testDetect_GivenIgnoredField() { .addAggregatableField("_id", "float").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields().isEmpty(), is(true)); @@ -328,7 +328,7 @@ public void testDetect_GivenIncludedIgnoredField() { analyzedFields = new FetchSourceContext(true, new String[]{"_id"}, new String[0]); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("No field [_id] could be detected")); @@ -341,7 +341,7 @@ public void testDetect_GivenExcludedFieldIsMissing() { analyzedFields = new FetchSourceContext(true, new String[]{"*"}, new String[] {"bar"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("No field [bar] could be detected")); @@ -355,7 +355,7 @@ public void testDetect_GivenExcludedFieldIsUnsupported() { analyzedFields = new FetchSourceContext(true, null, new String[] {"categorical"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); @@ -386,7 +386,7 @@ public void testDetect_ShouldSortFieldsAlphabetically() { FieldCapabilitiesResponse fieldCapabilities = mockFieldCapsResponseBuilder.build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -403,7 +403,7 @@ public void testDetect_GivenIncludeWithMissingField() { analyzedFields = new FetchSourceContext(true, new String[]{"your_field1", "my*"}, new String[0]); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("No field [your_field1] could be detected")); @@ -418,7 +418,7 @@ public void testDetect_GivenExcludeAllValidFields() { analyzedFields = new FetchSourceContext(true, new String[0], new String[]{"my_*"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields().isEmpty(), is(true)); @@ -436,7 +436,7 @@ public void testDetect_GivenInclusionsAndExclusions() { analyzedFields = new FetchSourceContext(true, new String[]{"your*", "my_*"}, new String[]{"*nope"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -461,7 +461,7 @@ public void testDetect_GivenIncludedFieldHasUnsupportedType() { analyzedFields = new FetchSourceContext(true, new String[]{"your*", "my_*"}, new String[]{"*nope"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("field [your_keyword] has unsupported type [keyword]. " + @@ -477,7 +477,7 @@ public void testDetect_GivenIndexContainsResultsField() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -502,7 +502,7 @@ public void testDetect_GivenIncludedResultsField() { analyzedFields = new FetchSourceContext(true, new String[]{RESULTS_FIELD}, new String[0]); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("No field [ml] could be detected")); @@ -517,7 +517,7 @@ public void testDetect_GivenLessFieldsThanDocValuesLimit() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 4, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 4, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -536,7 +536,7 @@ public void testDetect_GivenEqualFieldsToDocValuesLimit() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 3, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 3, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -555,7 +555,7 @@ public void testDetect_GivenMoreFieldsThanDocValuesLimit() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 2, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 2, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List extractedFieldNames = fieldExtraction.v1().getAllFields().stream().map(ExtractedField::getName) @@ -577,7 +577,7 @@ private void testDetect_GivenBooleanField(DataFrameAnalyticsConfig config, boole ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, config, 100, fieldCapabilities, fieldCardinalities); + config, 100, fieldCapabilities, fieldCardinalities); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -633,7 +633,7 @@ public void testDetect_GivenMultiFields() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("a_float"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("a_float"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(5)); @@ -664,7 +664,7 @@ public void testDetect_GivenMultiFieldAndParentIsRequired() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildClassificationConfig("field_1"), 100, fieldCapabilities, Collections.singletonMap("field_1", 2L)); + buildClassificationConfig("field_1"), 100, fieldCapabilities, Collections.singletonMap("field_1", 2L)); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -688,7 +688,7 @@ public void testDetect_GivenMultiFieldAndMultiFieldIsRequired() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildClassificationConfig("field_1.keyword"), 100, fieldCapabilities, + buildClassificationConfig("field_1.keyword"), 100, fieldCapabilities, Collections.singletonMap("field_1.keyword", 2L)); Tuple> fieldExtraction = extractedFieldsDetector.detect(); @@ -715,7 +715,7 @@ public void testDetect_GivenSeveralMultiFields_ShouldPickFirstSorted() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -741,7 +741,7 @@ public void testDetect_GivenMultiFields_OverDocValueLimit() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("field_2"), 0, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("field_2"), 0, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -766,7 +766,7 @@ public void testDetect_GivenParentAndMultiFieldBothAggregatable() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("field_2.double"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("field_2.double"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -791,7 +791,7 @@ public void testDetect_GivenParentAndMultiFieldNoneAggregatable() { .build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -816,7 +816,7 @@ public void testDetect_GivenMultiFields_AndExplicitlyIncludedFields() { analyzedFields = new FetchSourceContext(true, new String[] { "field_1", "field_2" }, new String[0]); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); + buildRegressionConfig("field_2"), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); assertThat(fieldExtraction.v1().getAllFields(), hasSize(2)); @@ -841,7 +841,7 @@ public void testDetect_GivenSourceFilteringWithIncludes() { sourceFiltering = new FetchSourceContext(true, new String[] {"field_1*"}, null); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -864,7 +864,7 @@ public void testDetect_GivenSourceFilteringWithExcludes() { sourceFiltering = new FetchSourceContext(true, null, new String[] {"field_1*"}); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -884,7 +884,7 @@ public void testDetect_GivenObjectFields() { .addNonAggregatableField("object_field_2", "object").build(); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); Tuple> fieldExtraction = extractedFieldsDetector.detect(); List allFields = fieldExtraction.v1().getAllFields(); @@ -900,7 +900,7 @@ public void testDetect_GivenAnalyzedFieldIncludesObjectField() { analyzedFields = new FetchSourceContext(true, new String[] { "float_field", "object_field" }, null); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("analyzed_fields must not include or exclude object fields: [object_field]")); @@ -914,7 +914,7 @@ public void testDetect_GivenAnalyzedFieldExcludesObjectField() { analyzedFields = new FetchSourceContext(true, null, new String[] { "object_field" }); ExtractedFieldsDetector extractedFieldsDetector = new ExtractedFieldsDetector( - SOURCE_INDEX, buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); + buildOutlierDetectionConfig(), 100, fieldCapabilities, Collections.emptyMap()); ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, extractedFieldsDetector::detect); assertThat(e.getMessage(), equalTo("analyzed_fields must not include or exclude object fields: [object_field]")); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java new file mode 100644 index 0000000000000..8e437492b3ecc --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.dataframe.inference; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; +import org.elasticsearch.xpack.core.ml.dataframe.analyses.RegressionTests; +import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults; +import org.elasticsearch.xpack.core.ml.inference.results.InferenceResults; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig; +import org.elasticsearch.xpack.ml.dataframe.stats.DataCountsTracker; +import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker; +import org.elasticsearch.xpack.ml.inference.loadingservice.LocalModel; +import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; +import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class InferenceRunnerTests extends ESTestCase { + + private Client client; + private ResultsPersisterService resultsPersisterService; + private ModelLoadingService modelLoadingService; + private DataFrameAnalyticsConfig config; + private ProgressTracker progressTracker; + private TaskId parentTaskId; + + @Before + public void setupTests() { + client = mock(Client.class); + resultsPersisterService = mock(ResultsPersisterService.class); + config = new DataFrameAnalyticsConfig.Builder() + .setId("test") + .setAnalysis(RegressionTests.createRandom()) + .setSource(new DataFrameAnalyticsSource(new String[] {"source_index"}, null, null)) + .setDest(new DataFrameAnalyticsDest("dest_index", "test_results_field")) + .build(); + progressTracker = ProgressTracker.fromZeroes(config.getAnalysis().getProgressPhases(), config.getAnalysis().supportsInference()); + parentTaskId = new TaskId(randomAlphaOfLength(10), randomLong()); + modelLoadingService = mock(ModelLoadingService.class); + } + + public void testInferTestDocs() { + Map doc1 = new HashMap<>(); + doc1.put("key", 1); + Map doc2 = new HashMap<>(); + doc2.put("key", 2); + TestDocsIterator testDocsIterator = mock(TestDocsIterator.class); + when(testDocsIterator.hasNext()).thenReturn(true, false); + when(testDocsIterator.next()).thenReturn(buildSearchHits(Arrays.asList(doc1, doc2))); + when(testDocsIterator.getTotalHits()).thenReturn(2L); + InferenceConfig config = ClassificationConfig.EMPTY_PARAMS; + + LocalModel localModel = localModelInferences(new ClassificationInferenceResults(1.0, + "foo", + Collections.emptyList(), + config), + new ClassificationInferenceResults(0.0, + "bar", + Collections.emptyList(), + config)); + + InferenceRunner inferenceRunner = createInferenceRunner(); + + inferenceRunner.inferTestDocs(localModel, testDocsIterator); + + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(BulkRequest.class); + + verify(resultsPersisterService).bulkIndexWithHeadersWithRetry(any(), argumentCaptor.capture(), any(), any(), any()); + assertThat(progressTracker.getInferenceProgressPercent(), equalTo(100)); + + BulkRequest bulkRequest = argumentCaptor.getAllValues().get(0); + List> indexRequests = bulkRequest.requests(); + Map doc1Source = ((IndexRequest)indexRequests.get(0)).sourceAsMap(); + Map doc2Source = ((IndexRequest)indexRequests.get(1)).sourceAsMap(); + + assertThat(doc1Source.get("test_results_field"), + equalTo(new HashMap<>(){{ + put("predicted_value", "foo"); + put("is_training", false); + }})); + assertThat(doc2Source.get("test_results_field"), + equalTo(new HashMap<>(){{ + put("predicted_value", "bar"); + put("is_training", false); + }})); + } + + public void testInferTestDocs_GivenCancelWasCalled() { + + LocalModel localModel = mock(LocalModel.class); + + TestDocsIterator infiniteDocsIterator = mock(TestDocsIterator.class); + when(infiniteDocsIterator.hasNext()).thenReturn(true); + + InferenceRunner inferenceRunner = createInferenceRunner(); + inferenceRunner.cancel(); + + inferenceRunner.inferTestDocs(localModel, infiniteDocsIterator); + + Mockito.verifyNoMoreInteractions(localModel, resultsPersisterService); + assertThat(progressTracker.getInferenceProgressPercent(), equalTo(0)); + } + + private static Deque buildSearchHits(List> vals) { + return vals.stream() + .map(InferenceRunnerTests::fromMap) + .map(reference -> SearchHit.createFromMap(Collections.singletonMap("_source", reference))) + .collect(Collectors.toCollection(ArrayDeque::new)); + } + + private static BytesReference fromMap(Map map) { + try(XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().map(map)) { + return BytesReference.bytes(xContentBuilder); + } catch (IOException ex) { + throw new ElasticsearchException(ex); + } + } + + private LocalModel localModelInferences(InferenceResults first, InferenceResults... rest) { + LocalModel localModel = mock(LocalModel.class); + when(localModel.inferNoStats(any())).thenReturn(first, rest); + return localModel; + } + + private InferenceRunner createInferenceRunner() { + return new InferenceRunner(client, modelLoadingService, resultsPersisterService, parentTaskId, config, progressTracker, + new DataCountsTracker()); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java index add561b0bb584..0803246f18ca1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsProcessManagerTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.ml.dataframe.stats.ProgressTracker; import org.elasticsearch.xpack.ml.dataframe.stats.StatsHolder; import org.elasticsearch.xpack.ml.extractor.ExtractedFields; +import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor; import org.elasticsearch.xpack.ml.utils.persistence.ResultsPersisterService; @@ -64,6 +65,7 @@ public class AnalyticsProcessManagerTests extends ESTestCase { private Client client; private DataFrameAnalyticsAuditor auditor; private TrainedModelProvider trainedModelProvider; + private ModelLoadingService modelLoadingService; private ExecutorService executorServiceForJob; private ExecutorService executorServiceForProcess; private AnalyticsProcess process; @@ -96,7 +98,7 @@ public void setUpMocks() { task = mock(DataFrameAnalyticsTask.class); when(task.getAllocationId()).thenReturn(TASK_ALLOCATION_ID); when(task.getStatsHolder()).thenReturn(new StatsHolder( - ProgressTracker.fromZeroes(Collections.singletonList("analyzing")).report())); + ProgressTracker.fromZeroes(Collections.singletonList("analyzing"), false).report())); when(task.getParentTaskId()).thenReturn(new TaskId("")); dataFrameAnalyticsConfig = DataFrameAnalyticsConfigTests.createRandomBuilder(CONFIG_ID, false, @@ -109,9 +111,9 @@ public void setUpMocks() { when(dataExtractorFactory.getExtractedFields()).thenReturn(mock(ExtractedFields.class)); resultsPersisterService = mock(ResultsPersisterService.class); - + modelLoadingService = mock(ModelLoadingService.class); processManager = new AnalyticsProcessManager(client, executorServiceForJob, executorServiceForProcess, processFactory, auditor, - trainedModelProvider, resultsPersisterService, 1); + trainedModelProvider, modelLoadingService, resultsPersisterService, 1); } public void testRunJob_TaskIsStopping() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java index 5c3e2b9ce1985..f2b33c30755f6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java @@ -47,7 +47,7 @@ public class AnalyticsResultProcessorTests extends ESTestCase { private AnalyticsProcess process; private DataFrameRowsJoiner dataFrameRowsJoiner; - private StatsHolder statsHolder = new StatsHolder(ProgressTracker.fromZeroes(Collections.singletonList("analyzing")).report()); + private StatsHolder statsHolder = new StatsHolder(ProgressTracker.fromZeroes(Collections.singletonList("analyzing"), false).report()); private TrainedModelProvider trainedModelProvider; private DataFrameAnalyticsAuditor auditor; private StatsPersister statsPersister; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java index 129b9b7fc92e6..d93849af04a6a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/DataFrameRowsJoinerTests.java @@ -65,7 +65,7 @@ public void testProcess_GivenSingleRowAndResult() throws IOException { String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row row = newTrainingRow(newHit(dataDoc), dataValues, 1); givenDataFrameBatches(List.of(Arrays.asList(row))); Map resultFields = new HashMap<>(); @@ -93,9 +93,9 @@ public void testProcess_GivenFullResultsBatch() throws IOException { String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; List firstBatch = new ArrayList<>(1000); - IntStream.range(0, 1000).forEach(i -> firstBatch.add(newRow(newHit(dataDoc), dataValues, i))); + IntStream.range(0, 1000).forEach(i -> firstBatch.add(newTrainingRow(newHit(dataDoc), dataValues, i))); List secondBatch = new ArrayList<>(1); - secondBatch.add(newRow(newHit(dataDoc), dataValues, 1000)); + secondBatch.add(newTrainingRow(newHit(dataDoc), dataValues, 1000)); givenDataFrameBatches(List.of(firstBatch, secondBatch)); Map resultFields = new HashMap<>(); @@ -116,7 +116,7 @@ public void testProcess_GivenSingleRowAndResultWithMismatchingIdHash() throws IO String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row row = newTrainingRow(newHit(dataDoc), dataValues, 1); givenDataFrameBatches(List.of(Arrays.asList(row))); Map resultFields = new HashMap<>(); @@ -131,10 +131,10 @@ public void testProcess_GivenSingleRowAndResultWithMismatchingIdHash() throws IO public void testProcess_GivenSingleBatchWithSkippedRows() throws IOException { givenClientHasNoFailures(); - DataFrameDataExtractor.Row skippedRow = newRow(newHit("{}"), null, 1); + DataFrameDataExtractor.Row skippedRow = newTrainingRow(newHit("{}"), null, 1); String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row normalRow = newRow(newHit(dataDoc), dataValues, 2); + DataFrameDataExtractor.Row normalRow = newTrainingRow(newHit(dataDoc), dataValues, 2); givenDataFrameBatches(List.of(Arrays.asList(skippedRow, normalRow))); Map resultFields = new HashMap<>(); @@ -161,10 +161,10 @@ public void testProcess_GivenTwoBatchesWhereFirstEndsWithSkippedRow() throws IOE String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row normalRow1 = newRow(newHit(dataDoc), dataValues, 1); - DataFrameDataExtractor.Row normalRow2 = newRow(newHit(dataDoc), dataValues, 2); - DataFrameDataExtractor.Row skippedRow = newRow(newHit("{}"), null, 3); - DataFrameDataExtractor.Row normalRow3 = newRow(newHit(dataDoc), dataValues, 4); + DataFrameDataExtractor.Row normalRow1 = newTrainingRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row normalRow2 = newTrainingRow(newHit(dataDoc), dataValues, 2); + DataFrameDataExtractor.Row skippedRow = newTrainingRow(newHit("{}"), null, 3); + DataFrameDataExtractor.Row normalRow3 = newTrainingRow(newHit(dataDoc), dataValues, 4); givenDataFrameBatches(List.of(Arrays.asList(normalRow1, normalRow2, skippedRow), Arrays.asList(normalRow3))); Map resultFields = new HashMap<>(); @@ -188,12 +188,72 @@ public void testProcess_GivenTwoBatchesWhereFirstEndsWithSkippedRow() throws IOE assertThat(indexedDocSource.get("b"), equalTo("2")); } + public void testProcess_GivenSingleBatchWithTestRows() throws IOException { + givenClientHasNoFailures(); + + String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; + String[] dataValues = {"42.0"}; + DataFrameDataExtractor.Row testRow = newTestRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row normalRow = newTrainingRow(newHit(dataDoc), dataValues, 2); + givenDataFrameBatches(Arrays.asList(Arrays.asList(testRow, normalRow))); + + Map resultFields = new HashMap<>(); + resultFields.put("a", "1"); + resultFields.put("b", "2"); + RowResults result = new RowResults(2, resultFields); + givenProcessResults(Arrays.asList(result)); + + List capturedBulkRequests = bulkRequestCaptor.getAllValues(); + assertThat(capturedBulkRequests.size(), equalTo(1)); + BulkRequest capturedBulkRequest = capturedBulkRequests.get(0); + assertThat(capturedBulkRequest.numberOfActions(), equalTo(1)); + IndexRequest indexRequest = (IndexRequest) capturedBulkRequest.requests().get(0); + Map indexedDocSource = indexRequest.sourceAsMap(); + assertThat(indexedDocSource.size(), equalTo(4)); + assertThat(indexedDocSource.get("f_1"), equalTo("foo")); + assertThat(indexedDocSource.get("f_2"), equalTo(42.0)); + assertThat(indexedDocSource.get("a"), equalTo("1")); + assertThat(indexedDocSource.get("b"), equalTo("2")); + } + + public void testProcess_GivenTwoBatchesWhereFirstEndsWithTestRow() throws IOException { + givenClientHasNoFailures(); + + String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; + String[] dataValues = {"42.0"}; + DataFrameDataExtractor.Row normalRow1 = newTrainingRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row normalRow2 = newTrainingRow(newHit(dataDoc), dataValues, 2); + DataFrameDataExtractor.Row testRow = newTestRow(newHit(dataDoc), dataValues, 3); + DataFrameDataExtractor.Row normalRow3 = newTrainingRow(newHit(dataDoc), dataValues, 4); + givenDataFrameBatches(Arrays.asList(Arrays.asList(normalRow1, normalRow2, testRow), Arrays.asList(normalRow3))); + + Map resultFields = new HashMap<>(); + resultFields.put("a", "1"); + resultFields.put("b", "2"); + RowResults result1 = new RowResults(1, resultFields); + RowResults result2 = new RowResults(2, resultFields); + RowResults result3 = new RowResults(4, resultFields); + givenProcessResults(Arrays.asList(result1, result2, result3)); + + List capturedBulkRequests = bulkRequestCaptor.getAllValues(); + assertThat(capturedBulkRequests.size(), equalTo(1)); + BulkRequest capturedBulkRequest = capturedBulkRequests.get(0); + assertThat(capturedBulkRequest.numberOfActions(), equalTo(3)); + IndexRequest indexRequest = (IndexRequest) capturedBulkRequest.requests().get(0); + Map indexedDocSource = indexRequest.sourceAsMap(); + assertThat(indexedDocSource.size(), equalTo(4)); + assertThat(indexedDocSource.get("f_1"), equalTo("foo")); + assertThat(indexedDocSource.get("f_2"), equalTo(42.0)); + assertThat(indexedDocSource.get("a"), equalTo("1")); + assertThat(indexedDocSource.get("b"), equalTo("2")); + } + public void testProcess_GivenMoreResultsThanRows() throws IOException { givenClientHasNoFailures(); String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row row = newRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row row = newTrainingRow(newHit(dataDoc), dataValues, 1); givenDataFrameBatches(List.of(List.of(row))); Map resultFields = new HashMap<>(); @@ -211,8 +271,8 @@ public void testProcess_GivenNoResults_ShouldCancelAndConsumeExtractor() throws String dataDoc = "{\"f_1\": \"foo\", \"f_2\": 42.0}"; String[] dataValues = {"42.0"}; - DataFrameDataExtractor.Row row1 = newRow(newHit(dataDoc), dataValues, 1); - DataFrameDataExtractor.Row row2 = newRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row row1 = newTrainingRow(newHit(dataDoc), dataValues, 1); + DataFrameDataExtractor.Row row2 = newTrainingRow(newHit(dataDoc), dataValues, 1); givenDataFrameBatches(List.of(List.of(row1), List.of(row2))); givenProcessResults(Collections.emptyList()); @@ -240,10 +300,19 @@ private static SearchHit newHit(String json) { return hit; } - private static DataFrameDataExtractor.Row newRow(SearchHit hit, String[] values, int checksum) { + private static DataFrameDataExtractor.Row newTrainingRow(SearchHit hit, String[] values, int checksum) { + return newRow(hit, values, true, checksum); + } + + private static DataFrameDataExtractor.Row newTestRow(SearchHit hit, String[] values, int checksum) { + return newRow(hit, values, false, checksum); + } + + private static DataFrameDataExtractor.Row newRow(SearchHit hit, String[] values, boolean isTraining, int checksum) { DataFrameDataExtractor.Row row = mock(DataFrameDataExtractor.Row.class); when(row.getHit()).thenReturn(hit); when(row.getValues()).thenReturn(values); + when(row.isTraining()).thenReturn(isTraining); when(row.getChecksum()).thenReturn(checksum); when(row.shouldSkip()).thenReturn(values == null); return row; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/SingleClassReservoirCrossValidationSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/SingleClassReservoirCrossValidationSplitterTests.java index fee589df371d6..84df5d3256316 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/SingleClassReservoirCrossValidationSplitterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/SingleClassReservoirCrossValidationSplitterTests.java @@ -14,7 +14,6 @@ import java.util.List; import java.util.stream.IntStream; -import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.is; @@ -24,8 +23,6 @@ public class SingleClassReservoirCrossValidationSplitterTests extends ESTestCase private int dependentVariableIndex; private String dependentVariable; private long randomizeSeed; - private long trainingDocsCount; - private long testDocsCount; @Before public void setUpTests() { @@ -39,8 +36,8 @@ public void setUpTests() { randomizeSeed = randomLong(); } - public void testProcess_GivenRowsWithoutDependentVariableValue() { - CrossValidationSplitter crossValidationSplitter = createSplitter(50.0, 0); + public void testIsTraining_GivenRowsWithoutDependentVariableValue() { + CrossValidationSplitter splitter = createSplitter(50.0, 0); for (int i = 0; i < 100; i++) { String[] row = new String[fields.size()]; @@ -50,17 +47,13 @@ public void testProcess_GivenRowsWithoutDependentVariableValue() { } String[] processedRow = Arrays.copyOf(row, row.length); - crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); - - // As all these rows have no dependent variable value, they're not for training and should be unaffected + assertThat(splitter.isTraining(processedRow), is(false)); assertThat(Arrays.equals(processedRow, row), is(true)); } - assertThat(trainingDocsCount, equalTo(0L)); - assertThat(testDocsCount, equalTo(100L)); } - public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() { - CrossValidationSplitter crossValidationSplitter = createSplitter(100.0, 100L); + public void testIsTraining_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() { + CrossValidationSplitter splitter = createSplitter(100.0, 100L); for (int i = 0; i < 100; i++) { String[] row = new String[fields.size()]; @@ -69,16 +62,12 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs } String[] processedRow = Arrays.copyOf(row, row.length); - crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); - - // We should pick them all as training percent is 100 + assertThat(splitter.isTraining(processedRow), is(true)); assertThat(Arrays.equals(processedRow, row), is(true)); } - assertThat(trainingDocsCount, equalTo(100L)); - assertThat(testDocsCount, equalTo(0L)); } - public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() { + public void testIsTraining_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() { double trainingPercent = randomDoubleBetween(1.0, 100.0, true); double trainingFraction = trainingPercent / 100; long rowCount = 1000; @@ -86,7 +75,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs int runCount = 20; int[] trainingRowsPerRun = new int[runCount]; for (int testIndex = 0; testIndex < runCount; testIndex++) { - CrossValidationSplitter crossValidationSplitter = createSplitter(trainingPercent, rowCount); + CrossValidationSplitter splitter = createSplitter(trainingPercent, rowCount); int trainingRows = 0; for (int i = 0; i < rowCount; i++) { String[] row = new String[fields.size()]; @@ -95,15 +84,10 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs } String[] processedRow = Arrays.copyOf(row, row.length); - crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); + boolean isTraining = splitter.isTraining(processedRow); + assertThat(Arrays.equals(processedRow, row), is(true)); - for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) { - if (fieldIndex != dependentVariableIndex) { - assertThat(processedRow[fieldIndex], equalTo(row[fieldIndex])); - } - } - if (DataFrameDataExtractor.NULL_VALUE.equals(processedRow[dependentVariableIndex]) == false) { - assertThat(processedRow[dependentVariableIndex], equalTo(row[dependentVariableIndex])); + if (isTraining) { trainingRows++; } } @@ -114,8 +98,8 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs assertThat(meanTrainingRows, closeTo(trainingFraction * rowCount, 1.0)); } - public void testProcess_ShouldHaveAtLeastOneTrainingRow() { - CrossValidationSplitter crossValidationSplitter = createSplitter(1.0, 1); + public void testIsTraining_ShouldHaveAtLeastOneTrainingRow() { + CrossValidationSplitter splitter = createSplitter(1.0, 1); // We have some non-training rows and then a training row to check // we maintain the first training row and not just the first row @@ -130,23 +114,14 @@ public void testProcess_ShouldHaveAtLeastOneTrainingRow() { } String[] processedRow = Arrays.copyOf(row, row.length); - crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); + boolean shouldBeForTraining = row[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE; + assertThat(splitter.isTraining(processedRow), is(shouldBeForTraining)); assertThat(Arrays.equals(processedRow, row), is(true)); } - assertThat(trainingDocsCount, equalTo(1L)); - assertThat(testDocsCount, equalTo(9L)); } private CrossValidationSplitter createSplitter(double trainingPercent, long classCount) { return new SingleClassReservoirCrossValidationSplitter(fields, dependentVariable, trainingPercent, randomizeSeed, classCount); } - - private void incrementTrainingDocsCount() { - trainingDocsCount++; - } - - private void incrementTestDocsCount() { - testDocsCount++; - } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java index 57700803d0a1a..ad26fd7190cd8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/crossvalidation/StratifiedCrossValidationSplitterTests.java @@ -34,8 +34,6 @@ public class StratifiedCrossValidationSplitterTests extends ESTestCase { private long randomizeSeed; private Map classCounts; private String[] classValuesPerRow; - private long trainingDocsCount; - private long testDocsCount; @Before public void setUpTests() { @@ -80,7 +78,7 @@ public void testConstructor_GivenMissingDependentVariable() { assertThat(e.getMessage(), equalTo("Could not find dependent variable [foo] in fields []")); } - public void testProcess_GivenUnknownClass() { + public void testIsTraining_GivenUnknownClass() { CrossValidationSplitter splitter = createSplitter(100.0); String[] row = new String[fields.size()]; for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) { @@ -89,12 +87,12 @@ public void testProcess_GivenUnknownClass() { row[dependentVariableIndex] = "unknown_class"; IllegalStateException e = expectThrows(IllegalStateException.class, - () -> splitter.process(row, this::incrementTrainingDocsCount, this::incrementTestDocsCount)); + () -> splitter.isTraining(row)); assertThat(e.getMessage(), equalTo("Unknown class [unknown_class]; expected one of [a, b, c]")); } - public void testProcess_GivenRowsWithoutDependentVariableValue() { + public void testIsTraining_GivenRowsWithoutDependentVariableValue() { CrossValidationSplitter splitter = createSplitter(50.0); for (int i = 0; i < classValuesPerRow.length; i++) { @@ -105,16 +103,12 @@ public void testProcess_GivenRowsWithoutDependentVariableValue() { } String[] processedRow = Arrays.copyOf(row, row.length); - splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); - - // As all these rows have no dependent variable value, they're not for training and should be unaffected + assertThat(splitter.isTraining(processedRow), is(false)); assertThat(Arrays.equals(processedRow, row), is(true)); } - assertThat(trainingDocsCount, equalTo(0L)); - assertThat(testDocsCount, equalTo(500L)); } - public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() { + public void testIsTraining_GivenRowsWithDependentVariableValue_AndTrainingPercentIsHundred() { CrossValidationSplitter splitter = createSplitter(100.0); for (int i = 0; i < classValuesPerRow.length; i++) { @@ -125,16 +119,12 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs } String[] processedRow = Arrays.copyOf(row, row.length); - splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); - - // As training percent is 100 all rows should be unaffected + assertThat(splitter.isTraining(processedRow), is(true)); assertThat(Arrays.equals(processedRow, row), is(true)); } - assertThat(trainingDocsCount, equalTo(500L)); - assertThat(testDocsCount, equalTo(0L)); } - public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() { + public void testIsTraining_GivenRowsWithDependentVariableValue_AndTrainingPercentIsRandom() { // We don't go too low here to avoid flakiness double trainingPercent = randomDoubleBetween(50.0, 100.0, true); @@ -156,19 +146,13 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs } String[] processedRow = Arrays.copyOf(row, row.length); - splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); - - for (int fieldIndex = 0; fieldIndex < fields.size(); fieldIndex++) { - if (fieldIndex != dependentVariableIndex) { - assertThat(processedRow[fieldIndex], equalTo(row[fieldIndex])); - } - } + boolean isTraining = splitter.isTraining(processedRow); + assertThat(Arrays.equals(processedRow, row), is(true)); String classValue = row[dependentVariableIndex]; totalRowsPerClass.compute(classValue, (k, v) -> v + 1); - if (DataFrameDataExtractor.NULL_VALUE.equals(processedRow[dependentVariableIndex]) == false) { - assertThat(processedRow[dependentVariableIndex], equalTo(row[dependentVariableIndex])); + if (isTraining) { trainingRowsPerClass.compute(classValue, (k, v) -> v + 1); } } @@ -177,13 +161,17 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs // We can assert we're plus/minus 1 from rounding error + long actualTotalTrainingCount = 0; + for (long trainingCount : trainingRowsPerClass.values()) { + actualTotalTrainingCount += trainingCount; + } + long expectedTotalTrainingCount = 0; for (long classCount : classCounts.values()) { expectedTotalTrainingCount += trainingFraction * classCount; } - assertThat(trainingDocsCount + testDocsCount, equalTo((long) ROWS_COUNT)); - assertThat(trainingDocsCount, greaterThanOrEqualTo(expectedTotalTrainingCount - 2)); - assertThat(trainingDocsCount, lessThanOrEqualTo(expectedTotalTrainingCount)); + assertThat(actualTotalTrainingCount, greaterThanOrEqualTo(expectedTotalTrainingCount - 2)); + assertThat(actualTotalTrainingCount, lessThanOrEqualTo(expectedTotalTrainingCount)); for (String classValue : classCounts.keySet()) { double expectedClassTrainingCount = totalRowsPerClass.get(classValue) * trainingFraction; @@ -192,7 +180,7 @@ public void testProcess_GivenRowsWithDependentVariableValue_AndTrainingPercentIs } } - public void testProcess_SelectsTrainingRowsUniformly() { + public void testIsTraining_SelectsTrainingRowsUniformly() { double trainingPercent = 50.0; int runCount = 500; @@ -211,9 +199,10 @@ public void testProcess_SelectsTrainingRowsUniformly() { } String[] processedRow = Arrays.copyOf(row, row.length); - crossValidationSplitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); + boolean isTraining = crossValidationSplitter.isTraining(processedRow); + assertThat(Arrays.equals(processedRow, row), is(true)); - if (processedRow[dependentVariableIndex] != DataFrameDataExtractor.NULL_VALUE) { + if (isTraining) { trainingCountPerRow[i]++; } } @@ -228,7 +217,7 @@ public void testProcess_SelectsTrainingRowsUniformly() { } } - public void testProcess_GivenTwoClassesWithCountEqualToOne_ShouldUseForTraining() { + public void testIsTraining_GivenTwoClassesWithCountEqualToOne_ShouldUseForTraining() { dependentVariable = "dep_var"; fields = Arrays.asList(dependentVariable, "feature"); classCounts = new HashMap<>(); @@ -240,7 +229,8 @@ public void testProcess_GivenTwoClassesWithCountEqualToOne_ShouldUseForTraining( String[] row = new String[]{"class_a", "42.0"}; String[] processedRow = Arrays.copyOf(row, row.length); - splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); + assertThat(splitter.isTraining(processedRow), is(true)); + assertThat(Arrays.equals(processedRow, row), is(true)); assertThat(Arrays.equals(processedRow, row), is(true)); } @@ -248,24 +238,14 @@ public void testProcess_GivenTwoClassesWithCountEqualToOne_ShouldUseForTraining( String[] row = new String[]{"class_b", "42.0"}; String[] processedRow = Arrays.copyOf(row, row.length); - splitter.process(processedRow, this::incrementTrainingDocsCount, this::incrementTestDocsCount); + assertThat(splitter.isTraining(processedRow), is(true)); + assertThat(Arrays.equals(processedRow, row), is(true)); assertThat(Arrays.equals(processedRow, row), is(true)); } - - assertThat(trainingDocsCount, equalTo(2L)); - assertThat(testDocsCount, equalTo(0L)); } private CrossValidationSplitter createSplitter(double trainingPercent) { return new StratifiedCrossValidationSplitter(fields, dependentVariable, classCounts, trainingPercent, randomizeSeed); } - - private void incrementTrainingDocsCount() { - trainingDocsCount++; - } - - private void incrementTestDocsCount() { - testDocsCount++; - } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTrackerTests.java index c0890f59e2346..8e52301ee0b0c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTrackerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/ProgressTrackerTests.java @@ -37,7 +37,7 @@ public void testCtor() { } public void testFromZeroes() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Arrays.asList("a", "b", "c")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Arrays.asList("a", "b", "c"), false); List phases = progressTracker.report(); @@ -47,8 +47,28 @@ public void testFromZeroes() { assertThat(phases.stream().map(PhaseProgress::getProgressPercent).allMatch(p -> p == 0), is(true)); } + public void testFromZeroes_GivenAnalysisWithoutInference() { + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Arrays.asList("a", "b"), false); + + List phaseProgresses = progressTracker.report(); + + assertThat(phaseProgresses.size(), equalTo(5)); + assertThat(phaseProgresses.stream().map(PhaseProgress::getPhase).collect(Collectors.toList()), + contains("reindexing", "loading_data", "a", "b", "writing_results")); + } + + public void testFromZeroes_GivenAnalysisWithInference() { + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Arrays.asList("a", "b"), true); + + List phaseProgresses = progressTracker.report(); + + assertThat(phaseProgresses.size(), equalTo(6)); + assertThat(phaseProgresses.stream().map(PhaseProgress::getPhase).collect(Collectors.toList()), + contains("reindexing", "loading_data", "a", "b", "writing_results", "inference")); + } + public void testUpdates() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updateReindexingProgress(1); progressTracker.updateLoadingDataProgress(2); @@ -70,7 +90,7 @@ public void testUpdates() { } public void testUpdatePhase_GivenUnknownPhase() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updatePhase(new PhaseProgress("unknown", 42)); List phases = progressTracker.report(); @@ -81,7 +101,7 @@ public void testUpdatePhase_GivenUnknownPhase() { } public void testUpdateReindexingProgress_GivenLowerValueThanCurrentProgress() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updateReindexingProgress(10); @@ -93,7 +113,7 @@ public void testUpdateReindexingProgress_GivenLowerValueThanCurrentProgress() { } public void testUpdateLoadingDataProgress_GivenLowerValueThanCurrentProgress() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updateLoadingDataProgress(20); @@ -105,7 +125,7 @@ public void testUpdateLoadingDataProgress_GivenLowerValueThanCurrentProgress() { } public void testUpdateWritingResultsProgress_GivenLowerValueThanCurrentProgress() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updateWritingResultsProgress(30); @@ -117,7 +137,7 @@ public void testUpdateWritingResultsProgress_GivenLowerValueThanCurrentProgress( } public void testUpdatePhase_GivenLowerValueThanCurrentProgress() { - ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo")); + ProgressTracker progressTracker = ProgressTracker.fromZeroes(Collections.singletonList("foo"), false); progressTracker.updatePhase(new PhaseProgress("foo", 40)); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolderTests.java index 39736cc906893..f71afeabcedb4 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/stats/StatsHolderTests.java @@ -31,7 +31,7 @@ public void testResetProgressTrackerPreservingReindexingProgress_GivenSameAnalys ); StatsHolder statsHolder = new StatsHolder(phases); - statsHolder.resetProgressTrackerPreservingReindexingProgress(Arrays.asList("a", "b")); + statsHolder.resetProgressTrackerPreservingReindexingProgress(Arrays.asList("a", "b"), false); List phaseProgresses = statsHolder.getProgressTracker().report(); @@ -57,7 +57,7 @@ public void testResetProgressTrackerPreservingReindexingProgress_GivenDifferentA ); StatsHolder statsHolder = new StatsHolder(phases); - statsHolder.resetProgressTrackerPreservingReindexingProgress(Arrays.asList("c", "d")); + statsHolder.resetProgressTrackerPreservingReindexingProgress(Arrays.asList("c", "d"), false); List phaseProgresses = statsHolder.getProgressTracker().report(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilderTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilderTests.java index d1ea66dfefc1b..40b50488f3504 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilderTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/InferencePipelineAggregationBuilderTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase; @@ -61,7 +62,8 @@ protected InferencePipelineAggregationBuilder createTestAggregatorFactory() { .collect(Collectors.toMap(Function.identity(), (t) -> randomAlphaOfLength(5))); InferencePipelineAggregationBuilder builder = - new InferencePipelineAggregationBuilder(NAME, new SetOnce<>(mock(ModelLoadingService.class)), bucketPaths); + new InferencePipelineAggregationBuilder(NAME, new SetOnce<>(mock(ModelLoadingService.class)), + mock(XPackLicenseState.class), bucketPaths); builder.setModelId(randomAlphaOfLength(6)); if (randomBoolean()) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/ParsedInference.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/ParsedInference.java similarity index 100% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/aggs/ParsedInference.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/aggs/ParsedInference.java diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java index 4afe915a1fc8d..4539d5260bda6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/LocalModelTests.java @@ -6,7 +6,9 @@ package org.elasticsearch.xpack.ml.inference.loadingservice; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.license.License; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; import org.elasticsearch.xpack.core.ml.inference.preprocessing.OneHotEncoding; @@ -50,9 +52,11 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.internal.verification.VerificationModeFactory.times; public class LocalModelTests extends ESTestCase { @@ -73,7 +77,9 @@ public void testClassificationInfer() throws Exception { new TrainedModelInput(inputFields), Collections.singletonMap("field.foo", "field.foo.keyword"), ClassificationConfig.EMPTY_PARAMS, - modelStatsService); + randomFrom(License.OperationMode.values()), + modelStatsService, + mock(CircuitBreaker.class)); Map fields = new HashMap<>() {{ put("field.foo", 1.0); put("field", Collections.singletonMap("bar", 0.5)); @@ -102,7 +108,9 @@ public void testClassificationInfer() throws Exception { new TrainedModelInput(inputFields), Collections.singletonMap("field.foo", "field.foo.keyword"), ClassificationConfig.EMPTY_PARAMS, - modelStatsService); + License.OperationMode.PLATINUM, + modelStatsService, + mock(CircuitBreaker.class)); result = getSingleValue(model, fields, ClassificationConfigUpdate.EMPTY_PARAMS); assertThat(result.value(), equalTo(0.0)); assertThat(result.valueAsString(), equalTo("not_to_be")); @@ -144,7 +152,9 @@ public void testClassificationInferWithDifferentPredictionFieldTypes() throws Ex new TrainedModelInput(inputFields), Collections.singletonMap("field.foo", "field.foo.keyword"), ClassificationConfig.EMPTY_PARAMS, - modelStatsService); + License.OperationMode.PLATINUM, + modelStatsService, + mock(CircuitBreaker.class)); Map fields = new HashMap<>() {{ put("field.foo", 1.0); put("field.bar", 0.5); @@ -199,7 +209,9 @@ public void testRegression() throws Exception { new TrainedModelInput(inputFields), Collections.singletonMap("bar", "bar.keyword"), RegressionConfig.EMPTY_PARAMS, - modelStatsService); + License.OperationMode.PLATINUM, + modelStatsService, + mock(CircuitBreaker.class)); Map fields = new HashMap<>() {{ put("foo", 1.0); @@ -226,7 +238,9 @@ public void testAllFieldsMissing() throws Exception { new TrainedModelInput(inputFields), null, RegressionConfig.EMPTY_PARAMS, - modelStatsService); + License.OperationMode.PLATINUM, + modelStatsService, + mock(CircuitBreaker.class)); Map fields = new HashMap<>() {{ put("something", 1.0); @@ -256,7 +270,9 @@ public void testInferPersistsStatsAfterNumberOfCalls() throws Exception { new TrainedModelInput(inputFields), null, ClassificationConfig.EMPTY_PARAMS, - modelStatsService + License.OperationMode.PLATINUM, + modelStatsService, + mock(CircuitBreaker.class) ); Map fields = new HashMap<>() {{ put("field.foo", 1.0); @@ -302,6 +318,63 @@ public void testMapFieldsIfNecessary() { assertThat(fields, equalTo(expectedMap)); } + public void testReferenceCounting() throws IOException { + TrainedModelStatsService modelStatsService = mock(TrainedModelStatsService.class); + String modelId = "ref-count-model"; + List inputFields = Arrays.asList("field.foo", "field.bar"); + InferenceDefinition definition = InferenceDefinition.builder() + .setTrainedModel(buildClassificationInference(false)) + .build(); + + { + CircuitBreaker breaker = mock(CircuitBreaker.class); + LocalModel model = new LocalModel(modelId, + "test-node", + definition, + new TrainedModelInput(inputFields), + null, + ClassificationConfig.EMPTY_PARAMS, + License.OperationMode.PLATINUM, + modelStatsService, + breaker + ); + + model.release(); + verify(breaker, times(1)).addWithoutBreaking(eq(-definition.ramBytesUsed())); + verifyNoMoreInteractions(breaker); + assertEquals(0L, model.getReferenceCount()); + + // reacquire + model.acquire(); + verify(breaker, times(1)).addEstimateBytesAndMaybeBreak(eq(definition.ramBytesUsed()), eq(modelId)); + verifyNoMoreInteractions(breaker); + assertEquals(1L, model.getReferenceCount()); + } + + { + CircuitBreaker breaker = mock(CircuitBreaker.class); + LocalModel model = new LocalModel(modelId, + "test-node", + definition, + new TrainedModelInput(inputFields), + null, + ClassificationConfig.EMPTY_PARAMS, + License.OperationMode.PLATINUM, + modelStatsService, + breaker + ); + + model.acquire(); + model.acquire(); + model.release(); + model.release(); + model.release(); + verify(breaker, times(1)).addWithoutBreaking(eq(-definition.ramBytesUsed())); + verifyNoMoreInteractions(breaker); + assertEquals(0L, model.getReferenceCount()); + } + } + private static SingleValueInferenceResults getSingleValue(LocalModel model, Map fields, InferenceConfigUpdate config) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java index a17a715d1b48c..9a7f671aa52a1 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/loadingservice/ModelLoadingServiceTests.java @@ -58,6 +58,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import static org.elasticsearch.xpack.ml.MachineLearning.UTILITY_THREAD_POOL_NAME; @@ -451,6 +452,89 @@ public void testCircuitBreakerBreak() throws Exception { }); } + public void testReferenceCounting() throws Exception { + String modelId = "test-reference-counting"; + withTrainedModel(modelId, 1L); + + ModelLoadingService modelLoadingService = new ModelLoadingService(trainedModelProvider, + auditor, + threadPool, + clusterService, + trainedModelStatsService, + Settings.EMPTY, + "test-node", + circuitBreaker); + + modelLoadingService.clusterChanged(ingestChangedEvent(modelId)); + + PlainActionFuture forPipeline = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, forPipeline); + final LocalModel model = forPipeline.get(); + assertBusy(() -> assertEquals(2, model.getReferenceCount())); + + PlainActionFuture forSearch = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, forSearch); + forSearch.get(); + assertBusy(() -> assertEquals(3, model.getReferenceCount())); + + model.release(); + assertBusy(() -> assertEquals(2, model.getReferenceCount())); + + PlainActionFuture forSearch2 = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, forSearch2); + forSearch2.get(); + assertBusy(() -> assertEquals(3, model.getReferenceCount())); + } + + public void testReferenceCountingForPipeline() throws Exception { + String modelId = "test-reference-counting-for-pipeline"; + withTrainedModel(modelId, 1L); + + ModelLoadingService modelLoadingService = new ModelLoadingService(trainedModelProvider, + auditor, + threadPool, + clusterService, + trainedModelStatsService, + Settings.EMPTY, + "test-node", + circuitBreaker); + + modelLoadingService.clusterChanged(ingestChangedEvent(modelId)); + + PlainActionFuture forPipeline = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, forPipeline); + final LocalModel model = forPipeline.get(); + assertBusy(() -> assertEquals(2, model.getReferenceCount())); + + PlainActionFuture forPipeline2 = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, forPipeline2); + forPipeline2.get(); + assertBusy(() -> assertEquals(3, model.getReferenceCount())); + + // will cause the model to be evicted + modelLoadingService.clusterChanged(ingestChangedEvent()); + assertBusy(() -> assertEquals(2, model.getReferenceCount())); + } + + public void testReferenceCounting_ModelIsNotCached() throws ExecutionException, InterruptedException { + String modelId = "test-reference-counting-not-cached"; + withTrainedModel(modelId, 1L); + + ModelLoadingService modelLoadingService = new ModelLoadingService(trainedModelProvider, + auditor, + threadPool, + clusterService, + trainedModelStatsService, + Settings.EMPTY, + "test-node", + circuitBreaker); + + PlainActionFuture future = new PlainActionFuture<>(); + modelLoadingService.getModelForPipeline(modelId, future); + LocalModel model = future.get(); + assertEquals(1, model.getReferenceCount()); + } + @SuppressWarnings("unchecked") private void withTrainedModel(String modelId, long size) { InferenceDefinition definition = mock(InferenceDefinition.class); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java index b62cd98b79613..82a34f6047359 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizationStatus; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; @@ -68,6 +67,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.same; import static org.mockito.Mockito.doThrow; @@ -293,6 +293,9 @@ public void testProcessResult_annotation() { verify(persister).bulkPersisterBuilder(eq(JOB_ID)); verify(bulkAnnotationsPersister).persistAnnotation(annotation); + if (annotation.getEvent() == Annotation.Event.CATEGORIZATION_STATUS_CHANGE) { + verify(auditor).warning(eq(JOB_ID), anyString()); + } } public void testProcessResult_modelSizeStats() { @@ -343,46 +346,27 @@ public void testProcessResult_modelSizeStatsWithMemoryStatusChanges() { verify(auditor).error(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_MEMORY_STATUS_HARD_LIMIT, "512mb", "1kb")); } - public void testProcessResult_modelSizeStatsWithCategorizationStatusChanges() { + public void testProcessResult_categorizationStatusChangeAnnotationCausesNotification() { AutodetectResult result = mock(AutodetectResult.class); processorUnderTest.setDeleteInterimRequired(false); - // First one with ok - ModelSizeStats modelSizeStats = - new ModelSizeStats.Builder(JOB_ID).setCategorizationStatus(CategorizationStatus.OK).build(); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); - processorUnderTest.processResult(result); - - // Now one with warn - modelSizeStats = new ModelSizeStats.Builder(JOB_ID).setCategorizationStatus(CategorizationStatus.WARN).build(); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); - processorUnderTest.processResult(result); - - // Another with warn - modelSizeStats = new ModelSizeStats.Builder(JOB_ID).setCategorizationStatus(CategorizationStatus.WARN).build(); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); - processorUnderTest.processResult(result); - - verify(persister).bulkPersisterBuilder(eq(JOB_ID)); - verify(persister, times(3)).persistModelSizeStats(any(ModelSizeStats.class), any()); - // We should have only fired one notification; only the change from ok to warn should have fired, not the subsequent warn - verify(auditor).warning(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_CATEGORIZATION_STATUS_WARN, "warn", 0)); - } - - public void testProcessResult_modelSizeStatsWithFirstCategorizationStatusWarn() { - AutodetectResult result = mock(AutodetectResult.class); - processorUnderTest.setDeleteInterimRequired(false); - - // First one with warn - this works because a default constructed ModelSizeStats has CategorizationStatus.OK - ModelSizeStats modelSizeStats = - new ModelSizeStats.Builder(JOB_ID).setCategorizationStatus(CategorizationStatus.WARN).build(); - when(result.getModelSizeStats()).thenReturn(modelSizeStats); + Annotation annotation = new Annotation.Builder() + .setType(Annotation.Type.ANNOTATION) + .setJobId(JOB_ID) + .setAnnotation("Categorization status changed to 'warn' for partition 'foo'") + .setEvent(Annotation.Event.CATEGORIZATION_STATUS_CHANGE) + .setCreateTime(new Date()) + .setCreateUsername(XPackUser.NAME) + .setTimestamp(new Date()) + .setPartitionFieldName("part") + .setPartitionFieldValue("foo") + .build(); + when(result.getAnnotation()).thenReturn(annotation); processorUnderTest.processResult(result); verify(persister).bulkPersisterBuilder(eq(JOB_ID)); - verify(persister).persistModelSizeStats(any(ModelSizeStats.class), any()); - // We should have only fired one notification; only the change from ok to warn should have fired, not the subsequent warn - verify(auditor).warning(JOB_ID, Messages.getMessage(Messages.JOB_AUDIT_CATEGORIZATION_STATUS_WARN, "warn", 0)); + verify(bulkAnnotationsPersister).persistAnnotation(annotation); + verify(auditor).warning(JOB_ID, "Categorization status changed to 'warn' for partition 'foo' after 0 buckets"); } public void testProcessResult_modelSnapshot() { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java index 6af22aa57c06b..8d6df3004b019 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportRollupSearchAction.java @@ -100,7 +100,7 @@ public TransportRollupSearchAction(TransportService transportService, @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { - String[] indices = resolver.concreteIndexNames(clusterService.state(), request.indicesOptions(), request.indices()); + String[] indices = resolver.concreteIndexNames(clusterService.state(), request); RollupSearchContext rollupSearchContext = separateIndices(indices, clusterService.state().getMetadata().indices()); MultiSearchRequest msearch = createMSearchRequest(request, registry, rollupSearchContext); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java index 9bc7b2b391771..d12605c27f3a6 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; +import org.elasticsearch.search.aggregations.bucket.histogram.LongBounds; import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.range.GeoDistanceAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; @@ -59,7 +59,7 @@ public void testBasicDateHisto() { DateHistogramAggregationBuilder histo = new DateHistogramAggregationBuilder("test_histo"); histo.calendarInterval(new DateHistogramInterval("1d")) .field("foo") - .extendedBounds(new ExtendedBounds(0L, 1000L)) + .extendedBounds(new LongBounds(0L, 1000L)) .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); @@ -95,7 +95,7 @@ public void testFormattedDateHisto() { DateHistogramAggregationBuilder histo = new DateHistogramAggregationBuilder("test_histo"); histo.calendarInterval(new DateHistogramInterval("1d")) .field("foo") - .extendedBounds(new ExtendedBounds(0L, 1000L)) + .extendedBounds(new LongBounds(0L, 1000L)) .format("yyyy-MM-dd") .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index 111f9b4ccfbaa..9bcc5accfd340 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -20,6 +20,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobContainer; @@ -52,6 +53,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; @@ -463,6 +465,17 @@ public static Directory create( ); } + if (indexSettings.hasCustomDataPath()) { + // cache management requires the shard data path to be in a non-custom location + throw new IllegalArgumentException( + "setting [" + + IndexMetadata.INDEX_DATA_PATH_SETTING.getKey() + + "] is not permitted on searchable snapshots, but was [" + + IndexMetadata.INDEX_DATA_PATH_SETTING.get(indexSettings.getSettings()) + + "]" + ); + } + final Repository repository = repositories.repository(SNAPSHOT_REPOSITORY_SETTING.get(indexSettings.getSettings())); if (repository instanceof BlobStoreRepository == false) { throw new IllegalArgumentException("Repository [" + repository + "] is not searchable"); @@ -485,8 +498,9 @@ public static Directory create( () -> blobStoreRepository.loadShardSnapshot(lazyBlobContainer.getOrCompute(), snapshotId) ); - final Path cacheDir = shardPath.getDataPath().resolve("snapshots").resolve(snapshotId.getUUID()); + final Path cacheDir = CacheService.getShardCachePath(shardPath).resolve(snapshotId.getUUID()); Files.createDirectories(cacheDir); + assert assertCacheIsEmpty(cacheDir); return new InMemoryNoOpCommitDirectory( new SearchableSnapshotDirectory( @@ -504,6 +518,17 @@ public static Directory create( ); } + private static boolean assertCacheIsEmpty(Path cacheDir) { + try (DirectoryStream cacheDirStream = Files.newDirectoryStream(cacheDir)) { + final Set cacheFiles = new HashSet<>(); + cacheDirStream.forEach(cacheFiles::add); + assert cacheFiles.isEmpty() : "should start with empty cache, but found " + cacheFiles; + } catch (IOException e) { + assert false : e; + } + return true; + } + public static SearchableSnapshotDirectory unwrapDirectory(Directory dir) { while (dir != null) { if (dir instanceof SearchableSnapshotDirectory) { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index c87dfcfec1434..083aab4a5141a 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -57,6 +57,7 @@ import org.elasticsearch.xpack.searchablesnapshots.action.TransportRepositoryStatsAction; import org.elasticsearch.xpack.searchablesnapshots.action.TransportSearchableSnapshotsStatsAction; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; +import org.elasticsearch.xpack.searchablesnapshots.cache.NodeEnvironmentCacheCleaner; import org.elasticsearch.xpack.searchablesnapshots.rest.RestClearSearchableSnapshotsCacheAction; import org.elasticsearch.xpack.searchablesnapshots.rest.RestMountSearchableSnapshotAction; import org.elasticsearch.xpack.searchablesnapshots.rest.RestRepositoryStatsAction; @@ -182,7 +183,7 @@ public Collection createComponents( final Supplier repositoriesServiceSupplier ) { if (SEARCHABLE_SNAPSHOTS_FEATURE_ENABLED) { - final CacheService cacheService = new CacheService(settings); + final CacheService cacheService = new CacheService(new NodeEnvironmentCacheCleaner(nodeEnvironment), settings); this.cacheService.set(cacheService); this.repositoriesServiceSupplier = repositoriesServiceSupplier; this.threadPool.set(threadPool); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index a5d99ce9f6287..56338d41f353a 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -159,6 +160,9 @@ protected void masterOperation( // by one with the same name while we are restoring it) or else the index metadata might bear no relation to the snapshot we're // searching. + final String[] ignoreIndexSettings = Arrays.copyOf(request.ignoreIndexSettings(), request.ignoreIndexSettings().length + 1); + ignoreIndexSettings[ignoreIndexSettings.length - 1] = IndexMetadata.SETTING_DATA_PATH; + client.admin() .cluster() .restoreSnapshot( @@ -178,7 +182,7 @@ protected void masterOperation( .build() ) // Pass through ignored index settings - .ignoreIndexSettings(request.ignoreIndexSettings()) + .ignoreIndexSettings(ignoreIndexSettings) // Don't include global state .includeGlobalState(false) // Don't include aliases diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/CacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/CacheService.java index 0abba9b20ef61..b025df390e03c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/CacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/CacheService.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.cache.CacheFile; import org.elasticsearch.index.store.cache.CacheKey; @@ -49,15 +50,17 @@ public class CacheService extends AbstractLifecycleComponent { private final Cache cache; private final ByteSizeValue cacheSize; + private final Runnable cacheCleaner; private final ByteSizeValue rangeSize; - public CacheService(final Settings settings) { - this(SNAPSHOT_CACHE_SIZE_SETTING.get(settings), SNAPSHOT_CACHE_RANGE_SIZE_SETTING.get(settings)); + public CacheService(final Runnable cacheCleaner, final Settings settings) { + this(cacheCleaner, SNAPSHOT_CACHE_SIZE_SETTING.get(settings), SNAPSHOT_CACHE_RANGE_SIZE_SETTING.get(settings)); } - // overridable by tests - public CacheService(final ByteSizeValue cacheSize, final ByteSizeValue rangeSize) { + // exposed for tests + public CacheService(final Runnable cacheCleaner, final ByteSizeValue cacheSize, final ByteSizeValue rangeSize) { this.cacheSize = Objects.requireNonNull(cacheSize); + this.cacheCleaner = Objects.requireNonNull(cacheCleaner); this.rangeSize = Objects.requireNonNull(rangeSize); this.cache = CacheBuilder.builder() .setMaximumWeight(cacheSize.getBytes()) @@ -68,9 +71,13 @@ public CacheService(final ByteSizeValue cacheSize, final ByteSizeValue rangeSize .build(); } + public static Path getShardCachePath(ShardPath shardPath) { + return shardPath.getDataPath().resolve("snapshot_cache"); + } + @Override protected void doStart() { - // NORELEASE TODO clean up (or rebuild) cache from disk as a node crash may leave cached files + cacheCleaner.run(); } @Override @@ -83,6 +90,7 @@ protected void doClose() {} private void ensureLifecycleStarted() { final Lifecycle.State state = lifecycleState(); + assert state != Lifecycle.State.INITIALIZED : state; if (state != Lifecycle.State.STARTED) { throw new IllegalStateException("Failed to read data from cache: cache service is not started [" + state + "]"); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/NodeEnvironmentCacheCleaner.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/NodeEnvironmentCacheCleaner.java new file mode 100644 index 0000000000000..f7edc2c663bb8 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/NodeEnvironmentCacheCleaner.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.cache; + +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Cleans any leftover searchable snapshot caches when a node is starting up. + */ +public class NodeEnvironmentCacheCleaner implements Runnable { + + private final NodeEnvironment nodeEnvironment; + + public NodeEnvironmentCacheCleaner(NodeEnvironment nodeEnvironment) { + this.nodeEnvironment = nodeEnvironment; + } + + @Override + public void run() { + try { + for (NodeEnvironment.NodePath nodePath : nodeEnvironment.nodePaths()) { + for (String indexUUID : nodeEnvironment.availableIndexFoldersForPath(nodePath)) { + for (ShardId shardId : nodeEnvironment.findAllShardIds(new Index("_unknown_", indexUUID))) { + final Path shardDataPath = nodePath.resolve(shardId); + final ShardPath shardPath = new ShardPath(false, shardDataPath, shardDataPath, shardId); + final Path shardCachePath = CacheService.getShardCachePath(shardPath); + if (Files.isDirectory(shardCachePath)) { + IOUtils.rm(shardCachePath); + } + } + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java index 20e42ca0ef1da..c5c5e2aa387c6 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryStatsTests.java @@ -565,7 +565,7 @@ private static void executeTestCaseWithCache( final TriConsumer test ) { executeTestCase( - new CacheService(cacheSize, cacheRangeSize), + new CacheService(TestUtils::noOpCacheCleaner, cacheSize, cacheRangeSize), Settings.builder() .put(SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true) .put(SNAPSHOT_CACHE_PREWARM_ENABLED_SETTING.getKey(), false) // disable prewarming as it impacts the stats diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java index 505ffd0de332a..55ed40fd094bb 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/SearchableSnapshotDirectoryTests.java @@ -64,6 +64,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; +import org.elasticsearch.index.store.cache.TestUtils; import org.elasticsearch.index.store.checksum.ChecksumBlobContainerIndexInput; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -548,7 +549,7 @@ protected void assertSnapshotOrGenericThread() { final BlobStoreIndexShardSnapshot snapshot = repository.loadShardSnapshot(blobContainer, snapshotId); final Path cacheDir = createTempDir(); - final CacheService cacheService = new CacheService(Settings.EMPTY); + final CacheService cacheService = TestUtils.createDefaultCacheService(); releasables.add(cacheService); cacheService.start(); @@ -604,7 +605,7 @@ private void testIndexInputs(final CheckedBiConsumer indexRequestBuilders = new ArrayList<>(); + for (int i = between(10, maxIndexRequests); i >= 0; i--) { + indexRequestBuilders.add(client().prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); + } + indexRandom(true, true, indexRequestBuilders); + refresh(indexName); + assertThat( + client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), + equalTo(0) + ); + } } diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java index a9861c9ffdfd3..4eda382137b48 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/ClusterStateApplierOrderingTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -43,10 +42,7 @@ public void testRepositoriesServiceClusterStateApplierIsCalledBeforeIndicesClust final String indexName = "test-index"; final String restoredIndexName = "restored-index"; - final Path repo = randomRepoPath(); - assertAcked( - client().admin().cluster().preparePutRepository(fsRepoName).setType("fs").setSettings(Settings.builder().put("location", repo)) - ); + createRepo(fsRepoName); // Peer recovery always copies .liv files but we do not permit writing to searchable snapshot directories so this doesn't work, but // we can bypass this by forcing soft deletes to be used. TODO this restriction can be lifted when #55142 is resolved. diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCacheClearingIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCacheClearingIntegTests.java new file mode 100644 index 0000000000000..9dc498ba8f830 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCacheClearingIntegTests.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots; + +import org.apache.lucene.mockfile.FilterFileSystemProvider; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.io.PathUtilsForTesting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; +import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; +import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX; +import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class SearchableSnapshotsCacheClearingIntegTests extends BaseSearchableSnapshotsIntegTestCase { + + private static DeleteBlockingFileSystemProvider deleteBlockingFileSystemProvider; + + @BeforeClass + public static void installDeleteBlockingFileSystemProvider() { + FileSystem current = PathUtils.getDefaultFileSystem(); + deleteBlockingFileSystemProvider = new DeleteBlockingFileSystemProvider(current); + PathUtilsForTesting.installMock(deleteBlockingFileSystemProvider.getFileSystem(null)); + } + + @AfterClass + public static void removeDeleteBlockingFileSystemProvider() { + PathUtilsForTesting.teardown(); + } + + void startBlockingDeletes() { + deleteBlockingFileSystemProvider.injectFailures.set(true); + } + + void stopBlockingDeletes() { + deleteBlockingFileSystemProvider.injectFailures.set(false); + } + + private static class DeleteBlockingFileSystemProvider extends FilterFileSystemProvider { + + AtomicBoolean injectFailures = new AtomicBoolean(); + + DeleteBlockingFileSystemProvider(FileSystem inner) { + super("deleteblocking://", inner); + } + + @Override + public boolean deleteIfExists(Path path) throws IOException { + if (injectFailures.get()) { + throw new IOException("blocked deletion of " + path); + } else { + return super.deleteIfExists(path); + } + } + + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + // ensure the cache is definitely used + .put(CacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(1L, ByteSizeUnit.GB)) + .build(); + } + + public void testCacheDirectoriesRemovedOnStartup() throws Exception { + final String fsRepoName = randomAlphaOfLength(10); + final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + + createRepo(fsRepoName); + + final Settings.Builder originalIndexSettings = Settings.builder() + .put(INDEX_SOFT_DELETES_SETTING.getKey(), true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + createAndPopulateIndex(indexName, originalIndexSettings); + + CreateSnapshotResponse createSnapshotResponse = client().admin() + .cluster() + .prepareCreateSnapshot(fsRepoName, snapshotName) + .setWaitForCompletion(true) + .get(); + final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo(); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + assertAcked(client().admin().indices().prepareDelete(indexName)); + + final DiscoveryNodes discoveryNodes = client().admin().cluster().prepareState().clear().setNodes(true).get().getState().nodes(); + final String dataNode = randomFrom(discoveryNodes.getDataNodes().values().toArray(DiscoveryNode.class)).getName(); + + final MountSearchableSnapshotRequest req = new MountSearchableSnapshotRequest( + restoredIndexName, + fsRepoName, + snapshotName, + indexName, + Settings.builder().put(INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name", dataNode).build(), + Strings.EMPTY_ARRAY, + true + ); + + final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); + ensureGreen(restoredIndexName); + + final Index restoredIndex = client().admin() + .cluster() + .prepareState() + .clear() + .setMetadata(true) + .get() + .getState() + .metadata() + .index(restoredIndexName) + .getIndex(); + + final IndexService indexService = internalCluster().getInstance(IndicesService.class, dataNode).indexService(restoredIndex); + final Path shardCachePath = CacheService.getShardCachePath(indexService.getShard(0).shardPath()); + assertTrue(Files.isDirectory(shardCachePath)); + final Set cacheFiles = new HashSet<>(); + try (DirectoryStream snapshotCacheStream = Files.newDirectoryStream(shardCachePath)) { + for (final Path snapshotCachePath : snapshotCacheStream) { + assertTrue(snapshotCachePath + " should be a directory", Files.isDirectory(snapshotCachePath)); + try (DirectoryStream cacheFileStream = Files.newDirectoryStream(snapshotCachePath)) { + for (final Path cacheFilePath : cacheFileStream) { + assertTrue(cacheFilePath + " should be a file", Files.isRegularFile(cacheFilePath)); + cacheFiles.add(cacheFilePath); + } + } + } + } + assertFalse("no cache files found", cacheFiles.isEmpty()); + + startBlockingDeletes(); + internalCluster().restartNode(dataNode, new InternalTestCluster.RestartCallback() { + @Override + public Settings onNodeStopped(String nodeName) { + assertTrue(Files.isDirectory(shardCachePath)); + for (Path cacheFile : cacheFiles) { + assertTrue(cacheFile + " should not have been cleaned up yet", Files.isRegularFile(cacheFile)); + } + stopBlockingDeletes(); + return Settings.EMPTY; + } + }); + + ensureGreen(restoredIndexName); + + for (Path cacheFile : cacheFiles) { + assertFalse(cacheFile + " should have been cleaned up", Files.exists(cacheFile)); + } + + assertAcked(client().admin().indices().prepareDelete(restoredIndexName)); + } +} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 9ff6b959c07f5..25eb740955bc3 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -89,16 +89,7 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { assertAcked(prepareCreate(indexName, Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true))); assertAcked(client().admin().indices().prepareAliases().addAlias(indexName, aliasName)); - final List indexRequestBuilders = new ArrayList<>(); - for (int i = between(10, 10_000); i >= 0; i--) { - indexRequestBuilders.add(client().prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); - } - indexRandom(true, true, indexRequestBuilders); - refresh(indexName); - assertThat( - client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), - equalTo(0) - ); + populateIndex(indexName, 10_000); final TotalHits originalAllHits = internalCluster().client() .prepareSearch(indexName) @@ -359,6 +350,7 @@ public void testCanMountSnapshotTakenWhileConcurrentlyIndexing() throws Exceptio ensureGreen(restoredIndexName); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/59287") public void testMaxRestoreBytesPerSecIsUsed() throws Exception { final String repositoryName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); final Settings.Builder repositorySettings = Settings.builder().put("location", randomRepoPath()); @@ -441,10 +433,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception { final String restoredIndexName = randomBoolean() ? indexName : randomAlphaOfLength(10).toLowerCase(Locale.ROOT); final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - final Path repo = randomRepoPath(); - assertAcked( - client().admin().cluster().preparePutRepository(fsRepoName).setType("fs").setSettings(Settings.builder().put("location", repo)) - ); + createRepo(fsRepoName); final int dataNodesCount = internalCluster().numDataNodes(); final Settings.Builder originalIndexSettings = Settings.builder(); @@ -459,19 +448,7 @@ public void testMountedSnapshotHasNoReplicasByDefault() throws Exception { replicaLimit == dataNodesCount ? "0-all" : "0-" + replicaLimit ); } - assertAcked(prepareCreate(indexName, originalIndexSettings)); - ensureGreen(indexName); - - final List indexRequestBuilders = new ArrayList<>(); - for (int i = between(10, 100); i >= 0; i--) { - indexRequestBuilders.add(client().prepareIndex(indexName).setSource("foo", randomBoolean() ? "bar" : "baz")); - } - indexRandom(true, true, indexRequestBuilders); - refresh(indexName); - assertThat( - client().admin().indices().prepareForceMerge(indexName).setOnlyExpungeDeletes(true).setFlush(true).get().getFailedShards(), - equalTo(0) - ); + createAndPopulateIndex(indexName, originalIndexSettings); CreateSnapshotResponse createSnapshotResponse = client().admin() .cluster() diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java index 2878f8e91c19e..b8e06bae91223 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsLicenseIntegTests.java @@ -49,7 +49,6 @@ import org.elasticsearch.xpack.searchablesnapshots.action.SearchableSnapshotsStatsResponse; import org.junit.Before; -import java.nio.file.Path; import java.util.concurrent.ExecutionException; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -68,10 +67,7 @@ public class SearchableSnapshotsLicenseIntegTests extends BaseSearchableSnapshot @Before public void createAndMountSearchableSnapshot() throws Exception { - final Path repo = randomRepoPath(); - assertAcked( - client().admin().cluster().preparePutRepository(repoName).setType("fs").setSettings(Settings.builder().put("location", repo)) - ); + createRepo(repoName); createIndex(indexName); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java index e63af77b6c461..00b00718525a3 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/action/MountSearchableSnapshotRequestTests.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.searchablesnapshots.action; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; @@ -15,6 +17,8 @@ import java.util.Arrays; +import static org.hamcrest.Matchers.containsString; + public class MountSearchableSnapshotRequestTests extends AbstractWireSerializingTestCase { private MountSearchableSnapshotRequest randomState(MountSearchableSnapshotRequest instance) { @@ -168,4 +172,17 @@ private static String[] mutateStringArray(String[] strings) { return Strings.EMPTY_ARRAY; } } + + public void testForbidsCustomDataPath() { + final ActionRequestValidationException validationException = new MountSearchableSnapshotRequest( + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + randomAlphaOfLength(5), + Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, randomAlphaOfLength(5)).build(), + Strings.EMPTY_ARRAY, + randomBoolean() + ).validate(); + assertThat(validationException.getMessage(), containsString(IndexMetadata.SETTING_DATA_PATH)); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java index 8445238738d5b..432b80578c76f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrail.java @@ -10,7 +10,6 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; @@ -22,9 +21,9 @@ public interface AuditTrail { String name(); - void authenticationSuccess(String requestId, String realm, User user, RestRequest request); + void authenticationSuccess(String requestId, Authentication authentication, RestRequest request); - void authenticationSuccess(String requestId, String realm, User user, String action, TransportRequest transportRequest); + void authenticationSuccess(String requestId, Authentication authentication, String action, TransportRequest transportRequest); void anonymousAccessDenied(String requestId, String action, TransportRequest transportRequest); @@ -52,7 +51,7 @@ void accessDenied(String requestId, Authentication authentication, String action void tamperedRequest(String requestId, String action, TransportRequest transportRequest); - void tamperedRequest(String requestId, User user, String action, TransportRequest transportRequest); + void tamperedRequest(String requestId, Authentication authentication, String action, TransportRequest transportRequest); /** * The {@link #connectionGranted(InetAddress, String, SecurityIpFilterRule)} and diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java index 018f88cfdf38e..4449cfca8190e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java @@ -12,7 +12,6 @@ import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; @@ -54,10 +53,11 @@ public String name() { } @Override - public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) {} + public void authenticationSuccess(String requestId, Authentication authentication, RestRequest request) {} @Override - public void authenticationSuccess(String requestId, String realm, User user, String action, TransportRequest transportRequest) {} + public void authenticationSuccess(String requestId, Authentication authentication, String action, + TransportRequest transportRequest) {} @Override public void anonymousAccessDenied(String requestId, String action, TransportRequest transportRequest) {} @@ -99,7 +99,7 @@ public void tamperedRequest(String requestId, RestRequest request) {} public void tamperedRequest(String requestId, String action, TransportRequest transportRequest) {} @Override - public void tamperedRequest(String requestId, User user, String action, TransportRequest transportRequest) {} + public void tamperedRequest(String requestId, Authentication authentication, String action, TransportRequest transportRequest) {} @Override public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) {} @@ -143,16 +143,17 @@ public String name() { } @Override - public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) { + public void authenticationSuccess(String requestId, Authentication authentication, RestRequest request) { for (AuditTrail auditTrail : auditTrails) { - auditTrail.authenticationSuccess(requestId, realm, user, request); + auditTrail.authenticationSuccess(requestId, authentication, request); } } @Override - public void authenticationSuccess(String requestId, String realm, User user, String action, TransportRequest transportRequest) { + public void authenticationSuccess(String requestId, Authentication authentication, String action, + TransportRequest transportRequest) { for (AuditTrail auditTrail : auditTrails) { - auditTrail.authenticationSuccess(requestId, realm, user, action, transportRequest); + auditTrail.authenticationSuccess(requestId, authentication, action, transportRequest); } } @@ -244,9 +245,9 @@ public void tamperedRequest(String requestId, String action, TransportRequest tr } @Override - public void tamperedRequest(String requestId, User user, String action, TransportRequest transportRequest) { + public void tamperedRequest(String requestId, Authentication authentication, String action, TransportRequest transportRequest) { for (AuditTrail auditTrail : auditTrails) { - auditTrail.tamperedRequest(requestId, user, action, transportRequest); + auditTrail.tamperedRequest(requestId, authentication, action, transportRequest); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 3be2ade57034d..d9182dbb010e2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.rest.RemoteHostHeader; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; @@ -106,7 +107,10 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { public static final String PRINCIPAL_REALM_FIELD_NAME = "user.realm"; public static final String PRINCIPAL_RUN_BY_REALM_FIELD_NAME = "user.run_by.realm"; public static final String PRINCIPAL_RUN_AS_REALM_FIELD_NAME = "user.run_as.realm"; + public static final String API_KEY_ID_FIELD_NAME = "apikey.id"; + public static final String API_KEY_NAME_FIELD_NAME = "apikey.name"; public static final String PRINCIPAL_ROLES_FIELD_NAME = "user.roles"; + public static final String AUTHENTICATION_TYPE_FIELD_NAME = "authentication.type"; public static final String REALM_FIELD_NAME = "realm"; public static final String URL_PATH_FIELD_NAME = "url.path"; public static final String URL_QUERY_FIELD_NAME = "url.query"; @@ -231,16 +235,22 @@ public LoggingAuditTrail(Settings settings, ClusterService clusterService, Threa } @Override - public void authenticationSuccess(String requestId, String realm, User user, RestRequest request) { + public void authenticationSuccess(String requestId, Authentication authentication, RestRequest request) { if (events.contains(AUTHENTICATION_SUCCESS) && eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), Optional.empty())) == false) { + .test(new AuditEventMetaInfo( + Optional.of(authentication.getUser()), + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), + Optional.empty(), + Optional.empty())) == false) { + // this is redundant information maintained for bwc purposes + final String authnRealm = authentication.getAuthenticatedBy().getName(); final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "authentication_success") - .with(REALM_FIELD_NAME, realm) + .with(REALM_FIELD_NAME, authnRealm) .withRestUriAndMethod(request) .withRequestId(requestId) - .withPrincipal(user) + .withAuthentication(authentication) .withRestOrigin(request) .withRequestBody(request) .withOpaqueId(threadContext) @@ -251,19 +261,22 @@ public void authenticationSuccess(String requestId, String realm, User user, Res } @Override - public void authenticationSuccess(String requestId, String realm, User user, String action, TransportRequest transportRequest) { + public void authenticationSuccess(String requestId, Authentication authentication, String action, TransportRequest transportRequest) { if (events.contains(AUTHENTICATION_SUCCESS)) { final Optional indices = indices(transportRequest); if (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), indices)) == false) { + .test(new AuditEventMetaInfo( + Optional.of(authentication.getUser()), + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), + Optional.empty(), + indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "authentication_success") - .with(REALM_FIELD_NAME, realm) .with(ACTION_FIELD_NAME, action) .with(REQUEST_NAME_FIELD_NAME, transportRequest.getClass().getSimpleName()) .withRequestId(requestId) - .withPrincipal(user) + .withAuthentication(authentication) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .withOpaqueId(threadContext) @@ -448,14 +461,14 @@ public void accessGranted(String requestId, Authentication authentication, Strin if ((isSystem && events.contains(SYSTEM_ACCESS_GRANTED)) || ((isSystem == false) && events.contains(ACCESS_GRANTED))) { final Optional indices = indices(msg); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(user), - Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "access_granted") .with(ACTION_FIELD_NAME, action) .with(REQUEST_NAME_FIELD_NAME, msg.getClass().getSimpleName()) .withRequestId(requestId) - .withSubject(authentication) + .withAuthentication(authentication) .withRestOrTransportOrigin(msg, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .withOpaqueId(threadContext) @@ -478,7 +491,7 @@ public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Aut } if (events.contains(eventType)) { if (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(effectiveRealmName(authentication)), + .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), Optional.ofNullable(indices))) == false) { final LogEntryBuilder logEntryBuilder = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) @@ -486,7 +499,7 @@ public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Aut .with(ACTION_FIELD_NAME, action) .with(REQUEST_NAME_FIELD_NAME, requestName) .withRequestId(requestId) - .withSubject(authentication) + .withAuthentication(authentication) .with(INDICES_FIELD_NAME, indices) .withOpaqueId(threadContext) .withXForwardedFor(threadContext) @@ -512,14 +525,14 @@ public void accessDenied(String requestId, Authentication authentication, String if (events.contains(ACCESS_DENIED)) { final Optional indices = indices(transportRequest); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), - Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "access_denied") .with(ACTION_FIELD_NAME, action) .with(REQUEST_NAME_FIELD_NAME, transportRequest.getClass().getSimpleName()) .withRequestId(requestId) - .withSubject(authentication) + .withAuthentication(authentication) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .with(authorizationInfo.asMap()) @@ -571,11 +584,14 @@ public void tamperedRequest(String requestId, String action, TransportRequest tr } @Override - public void tamperedRequest(String requestId, User user, String action, TransportRequest transportRequest) { + public void tamperedRequest(String requestId, Authentication authentication, String action, TransportRequest transportRequest) { if (events.contains(TAMPERED_REQUEST)) { final Optional indices = indices(transportRequest); - if (eventFilterPolicyRegistry.ignorePredicate() - .test(new AuditEventMetaInfo(Optional.of(user), Optional.empty(), Optional.empty(), indices)) == false) { + if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo( + Optional.of(authentication.getUser()), + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), + Optional.empty(), + indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "tampered_request") @@ -583,7 +599,7 @@ public void tamperedRequest(String requestId, User user, String action, Transpor .with(REQUEST_NAME_FIELD_NAME, transportRequest.getClass().getSimpleName()) .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) - .withPrincipal(user) + .withAuthentication(authentication) .with(INDICES_FIELD_NAME, indices.orElse(null)) .withOpaqueId(threadContext) .withXForwardedFor(threadContext) @@ -635,7 +651,7 @@ public void runAsGranted(String requestId, Authentication authentication, String if (events.contains(RUN_AS_GRANTED)) { final Optional indices = indices(transportRequest); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), - Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "run_as_granted") @@ -660,7 +676,7 @@ public void runAsDenied(String requestId, Authentication authentication, String if (events.contains(RUN_AS_DENIED)) { final Optional indices = indices(transportRequest); if (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), - Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), Optional.of(authorizationInfo), indices)) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "run_as_denied") @@ -681,9 +697,10 @@ public void runAsDenied(String requestId, Authentication authentication, String @Override public void runAsDenied(String requestId, Authentication authentication, RestRequest request, AuthorizationInfo authorizationInfo) { - if (events.contains(RUN_AS_DENIED) - && eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), - Optional.of(effectiveRealmName(authentication)), Optional.of(authorizationInfo), Optional.empty())) == false) { + if (events.contains(RUN_AS_DENIED) && eventFilterPolicyRegistry.ignorePredicate().test( + new AuditEventMetaInfo(Optional.of(authentication.getUser()), + Optional.of(ApiKeyService.getCreatorRealmName(authentication)), + Optional.of(authorizationInfo), Optional.empty())) == false) { final StringMapMessage logEntry = new LogEntryBuilder() .with(EVENT_TYPE_FIELD_NAME, REST_ORIGIN_FIELD_VALUE) .with(EVENT_ACTION_FIELD_NAME, "run_as_denied") @@ -797,22 +814,22 @@ LogEntryBuilder withXForwardedFor(ThreadContext threadContext) { return this; } - LogEntryBuilder withPrincipal(User user) { - logEntry.with(PRINCIPAL_FIELD_NAME, user.principal()); - if (user.isRunAs()) { - logEntry.with(PRINCIPAL_RUN_BY_FIELD_NAME, user.authenticatedUser().principal()); - } - return this; - } - - LogEntryBuilder withSubject(Authentication authentication) { + LogEntryBuilder withAuthentication(Authentication authentication) { logEntry.with(PRINCIPAL_FIELD_NAME, authentication.getUser().principal()); - if (authentication.getUser().isRunAs()) { - logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) - .with(PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) - .with(PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + logEntry.with(AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString()); + if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType()) { + logEntry.with(API_KEY_ID_FIELD_NAME, (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY)) + .with(API_KEY_NAME_FIELD_NAME, (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY)) + .with(PRINCIPAL_REALM_FIELD_NAME, + (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME)); } else { - logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + if (authentication.getUser().isRunAs()) { + logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) + .with(PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) + .with(PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } else { + logEntry.with(PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } } return this; } @@ -878,11 +895,6 @@ private static Optional indices(TransportRequest transportRequest) { return Optional.empty(); } - private static String effectiveRealmName(Authentication authentication) { - return authentication.getLookedUpBy() != null ? authentication.getLookedUpBy().getName() - : authentication.getAuthenticatedBy().getName(); - } - public static void registerSettings(List> settings) { settings.add(EMIT_HOST_ADDRESS_SETTING); settings.add(EMIT_HOST_NAME_SETTING); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 5aa923f7b3d23..f299cd8e28f6c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -35,11 +35,13 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CharArrays; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; @@ -51,9 +53,13 @@ import org.elasticsearch.common.util.concurrent.ListenableFuture; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.InstantiatingObjectParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ObjectParserHelper; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -105,11 +111,16 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.action.bulk.TransportSingleItemBulkWriteAction.toSingleItemBulkRequest; import static org.elasticsearch.search.SearchService.DEFAULT_KEEPALIVE_SETTING; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; import static org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; +import static org.elasticsearch.xpack.core.security.authc.Authentication.VERSION_API_KEY_ROLES_AS_BYTES; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY; import static org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames.SECURITY_MAIN_ALIAS; import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; @@ -123,9 +134,6 @@ public class ApiKeyService { public static final String API_KEY_REALM_TYPE = "_es_api_key"; public static final String API_KEY_CREATOR_REALM_NAME = "_security_api_key_creator_realm_name"; public static final String API_KEY_CREATOR_REALM_TYPE = "_security_api_key_creator_realm_type"; - static final String API_KEY_ROLE_DESCRIPTORS_KEY = "_security_api_key_role_descriptors"; - static final String API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY = "_security_api_key_limited_by_role_descriptors"; - public static final Setting PASSWORD_HASHING_ALGORITHM = new Setting<>( "xpack.security.authc.api_key.hashing.algorithm", "pbkdf2", Function.identity(), v -> { @@ -221,7 +229,6 @@ private void createApiKeyAndIndexIt(Authentication authentication, CreateApiKeyR try (XContentBuilder builder = newDocument(apiKey, request.getName(), authentication, roleDescriptorSet, created, expiration, request.getRoleDescriptors(), version)) { - final IndexRequest indexRequest = client.prepareIndex(SECURITY_MAIN_ALIAS) .setSource(builder) @@ -349,8 +356,13 @@ private void loadApiKeyAndValidateCredentials(ThreadContext ctx, ApiKeyCredentia .request(); executeAsyncWithOrigin(ctx, SECURITY_ORIGIN, getRequest, ActionListener.wrap(response -> { if (response.isExists()) { - final Map source = response.getSource(); - validateApiKeyCredentials(docId, source, credentials, clock, ActionListener.delegateResponse(listener, (l, e) -> { + final ApiKeyDoc apiKeyDoc; + try (XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef(), XContentType.JSON)) { + apiKeyDoc = ApiKeyDoc.fromXContent(parser); + } + validateApiKeyCredentials(docId, apiKeyDoc, credentials, clock, ActionListener.delegateResponse(listener, (l, e) -> { if (ExceptionsHelper.unwrapCause(e) instanceof EsRejectedExecutionException) { listener.onResponse(AuthenticationResult.terminate("server is too busy to respond", e)); } else { @@ -374,6 +386,9 @@ private void loadApiKeyAndValidateCredentials(ThreadContext ctx, ApiKeyCredentia } /** + * This method is kept for BWC and should only be used for authentication objects created before v7.9.0. + * For authentication of newer versions, use {@link #getApiKeyIdAndRoleBytes} + * * The current request has been authenticated by an API key and this method enables the * retrieval of role descriptors that are associated with the api key */ @@ -381,10 +396,11 @@ public void getRoleForApiKey(Authentication authentication, ActionListener metadata = authentication.getMetadata(); final String apiKeyId = (String) metadata.get(API_KEY_ID_KEY); - final Map roleDescriptors = (Map) metadata.get(API_KEY_ROLE_DESCRIPTORS_KEY); final Map authnRoleDescriptors = (Map) metadata.get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY); @@ -400,6 +416,19 @@ public void getRoleForApiKey(Authentication authentication, ActionListener getApiKeyIdAndRoleBytes(Authentication authentication, boolean limitedBy) { + if (authentication.getAuthenticationType() != AuthenticationType.API_KEY) { + throw new IllegalStateException("authentication type must be api key but is " + authentication.getAuthenticationType()); + } + assert authentication.getVersion() + .onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES) : "This method only applies to authentication objects created on or after v7.9.0"; + + final Map metadata = authentication.getMetadata(); + return new Tuple<>( + (String) metadata.get(API_KEY_ID_KEY), + (BytesReference) metadata.get(limitedBy ? API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY : API_KEY_ROLE_DESCRIPTORS_KEY)); + } + public static class ApiKeyRoleDescriptors { private final String apiKeyId; @@ -446,27 +475,48 @@ private List parseRoleDescriptors(final String apiKeyId, final M }).collect(Collectors.toList()); } + public List parseRoleDescriptors(final String apiKeyId, BytesReference bytesReference) { + if (bytesReference == null) { + return Collections.emptyList(); + } + + List roleDescriptors = new ArrayList<>(); + try ( + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + new ApiKeyLoggingDeprecationHandler(deprecationLogger, apiKeyId), + bytesReference, + XContentType.JSON)) { + parser.nextToken(); // skip outer start object + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + parser.nextToken(); // role name + String roleName = parser.currentName(); + roleDescriptors.add(RoleDescriptor.parse(roleName, parser, false)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return Collections.unmodifiableList(roleDescriptors); + } + /** * Validates the ApiKey using the source map * @param docId the identifier of the document that was retrieved from the security index - * @param source the source map from a get of the ApiKey document + * @param apiKeyDoc the partially deserialized API key document * @param credentials the credentials provided by the user * @param listener the listener to notify after verification */ - void validateApiKeyCredentials(String docId, Map source, ApiKeyCredentials credentials, Clock clock, + void validateApiKeyCredentials(String docId, ApiKeyDoc apiKeyDoc, ApiKeyCredentials credentials, Clock clock, ActionListener listener) { - final String docType = (String) source.get("doc_type"); - final Boolean invalidated = (Boolean) source.get("api_key_invalidated"); - if ("api_key".equals(docType) == false) { + if ("api_key".equals(apiKeyDoc.docType) == false) { listener.onResponse( - AuthenticationResult.unsuccessful("document [" + docId + "] is [" + docType + "] not an api key", null)); - } else if (invalidated == null) { + AuthenticationResult.unsuccessful("document [" + docId + "] is [" + apiKeyDoc.docType + "] not an api key", null)); + } else if (apiKeyDoc.invalidated == null) { listener.onResponse(AuthenticationResult.unsuccessful("api key document is missing invalidated field", null)); - } else if (invalidated) { + } else if (apiKeyDoc.invalidated) { listener.onResponse(AuthenticationResult.unsuccessful("api key has been invalidated", null)); } else { - final String apiKeyHash = (String) source.get("api_key_hash"); - if (apiKeyHash == null) { + if (apiKeyDoc.hash == null) { throw new IllegalStateException("api key hash is missing"); } @@ -489,7 +539,7 @@ void validateApiKeyCredentials(String docId, Map source, ApiKeyC if (result.success) { if (result.verify(credentials.getKey())) { // move on - validateApiKeyExpiration(source, credentials, clock, listener); + validateApiKeyExpiration(apiKeyDoc, credentials, clock, listener); } else { listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); } @@ -497,17 +547,17 @@ void validateApiKeyCredentials(String docId, Map source, ApiKeyC listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); } else { apiKeyAuthCache.invalidate(credentials.getId(), listenableCacheEntry); - validateApiKeyCredentials(docId, source, credentials, clock, listener); + validateApiKeyCredentials(docId, apiKeyDoc, credentials, clock, listener); } }, listener::onFailure), threadPool.generic(), threadPool.getThreadContext()); } else { - verifyKeyAgainstHash(apiKeyHash, credentials, ActionListener.wrap( + verifyKeyAgainstHash(apiKeyDoc.hash, credentials, ActionListener.wrap( verified -> { listenableCacheEntry.onResponse(new CachedApiKeyHashResult(verified, credentials.getKey())); if (verified) { // move on - validateApiKeyExpiration(source, credentials, clock, listener); + validateApiKeyExpiration(apiKeyDoc, credentials, clock, listener); } else { listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); } @@ -515,18 +565,17 @@ void validateApiKeyCredentials(String docId, Map source, ApiKeyC )); } } else { - verifyKeyAgainstHash(apiKeyHash, credentials, ActionListener.wrap( + verifyKeyAgainstHash(apiKeyDoc.hash, credentials, ActionListener.wrap( verified -> { if (verified) { // move on - validateApiKeyExpiration(source, credentials, clock, listener); + validateApiKeyExpiration(apiKeyDoc, credentials, clock, listener); } else { listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null)); } }, listener::onFailure )); - } } } @@ -537,23 +586,19 @@ CachedApiKeyHashResult getFromCache(String id) { } // package-private for testing - void validateApiKeyExpiration(Map source, ApiKeyCredentials credentials, Clock clock, + void validateApiKeyExpiration(ApiKeyDoc apiKeyDoc, ApiKeyCredentials credentials, Clock clock, ActionListener listener) { - final Long expirationEpochMilli = (Long) source.get("expiration_time"); - if (expirationEpochMilli == null || Instant.ofEpochMilli(expirationEpochMilli).isAfter(clock.instant())) { - final Map creator = Objects.requireNonNull((Map) source.get("creator")); - final String principal = Objects.requireNonNull((String) creator.get("principal")); - Map metadata = (Map) creator.get("metadata"); - final Map roleDescriptors = (Map) source.get("role_descriptors"); - final Map limitedByRoleDescriptors = (Map) source.get("limited_by_role_descriptors"); + if (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())) { + final String principal = Objects.requireNonNull((String) apiKeyDoc.creator.get("principal")); + Map metadata = (Map) apiKeyDoc.creator.get("metadata"); final User apiKeyUser = new User(principal, Strings.EMPTY_ARRAY, null, null, metadata, true); final Map authResultMetadata = new HashMap<>(); - authResultMetadata.put(API_KEY_CREATOR_REALM_NAME, creator.get("realm")); - authResultMetadata.put(API_KEY_CREATOR_REALM_TYPE, creator.get("realm_type")); - authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors); - authResultMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleDescriptors); + authResultMetadata.put(API_KEY_CREATOR_REALM_NAME, apiKeyDoc.creator.get("realm")); + authResultMetadata.put(API_KEY_CREATOR_REALM_TYPE, apiKeyDoc.creator.get("realm_type")); + authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, apiKeyDoc.roleDescriptorsBytes); + authResultMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, apiKeyDoc.limitedByRoleDescriptorsBytes); authResultMetadata.put(API_KEY_ID_KEY, credentials.getId()); - authResultMetadata.put(API_KEY_NAME_KEY, source.get("name")); + authResultMetadata.put(API_KEY_NAME_KEY, apiKeyDoc.name); listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata)); } else { listener.onResponse(AuthenticationResult.unsuccessful("api key is expired", null)); @@ -974,4 +1019,65 @@ private boolean verify(SecureString password) { return hash != null && cacheHasher.verify(password, hash); } } + + public static final class ApiKeyDoc { + + static final InstantiatingObjectParser PARSER; + static { + InstantiatingObjectParser.Builder builder = + InstantiatingObjectParser.builder("api_key_doc", true, ApiKeyDoc.class); + builder.declareString(constructorArg(), new ParseField("doc_type")); + builder.declareLong(constructorArg(), new ParseField("creation_time")); + builder.declareLongOrNull(constructorArg(), -1, new ParseField("expiration_time")); + builder.declareBoolean(constructorArg(), new ParseField("api_key_invalidated")); + builder.declareString(constructorArg(), new ParseField("api_key_hash")); + builder.declareStringOrNull(optionalConstructorArg(), new ParseField("name")); + builder.declareInt(constructorArg(), new ParseField("version")); + ObjectParserHelper parserHelper = new ObjectParserHelper<>(); + parserHelper.declareRawObject(builder, constructorArg(), new ParseField("role_descriptors")); + parserHelper.declareRawObject(builder, constructorArg(), new ParseField("limited_by_role_descriptors")); + builder.declareObject(constructorArg(), (p, c) -> p.map(), new ParseField("creator")); + PARSER = builder.build(); + } + + final String docType; + final long creationTime; + final long expirationTime; + final Boolean invalidated; + final String hash; + @Nullable + final String name; + final int version; + final BytesReference roleDescriptorsBytes; + final BytesReference limitedByRoleDescriptorsBytes; + final Map creator; + + public ApiKeyDoc( + String docType, + long creationTime, + long expirationTime, + Boolean invalidated, + String hash, + @Nullable String name, + int version, + BytesReference roleDescriptorsBytes, + BytesReference limitedByRoleDescriptorsBytes, + Map creator) { + + this.docType = docType; + this.creationTime = creationTime; + this.expirationTime = expirationTime; + this.invalidated = invalidated; + this.hash = hash; + this.name = name; + this.version = version; + this.roleDescriptorsBytes = roleDescriptorsBytes; + this.limitedByRoleDescriptorsBytes = limitedByRoleDescriptorsBytes; + this.creator = creator; + } + + static ApiKeyDoc fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java index b1b40d9aada6b..8d08a0dba5862 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java @@ -675,13 +675,13 @@ void finishAuthentication(User finalUser) { * successful */ void writeAuthToContext(Authentication authentication) { - request.authenticationSuccess(authentication.getAuthenticatedBy().getName(), authentication.getUser()); Runnable action = () -> { logger.trace("Established authentication [{}] for request [{}]", authentication, request); listener.onResponse(authentication); }; try { authenticationSerializer.writeToContext(authentication, threadContext); + request.authenticationSuccess(authentication); } catch (Exception e) { action = () -> { logger.debug( @@ -724,7 +724,7 @@ abstract static class AuditableRequest { abstract ElasticsearchSecurityException runAsDenied(Authentication authentication, AuthenticationToken token); - abstract void authenticationSuccess(String realm, User user); + abstract void authenticationSuccess(Authentication authentication); } @@ -744,8 +744,8 @@ static class AuditableTransportRequest extends AuditableRequest { } @Override - void authenticationSuccess(String realm, User user) { - auditTrail.authenticationSuccess(requestId, realm, user, action, transportRequest); + void authenticationSuccess(Authentication authentication) { + auditTrail.authenticationSuccess(requestId, authentication, action, transportRequest); } @Override @@ -808,8 +808,8 @@ static class AuditableRestRequest extends AuditableRequest { } @Override - void authenticationSuccess(String realm, User user) { - auditTrail.authenticationSuccess(requestId, realm, user, request); + void authenticationSuccess(Authentication authentication) { + auditTrail.authenticationSuccess(requestId, authentication, request); } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 569ecaa7f0498..68ed142af0f97 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.datastream.CreateDataStreamAction; import org.elasticsearch.action.bulk.BulkItemRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.bulk.TransportShardBulkAction; @@ -176,7 +177,7 @@ public void authorize(final Authentication authentication, final String action, if (isInternalUser(authentication.getUser()) != false) { auditId = AuditUtil.getOrGenerateRequestId(threadContext); } else { - auditTrailService.get().tamperedRequest(null, authentication.getUser(), action, originalRequest); + auditTrailService.get().tamperedRequest(null, authentication, action, originalRequest); final String message = "Attempt to authorize action [" + action + "] for [" + authentication.getUser().principal() + "] without an existing request-id"; assert false : message; @@ -294,11 +295,11 @@ private void handleIndexActionAuthorizationResult(final IndexAuthorizationResult } //if we are creating an index we need to authorize potential aliases created at the same time if (IndexPrivilege.CREATE_INDEX_MATCHER.test(action)) { - assert request instanceof CreateIndexRequest; - Set aliases = ((CreateIndexRequest) request).aliases(); - if (aliases.isEmpty()) { + assert (request instanceof CreateIndexRequest) || (request instanceof CreateDataStreamAction.Request); + if (request instanceof CreateDataStreamAction.Request || ((CreateIndexRequest) request).aliases().isEmpty()) { runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener); } else { + Set aliases = ((CreateIndexRequest) request).aliases(); final RequestInfo aliasesRequestInfo = new RequestInfo(authentication, request, IndicesAliasesAction.NAME); authzEngine.authorizeIndexAction(aliasesRequestInfo, authzInfo, ril -> { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java index 4a7a8fc7a8f7d..02c8dac918161 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -71,6 +72,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.util.set.Sets.newHashSet; +import static org.elasticsearch.xpack.core.security.authc.Authentication.VERSION_API_KEY_ROLES_AS_BYTES; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isIndexDeleted; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.isMoveFromRedToNonRed; @@ -221,20 +223,40 @@ public void getRoles(User user, Authentication authentication, ActionListener { - final List descriptors = apiKeyRoleDescriptors.getRoleDescriptors(); - if (descriptors == null) { - roleActionListener.onFailure(new IllegalStateException("missing role descriptors")); - } else if (apiKeyRoleDescriptors.getLimitedByRoleDescriptors() == null) { - buildAndCacheRoleFromDescriptors(descriptors, apiKeyRoleDescriptors.getApiKeyId() + "_role_desc", roleActionListener); - } else { - buildAndCacheRoleFromDescriptors(descriptors, apiKeyRoleDescriptors.getApiKeyId() + "_role_desc", - ActionListener.wrap(role -> buildAndCacheRoleFromDescriptors(apiKeyRoleDescriptors.getLimitedByRoleDescriptors(), - apiKeyRoleDescriptors.getApiKeyId() + "_limited_role_desc", ActionListener.wrap( - limitedBy -> roleActionListener.onResponse(LimitedRole.createLimitedRole(role, limitedBy)), - roleActionListener::onFailure)), roleActionListener::onFailure)); - } - }, roleActionListener::onFailure)); + if (authentication.getVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)) { + buildAndCacheRoleForApiKey(authentication, false, ActionListener.wrap( + role -> { + if (role == Role.EMPTY) { + buildAndCacheRoleForApiKey(authentication, true, roleActionListener); + } else { + buildAndCacheRoleForApiKey(authentication, true, ActionListener.wrap( + limitedByRole -> roleActionListener.onResponse( + limitedByRole == Role.EMPTY ? role : LimitedRole.createLimitedRole(role, limitedByRole)), + roleActionListener::onFailure + )); + } + }, + roleActionListener::onFailure + )); + } else { + apiKeyService.getRoleForApiKey(authentication, ActionListener.wrap(apiKeyRoleDescriptors -> { + final List descriptors = apiKeyRoleDescriptors.getRoleDescriptors(); + if (descriptors == null) { + roleActionListener.onFailure(new IllegalStateException("missing role descriptors")); + } else if (apiKeyRoleDescriptors.getLimitedByRoleDescriptors() == null) { + buildAndCacheRoleFromDescriptors(descriptors, + apiKeyRoleDescriptors.getApiKeyId() + "_role_desc", roleActionListener); + } else { + buildAndCacheRoleFromDescriptors(descriptors, apiKeyRoleDescriptors.getApiKeyId() + "_role_desc", + ActionListener.wrap( + role -> buildAndCacheRoleFromDescriptors(apiKeyRoleDescriptors.getLimitedByRoleDescriptors(), + apiKeyRoleDescriptors.getApiKeyId() + "_limited_role_desc", ActionListener.wrap( + limitedBy -> roleActionListener.onResponse(LimitedRole.createLimitedRole(role, limitedBy)), + roleActionListener::onFailure)), roleActionListener::onFailure)); + } + }, roleActionListener::onFailure)); + } + } else { Set roleNames = new HashSet<>(Arrays.asList(user.roles())); if (isAnonymousEnabled && anonymousUser.equals(user) == false) { @@ -295,6 +317,22 @@ private void buildThenMaybeCacheRole(RoleKey roleKey, Collection }, listener::onFailure)); } + private void buildAndCacheRoleForApiKey(Authentication authentication, boolean limitedBy, ActionListener roleActionListener) { + final Tuple apiKeyIdAndBytes = apiKeyService.getApiKeyIdAndRoleBytes(authentication, limitedBy); + final String roleDescriptorsHash = MessageDigests.toHexString( + MessageDigests.sha256().digest(BytesReference.toBytes(apiKeyIdAndBytes.v2()))); + final RoleKey roleKey = new RoleKey(Set.of("apikey:" + roleDescriptorsHash), limitedBy ? "apikey_limited_role" : "apikey_role"); + final Role existing = roleCache.get(roleKey); + if (existing == null) { + final long invalidationCounter = numInvalidation.get(); + final List roleDescriptors = apiKeyService.parseRoleDescriptors(apiKeyIdAndBytes.v1(), apiKeyIdAndBytes.v2()); + buildThenMaybeCacheRole(roleKey, roleDescriptors, Collections.emptySet(), + true, invalidationCounter, roleActionListener); + } else { + roleActionListener.onResponse(existing); + } + } + public void getRoleDescriptors(Set roleNames, ActionListener> listener) { roleDescriptors(roleNames, ActionListener.wrap(rolesRetrievalResult -> { if (rolesRetrievalResult.isSuccess()) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java index 86f1c800ddbd1..d839a02ae7e80 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.security; import org.elasticsearch.Version; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; @@ -23,8 +24,12 @@ import java.io.EOFException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY; import static org.hamcrest.Matchers.instanceOf; public class SecurityContextTests extends ESTestCase { @@ -130,4 +135,40 @@ public void testExecuteAfterRewritingAuthentication() throws IOException { originalContext.restore(); assertEquals(original, securityContext.getAuthentication()); } + + public void testExecuteAfterRewritingAuthenticationShouldRewriteApiKeyMetadataForBwc() throws IOException { + User user = new User("test", null, new User("authUser")); + RealmRef authBy = new RealmRef("_es_api_key", "_es_api_key", "node1"); + final Map metadata = Map.of( + API_KEY_ROLE_DESCRIPTORS_KEY, new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"), + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, new BytesArray("{\"limitedBy role\": {\"cluster\": [\"all\"]}}") + ); + final Authentication original = new Authentication(user, authBy, authBy, Version.V_8_0_0, + AuthenticationType.API_KEY, metadata); + original.writeToContext(threadContext); + + securityContext.executeAfterRewritingAuthentication(originalCtx -> { + Authentication authentication = securityContext.getAuthentication(); + assertEquals(Map.of("a role", Map.of("cluster", List.of("all"))), + authentication.getMetadata().get(API_KEY_ROLE_DESCRIPTORS_KEY)); + assertEquals(Map.of("limitedBy role", Map.of("cluster", List.of("all"))), + authentication.getMetadata().get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)); + }, Version.V_7_8_0); + } + + public void testExecuteAfterRewritingAuthenticationShouldNotRewriteApiKeyMetadataForOldAuthenticationObject() throws IOException { + User user = new User("test", null, new User("authUser")); + RealmRef authBy = new RealmRef("_es_api_key", "_es_api_key", "node1"); + final Map metadata = Map.of( + API_KEY_ROLE_DESCRIPTORS_KEY, Map.of("a role", Map.of("cluster", List.of("all"))), + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, Map.of("limitedBy role", Map.of("cluster", List.of("all"))) + ); + final Authentication original = new Authentication(user, authBy, authBy, Version.V_7_8_0, AuthenticationType.API_KEY, metadata); + original.writeToContext(threadContext); + + securityContext.executeAfterRewritingAuthentication(originalCtx -> { + Authentication authentication = securityContext.getAuthentication(); + assertSame(metadata, authentication.getMetadata()); + }, randomFrom(Version.V_8_0_0, Version.V_7_8_0)); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java index 58f43a11d3a40..005dfb71e7129 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/AuditTrailServiceTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; -import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; +import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.Before; @@ -212,14 +212,14 @@ public void testConnectionDenied() throws Exception { } public void testAuthenticationSuccessRest() throws Exception { - User user = new User("_username", "r1"); - String realm = "_realm"; + Authentication authentication = new Authentication(new User("_username", "r1"), new RealmRef("_realm", null, null), + new RealmRef(null, null, null)); final String requestId = randomAlphaOfLengthBetween(6, 12); - service.get().authenticationSuccess(requestId, realm, user, restRequest); + service.get().authenticationSuccess(requestId, authentication, restRequest); verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { - verify(auditTrail).authenticationSuccess(requestId, realm, user, restRequest); + verify(auditTrail).authenticationSuccess(requestId, authentication, restRequest); } } else { verifyZeroInteractions(auditTrails.toArray((Object[]) new AuditTrail[auditTrails.size()])); @@ -227,14 +227,14 @@ public void testAuthenticationSuccessRest() throws Exception { } public void testAuthenticationSuccessTransport() throws Exception { - User user = new User("_username", "r1"); - String realm = "_realm"; + Authentication authentication = new Authentication(new User("_username", "r1"), new RealmRef("_realm", null, null), + new RealmRef(null, null, null)); final String requestId = randomAlphaOfLengthBetween(6, 12); - service.get().authenticationSuccess(requestId, realm, user, "_action", request); + service.get().authenticationSuccess(requestId, authentication, "_action", request); verify(licenseState).checkFeature(Feature.SECURITY_AUDITING); if (isAuditingAllowed) { for (AuditTrail auditTrail : auditTrails) { - verify(auditTrail).authenticationSuccess(requestId, realm, user, "_action", request); + verify(auditTrail).authenticationSuccess(requestId, authentication, "_action", request); } } else { verifyZeroInteractions(auditTrails.toArray((Object[]) new AuditTrail[auditTrails.size()])); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java index 295c097e29a65..a11079f720f65 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -17,11 +18,13 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.FakeRestRequest.Builder; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.audit.logfile.CapturingLogger; import org.elasticsearch.xpack.core.security.authc.Authentication; @@ -33,7 +36,9 @@ import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.MockRequest; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrailTests.RestContent; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.rest.RemoteHostHeader; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.Before; import org.mockito.stubbing.Answer; @@ -41,6 +46,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.time.Clock; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -49,6 +55,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; +import static org.elasticsearch.xpack.security.authc.ApiKeyServiceTests.Utils.createApiKeyAuthentication; import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -61,6 +68,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { private Settings settings; private DiscoveryNode localNode; private ClusterService clusterService; + private ApiKeyService apiKeyService; @Before public void init() throws Exception { @@ -82,6 +90,8 @@ public void init() throws Exception { arg0.updateLocalNodeInfo(localNode); return null; }).when(clusterService).addListener(Mockito.isA(LoggingAuditTrail.class)); + apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), mock(Client.class), new XPackLicenseState(settings), + mock(SecurityIndexManager.class), clusterService, mock(ThreadPool.class)); } public void testPolicyDoesNotMatchNullValuesInEvent() throws Exception { @@ -463,7 +473,7 @@ public void testUsersFilter() throws Exception { Collections.emptyList()); } } - final Authentication filteredAuthentication; + Authentication filteredAuthentication; if (randomBoolean()) { filteredAuthentication = createAuthentication( new User(randomFrom(allFilteredUsers), new String[] { "r1" }, new User("authUsername", new String[] { "r2" })), @@ -472,7 +482,10 @@ public void testUsersFilter() throws Exception { filteredAuthentication = createAuthentication(new User(randomFrom(allFilteredUsers), new String[] { "r1" }), "effectiveRealmName"); } - final Authentication unfilteredAuthentication; + if (randomBoolean()) { + filteredAuthentication = createApiKeyAuthentication(apiKeyService, filteredAuthentication); + } + Authentication unfilteredAuthentication; if (randomBoolean()) { unfilteredAuthentication = createAuthentication(new User(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 4), new String[] { "r1" }, new User("authUsername", new String[] { "r2" })), "effectiveRealmName"); @@ -480,6 +493,9 @@ public void testUsersFilter() throws Exception { unfilteredAuthentication = createAuthentication( new User(UNFILTER_MARKER + randomAlphaOfLengthBetween(1, 4), new String[] { "r1" }), "effectiveRealmName"); } + if (randomBoolean()) { + unfilteredAuthentication = createApiKeyAuthentication(apiKeyService, unfilteredAuthentication); + } final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext, new String[] { "idx1", "idx2" }); final MockToken filteredToken = new MockToken(randomFrom(allFilteredUsers)); @@ -644,12 +660,12 @@ public void testUsersFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.tamperedRequest(randomAlphaOfLength(8), unfilteredAuthentication.getUser(), "_action", request); + auditTrail.tamperedRequest(randomAlphaOfLength(8), unfilteredAuthentication, "_action", request); assertThat("Tampered message: unfiltered user is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.tamperedRequest(randomAlphaOfLength(8), filteredAuthentication.getUser(), "_action", request); + auditTrail.tamperedRequest(randomAlphaOfLength(8), filteredAuthentication, "_action", request); assertThat("Tampered message: filtered user is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); @@ -711,22 +727,22 @@ public void testUsersFilter() throws Exception { threadContext.stashContext(); // authentication Success - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", unfilteredAuthentication.getUser(), getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), unfilteredAuthentication, getRestRequest()); assertThat("AuthenticationSuccess rest request: unfiltered user is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", filteredAuthentication.getUser(), getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), filteredAuthentication, getRestRequest()); assertThat("AuthenticationSuccess rest request: filtered user is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", unfilteredAuthentication.getUser(), "_action", request); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), unfilteredAuthentication, "_action", request); assertThat("AuthenticationSuccess message: unfiltered user is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", filteredAuthentication.getUser(), "_action", request); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), filteredAuthentication, "_action", request); assertThat("AuthenticationSuccess message: filtered user is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); @@ -849,14 +865,18 @@ public void testRealmsFilter() throws Exception { threadContext.stashContext(); // accessGranted - auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessGranted(randomAlphaOfLength(8), + randomBoolean() ? createAuthentication(user, filteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), + "_action", request, authzInfo(new String[]{"role1"})); assertThat("AccessGranted message: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessGranted(randomAlphaOfLength(8), + randomBoolean() ? createAuthentication(user, unfilteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), + "_action", request, authzInfo(new String[]{"role1"})); assertThat("AccessGranted message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); @@ -873,27 +893,31 @@ public void testRealmsFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "internal:_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessGranted(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), + "internal:_action", request, authzInfo(new String[]{"role1"})); assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.accessGranted(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "internal:_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessGranted(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), + "internal:_action", request, authzInfo(new String[] { "role1" })); assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); // accessDenied - auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "_action", request, + authzInfo(new String[]{"role1"})); assertThat("AccessDenied message: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "_action", request, + authzInfo(new String[]{"role1"})); assertThat("AccessDenied message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); @@ -910,14 +934,16 @@ public void testRealmsFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "internal:_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "internal:_action", + request, authzInfo(new String[]{"role1"})); assertThat("AccessGranted internal message: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.accessDenied(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "internal:_action", request, - authzInfo(new String[] { "role1" })); + auditTrail.accessDenied(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "internal:_action", + request, authzInfo(new String[]{"role1"})); assertThat("AccessGranted internal message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); @@ -941,12 +967,15 @@ public void testRealmsFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.tamperedRequest(randomAlphaOfLength(8), user, "_action", request); - if (filterMissingRealm) { - assertThat("Tampered message: is not filtered out by the missing realm filter", logOutput.size(), is(0)); - } else { - assertThat("Tampered message: is filtered out", logOutput.size(), is(1)); - } + auditTrail.tamperedRequest(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, filteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, filteredRealm)), "_action", request); + assertThat("Tampered message: filtered realm is not filtered out", logOutput.size(), is(0)); + logOutput.clear(); + threadContext.stashContext(); + + auditTrail.tamperedRequest(randomAlphaOfLength(8), randomBoolean() ? createAuthentication(user, unfilteredRealm) : + createApiKeyAuthentication(apiKeyService, createAuthentication(user, unfilteredRealm)), "_action", request); + assertThat("Tampered message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); @@ -1009,22 +1038,22 @@ public void testRealmsFilter() throws Exception { threadContext.stashContext(); // authentication Success - auditTrail.authenticationSuccess(randomAlphaOfLength(8), unfilteredRealm, user, getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), getRestRequest()); assertThat("AuthenticationSuccess rest request: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), filteredRealm, user, getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), getRestRequest()); assertThat("AuthenticationSuccess rest request: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), unfilteredRealm, user, "_action", request); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), createAuthentication(user, unfilteredRealm), "_action", request); assertThat("AuthenticationSuccess message: unfiltered realm is filtered out", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), filteredRealm, user, "_action", request); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), createAuthentication(user, filteredRealm), "_action", request); assertThat("AuthenticationSuccess message: filtered realm is not filtered out", logOutput.size(), is(0)); logOutput.clear(); threadContext.stashContext(); @@ -1079,13 +1108,16 @@ public void testRolesFilter() throws Exception { settingsBuilder.putList("xpack.security.audit.logfile.events.ignore_filters.otherPolicy.roles", otherRoles); } final String[] unfilteredRoles = _unfilteredRoles.toArray(new String[0]); - final Authentication authentication; + Authentication authentication; if (randomBoolean()) { authentication = createAuthentication(new User("user1", new String[] { "r1" }, new User("authUsername", new String[] { "r2" })), "effectiveRealmName"); } else { authentication = createAuthentication(new User("user1", new String[] { "r1" }), "effectiveRealmName"); } + if (randomBoolean()) { + authentication = createApiKeyAuthentication(apiKeyService, authentication); + } final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext, new String[] { "idx1", "idx2" }); final MockToken authToken = new MockToken("token1"); @@ -1288,7 +1320,7 @@ public void testRolesFilter() throws Exception { threadContext.stashContext(); // authentication Success - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, getRestRequest()); if (filterMissingRoles) { assertThat("AuthenticationSuccess rest request: is not filtered out by the missing roles filter", logOutput.size(), is(0)); } else { @@ -1297,7 +1329,7 @@ public void testRolesFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), "_action", request); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, "_action", request); if (filterMissingRoles) { assertThat("AuthenticationSuccess message: is not filtered out by the missing roles filter", logOutput.size(), is(0)); } else { @@ -1356,13 +1388,16 @@ public void testIndicesFilter() throws Exception { settingsBuilder.putList("xpack.security.audit.logfile.events.ignore_filters.otherPolicy.indices", otherIndices); } final String[] unfilteredIndices = _unfilteredIndices.toArray(new String[0]); - final Authentication authentication; + Authentication authentication; if (randomBoolean()) { authentication = createAuthentication(new User("user1", new String[] { "r1" }, new User("authUsername", new String[] { "r2" })), "effectiveRealmName"); } else { authentication = createAuthentication(new User("user1", new String[] { "r1" }), "effectiveRealmName"); } + if (randomBoolean()) { + authentication = createApiKeyAuthentication(apiKeyService, authentication); + } final MockToken authToken = new MockToken("token1"); final TransportRequest noIndexRequest = new MockRequest(threadContext); @@ -1657,7 +1692,7 @@ public void testIndicesFilter() throws Exception { threadContext.stashContext(); // authentication Success - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), getRestRequest()); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, getRestRequest()); if (filterMissingIndices) { assertThat("AuthenticationSuccess rest request: is not filtered out by the missing indices filter", logOutput.size(), is(0)); } else { @@ -1666,7 +1701,7 @@ public void testIndicesFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), "_action", noIndexRequest); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, "_action", noIndexRequest); if (filterMissingIndices) { assertThat("AuthenticationSuccess message no index: not filtered out by missing indices filter", logOutput.size(), is(0)); } else { @@ -1675,13 +1710,13 @@ public void testIndicesFilter() throws Exception { logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), "_action", - new MockIndicesRequest(threadContext, unfilteredIndices)); + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, + unfilteredIndices)); assertThat("AuthenticationSuccess message unfiltered indices: filtered out by indices filter", logOutput.size(), is(1)); logOutput.clear(); threadContext.stashContext(); - auditTrail.authenticationSuccess(randomAlphaOfLength(8), "_realm", authentication.getUser(), "_action", + auditTrail.authenticationSuccess(randomAlphaOfLength(8), authentication, "_action", new MockIndicesRequest(threadContext, filteredIndices)); assertThat("AuthenticationSuccess message filtered indices: not filtered out by indices filter", logOutput.size(), is(0)); logOutput.clear(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 3189894afd00e..6e252c4cd19bd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -8,9 +8,11 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.layout.PatternLayout; +import org.elasticsearch.Version; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.bulk.BulkItemRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -27,15 +29,18 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.FakeRestRequest.Builder; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.audit.logfile.CapturingLogger; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; @@ -47,7 +52,9 @@ import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditUtil; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.rest.RemoteHostHeader; +import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.elasticsearch.xpack.security.transport.filter.IPFilter; import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule; import org.junit.After; @@ -61,6 +68,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.time.Clock; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -72,6 +80,7 @@ import java.util.regex.Pattern; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; +import static org.elasticsearch.xpack.security.authc.ApiKeyServiceTests.Utils.createApiKeyAuthentication; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasToString; @@ -149,6 +158,7 @@ protected String expectedMessage() { private Map commonFields; private Logger logger; private LoggingAuditTrail auditTrail; + private ApiKeyService apiKeyService; @BeforeClass public static void lookupPatternLayout() throws Exception { @@ -188,6 +198,8 @@ public void init() throws Exception { localNode = mock(DiscoveryNode.class); when(localNode.getId()).thenReturn(randomAlphaOfLength(16)); when(localNode.getAddress()).thenReturn(buildNewFakeTransportAddress()); + Client client = mock(Client.class); + SecurityIndexManager securityIndexManager = mock(SecurityIndexManager.class); clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(localNode); Mockito.doAnswer((Answer) invocation -> { @@ -208,6 +220,8 @@ public void init() throws Exception { } logger = CapturingLogger.newCapturingLogger(randomFrom(Level.OFF, Level.FATAL, Level.ERROR, Level.WARN, Level.INFO), patternLayout); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); + apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, new XPackLicenseState(settings), + securityIndexManager, clusterService, mock(ThreadPool.class)); } @After @@ -551,19 +565,39 @@ public void testAccessGranted() throws Exception { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); - final Authentication authentication = createAuthentication(); final String requestId = randomRequestId(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + MapBuilder checkedArrayFields = new MapBuilder<>(); + Authentication authentication = createAuthentication(); + auditTrail.accessGranted(requestId, authentication, "_action", request, authorizationInfo); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); + authentication(authentication, checkedFields); + restOrTransportOrigin(request, threadContext, checkedFields); + indicesRequest(request, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + checkedArrayFields = new MapBuilder<>(); auditTrail.accessGranted(requestId, authentication, "_action", request, authorizationInfo); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); - final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); - subject(authentication, checkedFields); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); @@ -617,7 +651,7 @@ public void testSystemAccessGranted() throws Exception { .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); - subject(authentication, checkedFields); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); @@ -637,7 +671,7 @@ public void testSystemAccessGranted() throws Exception { .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, BulkItemRequest.class.getName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); - subject(authentication, checkedFields); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); opaqueId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); @@ -668,6 +702,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception { final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.AUTHENTICATION_TYPE_FIELD_NAME, AuthenticationType.REALM.toString()) .put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, systemUser.principal()) .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, "_reserved") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action") @@ -685,19 +720,39 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); - final Authentication authentication = createAuthentication(); final String requestId = randomRequestId(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + MapBuilder checkedArrayFields = new MapBuilder<>(); + Authentication authentication = createAuthentication(); auditTrail.accessGranted(requestId, authentication, "internal:_action", request, authorizationInfo); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); - final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action") .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); - subject(authentication, checkedFields); + authentication(authentication, checkedFields); + restOrTransportOrigin(request, threadContext, checkedFields); + indicesRequest(request, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + checkedArrayFields = new MapBuilder<>(); + auditTrail.accessGranted(requestId, authentication, "internal:_action", request, authorizationInfo); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_granted") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "internal:_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); @@ -719,24 +774,43 @@ public void testAccessDenied() throws Exception { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); - final Authentication authentication = createAuthentication(); final String requestId = randomRequestId(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + MapBuilder checkedArrayFields = new MapBuilder<>(); + Authentication authentication = createAuthentication(); auditTrail.accessDenied(requestId, authentication, "_action/bar", request, authorizationInfo); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); - final MapBuilder checkedArrayFields = new MapBuilder<>(); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_denied") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action/bar") .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); - subject(authentication, checkedFields); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + checkedArrayFields = new MapBuilder<>(); + auditTrail.accessDenied(requestId, authentication, "_action/bar", request, authorizationInfo); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "access_denied") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action/bar") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + checkedArrayFields.put(PRINCIPAL_ROLES_FIELD_NAME, (String[]) authorizationInfo.asMap().get(PRINCIPAL_ROLES_FIELD_NAME)); + authentication(authentication, checkedFields); + restOrTransportOrigin(request, threadContext, checkedFields); + indicesRequest(request, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); // test disabled @@ -822,29 +896,37 @@ public void testTamperedRequest() throws Exception { public void testTamperedRequestWithUser() throws Exception { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); - final boolean runAs = randomBoolean(); - final User user; - if (runAs) { - user = new User("running_as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); - } else { - user = new User("_username", new String[] { "r1" }); - } - final String requestId = randomRequestId(); - auditTrail.tamperedRequest(requestId, user, "_action", request); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); - final MapBuilder checkedArrayFields = new MapBuilder<>(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + MapBuilder checkedArrayFields = new MapBuilder<>(); + + Authentication authentication = createAuthentication(); + auditTrail.tamperedRequest(requestId, authentication, "_action", request); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "tampered_request") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); - if (runAs) { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running_as"); - checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); - } else { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); - } + authentication(authentication, checkedFields); + restOrTransportOrigin(request, threadContext, checkedFields); + indicesRequest(request, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + checkedArrayFields = new MapBuilder<>(); + auditTrail.tamperedRequest(requestId, authentication, "_action", request); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "tampered_request") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); @@ -858,7 +940,7 @@ public void testTamperedRequestWithUser() throws Exception { .put("xpack.security.audit.logfile.events.exclude", "tampered_request") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.tamperedRequest(requestId, user, "_action", request); + auditTrail.tamperedRequest(requestId, authentication, "_action", request); assertEmptyLog(logger); } @@ -1012,17 +1094,12 @@ public void testAuthenticationSuccessRest() throws Exception { final Tuple tuple = prepareRestContent("_uri", address, params); final String expectedMessage = tuple.v1().expectedMessage(); final RestRequest request = tuple.v2(); - final String realm = randomAlphaOfLengthBetween(1, 6); - final User user; - if (randomBoolean()) { - user = new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); - } else { - user = new User("_username", new String[] { "r1" }); - } final String requestId = randomRequestId(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + Authentication authentication = createAuthentication(); // event by default disabled - auditTrail.authenticationSuccess(requestId, realm, user, request); + auditTrail.authenticationSuccess(requestId, authentication, request); assertEmptyLog(logger); settings = Settings.builder() @@ -1030,11 +1107,10 @@ public void testAuthenticationSuccessRest() throws Exception { .put("xpack.security.audit.logfile.events.include", "authentication_success") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.authenticationSuccess(requestId, realm, user, request); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); + auditTrail.authenticationSuccess(requestId, authentication, request); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") - .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) + .put(LoggingAuditTrail.REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()) .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) .put(LoggingAuditTrail.REQUEST_METHOD_FIELD_NAME, request.method().toString()) @@ -1046,12 +1122,32 @@ public void testAuthenticationSuccessRest() throws Exception { if (params.isEmpty() == false) { checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "foo=bar&evac=true"); } - if (user.isRunAs()) { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running as"); - checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); - } else { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); + authentication(authentication, checkedFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap()); + + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + auditTrail.authenticationSuccess(requestId, authentication, request); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") + .put(LoggingAuditTrail.REALM_FIELD_NAME, "_es_api_key") + .put(LoggingAuditTrail.ORIGIN_TYPE_FIELD_NAME, LoggingAuditTrail.REST_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(address)) + .put(LoggingAuditTrail.REQUEST_METHOD_FIELD_NAME, request.method().toString()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId) + .put(LoggingAuditTrail.URL_PATH_FIELD_NAME, "_uri"); + if (includeRequestBody && Strings.hasLength(expectedMessage)) { + checkedFields.put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, expectedMessage); } + if (params.isEmpty() == false) { + checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "foo=bar&evac=true"); + } + authentication(authentication, checkedFields); opaqueId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.immutableMap()); @@ -1059,17 +1155,13 @@ public void testAuthenticationSuccessRest() throws Exception { public void testAuthenticationSuccessTransport() throws Exception { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); - final User user; - if (randomBoolean()) { - user = new User("running as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); - } else { - user = new User("_username", new String[] { "r1" }); - } - final String realm = randomAlphaOfLengthBetween(1, 6); final String requestId = randomRequestId(); + MapBuilder checkedFields = new MapBuilder<>(commonFields); + MapBuilder checkedArrayFields = new MapBuilder<>(); + Authentication authentication = createAuthentication(); // event by default disabled - auditTrail.authenticationSuccess(requestId, realm, user, "_action", request); + auditTrail.authenticationSuccess(requestId, authentication, "_action", request); assertEmptyLog(logger); settings = Settings.builder() @@ -1077,21 +1169,32 @@ public void testAuthenticationSuccessTransport() throws Exception { .put("xpack.security.audit.logfile.events.include", "authentication_success") .build(); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); - auditTrail.authenticationSuccess(requestId, realm, user, "_action", request); - final MapBuilder checkedFields = new MapBuilder<>(commonFields); - final MapBuilder checkedArrayFields = new MapBuilder<>(); + auditTrail.authenticationSuccess(requestId, authentication, "_action", request); checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") - .put(LoggingAuditTrail.REALM_FIELD_NAME, realm) .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); - if (user.isRunAs()) { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "running as"); - checkedFields.put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, "_username"); - } else { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, "_username"); - } + authentication(authentication, checkedFields); + restOrTransportOrigin(request, threadContext, checkedFields); + indicesRequest(request, checkedFields, checkedArrayFields); + opaqueId(threadContext, checkedFields); + forwardedFor(threadContext, checkedFields); + assertMsg(logger, checkedFields.immutableMap(), checkedArrayFields.immutableMap()); + + CapturingLogger.output(logger.getName(), Level.INFO).clear(); + + // audit for authn with API Key + authentication = createApiKeyAuthentication(apiKeyService, authentication); + checkedFields = new MapBuilder<>(commonFields); + checkedArrayFields = new MapBuilder<>(); + auditTrail.authenticationSuccess(requestId, authentication, "_action", request); + checkedFields.put(LoggingAuditTrail.EVENT_TYPE_FIELD_NAME, LoggingAuditTrail.TRANSPORT_ORIGIN_FIELD_VALUE) + .put(LoggingAuditTrail.EVENT_ACTION_FIELD_NAME, "authentication_success") + .put(LoggingAuditTrail.ACTION_FIELD_NAME, "_action") + .put(LoggingAuditTrail.REQUEST_NAME_FIELD_NAME, request.getClass().getSimpleName()) + .put(LoggingAuditTrail.REQUEST_ID_FIELD_NAME, requestId); + authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); @@ -1128,25 +1231,32 @@ public void testRequestsWithoutIndices() throws Exception { auditTrail.authenticationFailed("_req_id", realm, new MockToken(), "_action", request); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.accessGranted("_req_id", createAuthentication(), "_action", request, authorizationInfo); + auditTrail.accessGranted("_req_id", randomBoolean() ? createAuthentication() : createApiKeyAuthentication(apiKeyService, + createAuthentication()), "_action", request, authorizationInfo); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.accessDenied("_req_id", createAuthentication(), "_action", request, authorizationInfo); + auditTrail.accessDenied("_req_id", randomBoolean() ? createAuthentication() : createApiKeyAuthentication(apiKeyService, + createAuthentication()), "_action", request, authorizationInfo); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); auditTrail.tamperedRequest("_req_id", "_action", request); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.tamperedRequest("_req_id", user, "_action", request); + auditTrail.tamperedRequest("_req_id", randomBoolean() ? createAuthentication() : createApiKeyAuthentication(apiKeyService, + createAuthentication()), "_action", request); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.runAsGranted("_req_id", createAuthentication(), "_action", request, authorizationInfo); + auditTrail.runAsGranted("_req_id", randomBoolean() ? createAuthentication() : createApiKeyAuthentication(apiKeyService, + createAuthentication()), "_action", request, authorizationInfo); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.runAsDenied("_req_id", createAuthentication(), "_action", request, authorizationInfo); + auditTrail.runAsDenied("_req_id", randomBoolean() ? createAuthentication() : createApiKeyAuthentication(apiKeyService, + createAuthentication()), "_action", request, authorizationInfo); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); - auditTrail.authenticationSuccess("_req_id", realm, user, "_action", request); + auditTrail.authenticationSuccess("_req_id", randomBoolean() ? createAuthentication() : + createApiKeyAuthentication(apiKeyService, createAuthentication()), + "_action", request); assertThat(output.size(), is(logEntriesCount++)); assertThat(output.get(logEntriesCount - 2), not(containsString("indices="))); } @@ -1243,17 +1353,21 @@ protected static InetAddress forge(String hostname, String address) throws IOExc return InetAddress.getByAddress(hostname, bytes); } - private static Authentication createAuthentication() { + private Authentication createAuthentication() { final RealmRef lookedUpBy; + final RealmRef authBy; final User user; if (randomBoolean()) { - user = new User("running_as", new String[] { "r2" }, new User("_username", new String[] { "r1" })); - lookedUpBy = new RealmRef("lookRealm", "up", "by"); + user = new User(randomAlphaOfLength(4), new String[] { "r1" }, new User("authenticated_username", new String[] { "r2" })); + lookedUpBy = new RealmRef(randomAlphaOfLength(4), "lookup", "by"); + authBy = new RealmRef("authRealm", "auth", "foo"); } else { - user = new User("_username", new String[] { "r1" }); + user = new User(randomAlphaOfLength(4), new String[] { "r1" }); lookedUpBy = null; + authBy = new RealmRef(randomAlphaOfLength(4), "auth", "by"); } - return new Authentication(user, new RealmRef("authRealm", "test", "foo"), lookedUpBy); + return new Authentication(user, authBy, lookedUpBy, Version.CURRENT, randomFrom(AuthenticationType.REALM, + AuthenticationType.TOKEN, AuthenticationType.INTERNAL, AuthenticationType.ANONYMOUS), Collections.emptyMap()); } private ClusterSettings mockClusterSettings() { @@ -1341,14 +1455,25 @@ private static void restOrTransportOrigin(TransportRequest request, ThreadContex } } - private static void subject(Authentication authentication, MapBuilder checkedFields) { + private static void authentication(Authentication authentication, MapBuilder checkedFields) { checkedFields.put(LoggingAuditTrail.PRINCIPAL_FIELD_NAME, authentication.getUser().principal()); - if (authentication.getUser().isRunAs()) { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) - .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) - .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + checkedFields.put(LoggingAuditTrail.AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString()); + if (Authentication.AuthenticationType.API_KEY == authentication.getAuthenticationType()) { + assert false == authentication.getUser().isRunAs(); + checkedFields.put(LoggingAuditTrail.API_KEY_ID_FIELD_NAME, + (String) authentication.getMetadata().get(ApiKeyService.API_KEY_ID_KEY)) + .put(LoggingAuditTrail.API_KEY_NAME_FIELD_NAME, + (String) authentication.getMetadata().get(ApiKeyService.API_KEY_NAME_KEY)) + .put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, + (String) authentication.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME)); } else { - checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + if (authentication.getUser().isRunAs()) { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getLookedUpBy().getName()) + .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_FIELD_NAME, authentication.getUser().authenticatedUser().principal()) + .put(LoggingAuditTrail.PRINCIPAL_RUN_BY_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } else { + checkedFields.put(LoggingAuditTrail.PRINCIPAL_REALM_FIELD_NAME, authentication.getAuthenticatedBy().getName()); + } } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 97c8dcb43658f..d7c5679b52498 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -15,13 +15,17 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -31,20 +35,24 @@ import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; +import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authc.support.Hasher; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials; +import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyDoc; import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyRoleDescriptors; import org.elasticsearch.xpack.security.authc.ApiKeyService.CachedApiKeyHashResult; import org.elasticsearch.xpack.security.authz.store.NativePrivilegeStore; @@ -69,12 +77,16 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY; import static org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR; import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; import static org.hamcrest.Matchers.contains; @@ -182,7 +194,13 @@ public void testAuthenticateWithApiKey() throws Exception { final String id = randomAlphaOfLength(12); final String key = randomAlphaOfLength(16); - mockKeyDocument(service, id, key, new User("hulk", "superuser")); + final User user; + if (randomBoolean()) { + user = new User("hulk", new String[] { "superuser" }, new User("authenticated_user", new String[] { "other" })); + } else { + user = new User("hulk", new String[] { "superuser" }); + } + mockKeyDocument(service, id, key, user); final AuthenticationResult auth = tryAuthenticate(service, id, key); assertThat(auth.getStatus(), is(AuthenticationResult.Status.SUCCESS)); @@ -201,7 +219,14 @@ public void testAuthenticationIsSkippedIfLicenseDoesNotAllowIt() throws Exceptio final String id = randomAlphaOfLength(12); final String key = randomAlphaOfLength(16); - mockKeyDocument(service, id, key, new User(randomAlphaOfLength(6), randomAlphaOfLength(12))); + final User user; + if (randomBoolean()) { + user = new User(randomAlphaOfLength(6), new String[] { randomAlphaOfLength(12) }, new User("authenticated_user", + new String[] { "other" })); + } else { + user = new User(randomAlphaOfLength(6), new String[] { randomAlphaOfLength(12) }); + } + mockKeyDocument(service, id, key, user); when(licenseState.checkFeature(Feature.SECURITY_API_KEY_SERVICE)).thenReturn(false); final AuthenticationResult auth = tryAuthenticate(service, id, key); @@ -232,7 +257,13 @@ public void testAuthenticationFailureWithInvalidCredentials() throws Exception { final String realKey = randomAlphaOfLength(16); final String wrongKey = "#" + realKey.substring(1); - mockKeyDocument(service, id, realKey, new User("hulk", "superuser")); + final User user; + if (randomBoolean()) { + user = new User("hulk", new String[] { "superuser" }, new User("authenticated_user", new String[] { "other" })); + } else { + user = new User("hulk", new String[] { "superuser" }); + } + mockKeyDocument(service, id, realKey, user); final AuthenticationResult auth = tryAuthenticate(service, id, wrongKey); assertThat(auth.getStatus(), is(AuthenticationResult.Status.CONTINUE)); @@ -265,7 +296,13 @@ public void testMixingValidAndInvalidCredentials() throws Exception { final String id = randomAlphaOfLength(12); final String realKey = randomAlphaOfLength(16); - mockKeyDocument(service, id, realKey, new User("hulk", "superuser")); + final User user; + if (randomBoolean()) { + user = new User("hulk", new String[] { "superuser" }, new User("authenticated_user", new String[] { "other" })); + } else { + user = new User("hulk", new String[] { "superuser" }); + } + mockKeyDocument(service, id, realKey, user); for (int i = 0; i < 3; i++) { final String wrongKey = "=" + randomAlphaOfLength(14) + "@"; @@ -287,8 +324,17 @@ private void mockKeyDocument(ApiKeyService service, String id, String key, User private void mockKeyDocument(ApiKeyService service, String id, String key, User user, boolean invalidated, Duration expiry) throws IOException { - final Authentication authentication = new Authentication(user, new RealmRef("realm1", "native", - "node01"), null, Version.CURRENT); + final Authentication authentication; + if (user.isRunAs()) { + authentication = new Authentication(user, new RealmRef("authRealm", "test", "foo"), + new RealmRef("realm1", "native", "node01"), Version.CURRENT, + randomFrom(AuthenticationType.REALM, AuthenticationType.TOKEN, AuthenticationType.INTERNAL, + AuthenticationType.ANONYMOUS), Collections.emptyMap()); + } else { + authentication = new Authentication(user, new RealmRef("realm1", "native", "node01"), null, + Version.CURRENT, randomFrom(AuthenticationType.REALM, AuthenticationType.TOKEN, AuthenticationType.INTERNAL, + AuthenticationType.ANONYMOUS), Collections.emptyMap()); + } XContentBuilder docSource = service.newDocument(new SecureString(key.toCharArray()), "test", authentication, Collections.singleton(SUPERUSER_ROLE_DESCRIPTOR), Instant.now(), Instant.now().plus(expiry), null, Version.CURRENT); @@ -320,68 +366,49 @@ public void testValidateApiKey() throws Exception { Hasher hasher = randomFrom(Hasher.PBKDF2, Hasher.BCRYPT4, Hasher.BCRYPT); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - Map sourceMap = new HashMap<>(); - sourceMap.put("doc_type", "api_key"); - sourceMap.put("api_key_hash", new String(hash)); - sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", "all"))); - sourceMap.put("limited_by_role_descriptors", Collections.singletonMap("limited role", Collections.singletonMap("cluster", "all"))); - Map creatorMap = new HashMap<>(); - creatorMap.put("principal", "test_user"); - creatorMap.put("realm", "realm1"); - creatorMap.put("realm_type", "realm_type1"); - creatorMap.put("metadata", Collections.emptyMap()); - sourceMap.put("creator", creatorMap); - sourceMap.put("api_key_invalidated", false); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); ApiKeyService service = createApiKeyService(Settings.EMPTY); ApiKeyService.ApiKeyCredentials creds = new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray())); PlainActionFuture future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); AuthenticationResult result = future.get(); assertNotNull(result); assertTrue(result.isAuthenticated()); assertThat(result.getUser().principal(), is("test_user")); assertThat(result.getUser().roles(), is(emptyArray())); assertThat(result.getUser().metadata(), is(Collections.emptyMap())); - assertThat(result.getMetadata().get(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(sourceMap.get("role_descriptors"))); - assertThat(result.getMetadata().get(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), - equalTo(sourceMap.get("limited_by_role_descriptors"))); + assertThat(result.getMetadata().get(API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(apiKeyDoc.roleDescriptorsBytes)); + assertThat(result.getMetadata().get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), + equalTo(apiKeyDoc.limitedByRoleDescriptorsBytes)); assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME), is("realm1")); - sourceMap.put("expiration_time", Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli()); + apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli(), false); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertTrue(result.isAuthenticated()); assertThat(result.getUser().principal(), is("test_user")); assertThat(result.getUser().roles(), is(emptyArray())); assertThat(result.getUser().metadata(), is(Collections.emptyMap())); - assertThat(result.getMetadata().get(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(sourceMap.get("role_descriptors"))); - assertThat(result.getMetadata().get(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), - equalTo(sourceMap.get("limited_by_role_descriptors"))); + assertThat(result.getMetadata().get(API_KEY_ROLE_DESCRIPTORS_KEY), equalTo(apiKeyDoc.roleDescriptorsBytes)); + assertThat(result.getMetadata().get(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY), + equalTo(apiKeyDoc.limitedByRoleDescriptorsBytes)); assertThat(result.getMetadata().get(ApiKeyService.API_KEY_CREATOR_REALM_NAME), is("realm1")); - sourceMap.put("expiration_time", Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli()); - future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); - result = future.get(); - assertNotNull(result); - assertFalse(result.isAuthenticated()); - - sourceMap.remove("expiration_time"); - creds = new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(15).toCharArray())); + apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli(), false); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertFalse(result.isAuthenticated()); - sourceMap.put("api_key_invalidated", true); + apiKeyDoc = buildApiKeyDoc(hash, -1, true); creds = new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(randomAlphaOfLength(15).toCharArray())); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.get(); assertNotNull(result); assertFalse(result.isAuthenticated()); @@ -398,13 +425,14 @@ public void testGetRolesForApiKeyNotInContext() throws Exception { } Map authMetadata = new HashMap<>(); authMetadata.put(ApiKeyService.API_KEY_ID_KEY, randomAlphaOfLength(12)); - authMetadata.put(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY, + authMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, Collections.singletonMap(SUPERUSER_ROLE_DESCRIPTOR.getName(), superUserRdMap)); - authMetadata.put(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + authMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, Collections.singletonMap(SUPERUSER_ROLE_DESCRIPTOR.getName(), superUserRdMap)); final Authentication authentication = new Authentication(new User("joe"), new RealmRef("apikey", "apikey", "node"), null, - Version.CURRENT, AuthenticationType.API_KEY, authMetadata); + VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1), + AuthenticationType.API_KEY, authMetadata); ApiKeyService service = createApiKeyService(Settings.EMPTY); PlainActionFuture roleFuture = new PlainActionFuture<>(); @@ -427,7 +455,7 @@ public void testGetRolesForApiKey() throws Exception { roleARDMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), BytesReference.bytes(roleARoleDescriptor.toXContent(builder, ToXContent.EMPTY_PARAMS, true)).streamInput(), false); } - authMetadata.put(ApiKeyService.API_KEY_ROLE_DESCRIPTORS_KEY, + authMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, (emptyApiKeyRoleDescriptor) ? randomFrom(Arrays.asList(null, Collections.emptyMap())) : Collections.singletonMap("a role", roleARDMap)); @@ -443,10 +471,11 @@ public void testGetRolesForApiKey() throws Exception { .streamInput(), false); } - authMetadata.put(ApiKeyService.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, Collections.singletonMap("limited role", limitedRdMap)); + authMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, Collections.singletonMap("limited role", limitedRdMap)); final Authentication authentication = new Authentication(new User("joe"), new RealmRef("apikey", "apikey", "node"), null, - Version.CURRENT, AuthenticationType.API_KEY, authMetadata); + VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1), + AuthenticationType.API_KEY, authMetadata); final NativePrivilegeStore privilegesStore = mock(NativePrivilegeStore.class); doAnswer(i -> { @@ -475,6 +504,54 @@ public void testGetRolesForApiKey() throws Exception { } } + public void testGetApiKeyIdAndRoleBytes() { + Map authMetadata = new HashMap<>(); + final String apiKeyId = randomAlphaOfLength(12); + authMetadata.put(ApiKeyService.API_KEY_ID_KEY, apiKeyId); + final BytesReference roleBytes = new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"); + final BytesReference limitedByRoleBytes = new BytesArray("{\"limitedBy role\": {\"cluster\": [\"all\"]}}"); + authMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleBytes); + authMetadata.put(API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, limitedByRoleBytes); + + final Authentication authentication = new Authentication(new User("joe"), new RealmRef("apikey", "apikey", "node"), null, + Version.CURRENT, AuthenticationType.API_KEY, authMetadata); + ApiKeyService service = createApiKeyService(Settings.EMPTY); + + Tuple apiKeyIdAndRoleBytes = service.getApiKeyIdAndRoleBytes(authentication, false); + assertEquals(apiKeyId, apiKeyIdAndRoleBytes.v1()); + assertEquals(roleBytes, apiKeyIdAndRoleBytes.v2()); + apiKeyIdAndRoleBytes = service.getApiKeyIdAndRoleBytes(authentication, true); + assertEquals(apiKeyId, apiKeyIdAndRoleBytes.v1()); + assertEquals(limitedByRoleBytes, apiKeyIdAndRoleBytes.v2()); + } + + public void testParseRoleDescriptors() { + ApiKeyService service = createApiKeyService(Settings.EMPTY); + final String apiKeyId = randomAlphaOfLength(12); + List roleDescriptors = service.parseRoleDescriptors(apiKeyId, null); + assertTrue(roleDescriptors.isEmpty()); + + BytesReference roleBytes = new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"); + roleDescriptors = service.parseRoleDescriptors(apiKeyId, roleBytes); + assertEquals(1, roleDescriptors.size()); + assertEquals("a role", roleDescriptors.get(0).getName()); + assertArrayEquals(new String[] { "all" }, roleDescriptors.get(0).getClusterPrivileges()); + assertEquals(0, roleDescriptors.get(0).getIndicesPrivileges().length); + assertEquals(0, roleDescriptors.get(0).getApplicationPrivileges().length); + + roleBytes = new BytesArray( + "{\"reporting_user\":{\"cluster\":[],\"indices\":[],\"applications\":[],\"run_as\":[],\"metadata\":{\"_reserved\":true}," + + "\"transient_metadata\":{\"enabled\":true}},\"superuser\":{\"cluster\":[\"all\"],\"indices\":[{\"names\":[\"*\"]," + + "\"privileges\":[\"all\"],\"allow_restricted_indices\":true}],\"applications\":[{\"application\":\"*\"," + + "\"privileges\":[\"*\"],\"resources\":[\"*\"]}],\"run_as\":[\"*\"],\"metadata\":{\"_reserved\":true}," + + "\"transient_metadata\":{}}}\n"); + roleDescriptors = service.parseRoleDescriptors(apiKeyId, roleBytes); + assertEquals(2, roleDescriptors.size()); + assertEquals( + Set.of("reporting_user", "superuser"), + roleDescriptors.stream().map(RoleDescriptor::getName).collect(Collectors.toSet())); + } + public void testApiKeyServiceDisabled() throws Exception { final Settings settings = Settings.builder().put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), false).build(); final ApiKeyService service = createApiKeyService(settings); @@ -494,12 +571,12 @@ public void testApiKeyCache() { Hasher hasher = randomFrom(Hasher.PBKDF2, Hasher.BCRYPT4, Hasher.BCRYPT); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - Map sourceMap = buildApiKeySourceDoc(hash); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); ApiKeyService service = createApiKeyService(Settings.EMPTY); ApiKeyCredentials creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray())); PlainActionFuture future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); AuthenticationResult result = future.actionGet(); assertThat(result.isAuthenticated(), is(true)); CachedApiKeyHashResult cachedApiKeyHashResult = service.getFromCache(creds.getId()); @@ -508,17 +585,17 @@ public void testApiKeyCache() { creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar".toCharArray())); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.actionGet(); assertThat(result.isAuthenticated(), is(false)); final CachedApiKeyHashResult shouldBeSame = service.getFromCache(creds.getId()); assertNotNull(shouldBeSame); assertThat(shouldBeSame, sameInstance(cachedApiKeyHashResult)); - sourceMap.put("api_key_hash", new String(hasher.hash(new SecureString("foobar".toCharArray())))); + apiKeyDoc = buildApiKeyDoc(hasher.hash(new SecureString("foobar".toCharArray())), -1, false); creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString("foobar1".toCharArray())); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.actionGet(); assertThat(result.isAuthenticated(), is(false)); cachedApiKeyHashResult = service.getFromCache(creds.getId()); @@ -527,7 +604,7 @@ public void testApiKeyCache() { creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar2".toCharArray())); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.actionGet(); assertThat(result.isAuthenticated(), is(false)); assertThat(service.getFromCache(creds.getId()), not(sameInstance(cachedApiKeyHashResult))); @@ -535,7 +612,7 @@ public void testApiKeyCache() { creds = new ApiKeyCredentials(creds.getId(), new SecureString("foobar".toCharArray())); future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); result = future.actionGet(); assertThat(result.isAuthenticated(), is(true)); assertThat(service.getFromCache(creds.getId()), not(sameInstance(cachedApiKeyHashResult))); @@ -608,12 +685,12 @@ public void testApiKeyCacheDisabled() { .put(ApiKeyService.CACHE_TTL_SETTING.getKey(), "0s") .build(); - Map sourceMap = buildApiKeySourceDoc(hash); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); ApiKeyService service = createApiKeyService(settings); ApiKeyCredentials creds = new ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray())); PlainActionFuture future = new PlainActionFuture<>(); - service.validateApiKeyCredentials(creds.getId(), sourceMap, creds, Clock.systemUTC(), future); + service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); AuthenticationResult result = future.actionGet(); assertThat(result.isAuthenticated(), is(true)); CachedApiKeyHashResult cachedApiKeyHashResult = service.getFromCache(creds.getId()); @@ -721,20 +798,95 @@ public void testCachedApiKeyValidationWillNotBeBlockedByUnCachedApiKey() throws assertEquals(AuthenticationResult.Status.SUCCESS, authenticationResult3.getStatus()); } + public void testApiKeyDocDeserialization() throws IOException { + final String apiKeyDocumentSource = + "{\"doc_type\":\"api_key\",\"creation_time\":1591919944598,\"expiration_time\":1591919944599,\"api_key_invalidated\":false," + + "\"api_key_hash\":\"{PBKDF2}10000$abc\",\"role_descriptors\":{\"a\":{\"cluster\":[\"all\"]}}," + + "\"limited_by_role_descriptors\":{\"limited_by\":{\"cluster\":[\"all\"]," + + "\"metadata\":{\"_reserved\":true},\"type\":\"role\"}}," + + "\"name\":\"key-1\",\"version\":7000099," + + "\"creator\":{\"principal\":\"admin\",\"metadata\":{\"foo\":\"bar\"},\"realm\":\"file1\",\"realm_type\":\"file\"}}"; + final ApiKeyDoc apiKeyDoc = ApiKeyDoc.fromXContent(XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + new BytesArray(apiKeyDocumentSource), + XContentType.JSON)); + assertEquals("api_key", apiKeyDoc.docType); + assertEquals(1591919944598L, apiKeyDoc.creationTime); + assertEquals(1591919944599L, apiKeyDoc.expirationTime); + assertFalse(apiKeyDoc.invalidated); + assertEquals("{PBKDF2}10000$abc", apiKeyDoc.hash); + assertEquals("key-1", apiKeyDoc.name); + assertEquals(7000099, apiKeyDoc.version); + assertEquals(new BytesArray("{\"a\":{\"cluster\":[\"all\"]}}"), apiKeyDoc.roleDescriptorsBytes); + assertEquals(new BytesArray("{\"limited_by\":{\"cluster\":[\"all\"],\"metadata\":{\"_reserved\":true},\"type\":\"role\"}}"), + apiKeyDoc.limitedByRoleDescriptorsBytes); + + final Map creator = apiKeyDoc.creator; + assertEquals("admin", creator.get("principal")); + assertEquals("file1", creator.get("realm")); + assertEquals("file", creator.get("realm_type")); + assertEquals("bar", ((Map)creator.get("metadata")).get("foo")); + } + + public void testApiKeyDocDeserializationWithNullValues() throws IOException { + final String apiKeyDocumentSource = + "{\"doc_type\":\"api_key\",\"creation_time\":1591919944598,\"expiration_time\":null,\"api_key_invalidated\":false," + + "\"api_key_hash\":\"{PBKDF2}10000$abc\",\"role_descriptors\":{}," + + "\"limited_by_role_descriptors\":{\"limited_by\":{\"cluster\":[\"all\"]}}," + + "\"name\":null,\"version\":7000099," + + "\"creator\":{\"principal\":\"admin\",\"metadata\":{},\"realm\":\"file1\"}}"; + final ApiKeyDoc apiKeyDoc = ApiKeyDoc.fromXContent(XContentHelper.createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + new BytesArray(apiKeyDocumentSource), + XContentType.JSON)); + assertEquals(-1L, apiKeyDoc.expirationTime); + assertNull(apiKeyDoc.name); + assertEquals(new BytesArray("{}"), apiKeyDoc.roleDescriptorsBytes); + } + public static class Utils { + private static final AuthenticationContextSerializer authenticationContextSerializer = new AuthenticationContextSerializer(); public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyService, Authentication authentication, Set userRoles, - List keyRoles) throws Exception { + List keyRoles, + Version version) throws Exception { XContentBuilder keyDocSource = apiKeyService.newDocument(new SecureString("secret".toCharArray()), "test", authentication, userRoles, Instant.now(), Instant.now().plus(Duration.ofSeconds(3600)), keyRoles, Version.CURRENT); - Map keyDocMap = XContentHelper.convertToMap(BytesReference.bytes(keyDocSource), true, XContentType.JSON).v2(); + final ApiKeyDoc apiKeyDoc = ApiKeyDoc.fromXContent( + XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(keyDocSource), XContentType.JSON)); PlainActionFuture authenticationResultFuture = PlainActionFuture.newFuture(); - apiKeyService.validateApiKeyExpiration(keyDocMap, new ApiKeyService.ApiKeyCredentials("id", + apiKeyService.validateApiKeyExpiration(apiKeyDoc, new ApiKeyService.ApiKeyCredentials("id", new SecureString("pass".toCharArray())), Clock.systemUTC(), authenticationResultFuture); - return apiKeyService.createApiKeyAuthentication(authenticationResultFuture.get(), "node01"); + + final TestThreadPool threadPool = new TestThreadPool("utils"); + try { + final ThreadContext threadContext = threadPool.getThreadContext(); + final SecurityContext securityContext = new SecurityContext(Settings.EMPTY, threadContext); + authenticationContextSerializer.writeToContext( + apiKeyService.createApiKeyAuthentication(authenticationResultFuture.get(), "node01"), threadContext); + final CompletableFuture authFuture = new CompletableFuture<>(); + securityContext.executeAfterRewritingAuthentication((c) -> { + try { + authFuture.complete(authenticationContextSerializer.readFromContext(threadContext)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, version); + return authFuture.get(); + } finally { + terminate(threadPool); + } + } + + public static Authentication createApiKeyAuthentication(ApiKeyService apiKeyService, + Authentication authentication) throws Exception { + return createApiKeyAuthentication(apiKeyService, authentication, + Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[]{"manage"}, null, null)), + null, Version.CURRENT); } } @@ -750,7 +902,11 @@ private ApiKeyService createApiKeyService(Settings baseSettings) { private Map buildApiKeySourceDoc(char[] hash) { Map sourceMap = new HashMap<>(); sourceMap.put("doc_type", "api_key"); + sourceMap.put("creation_time", Clock.systemUTC().instant().toEpochMilli()); + sourceMap.put("expiration_time", -1); sourceMap.put("api_key_hash", new String(hash)); + sourceMap.put("name", randomAlphaOfLength(12)); + sourceMap.put("version", 0); sourceMap.put("role_descriptors", Collections.singletonMap("a role", Collections.singletonMap("cluster", "all"))); sourceMap.put("limited_by_role_descriptors", Collections.singletonMap("limited role", Collections.singletonMap("cluster", "all"))); Map creatorMap = new HashMap<>(); @@ -774,4 +930,18 @@ private void mockSourceDocument(String id, Map sourceMap) throws } } + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated) { + return new ApiKeyDoc( + "api_key", + Clock.systemUTC().instant().toEpochMilli(), + expirationTime, + invalidated, + new String(hash), + randomAlphaOfLength(12), + 0, + new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"), + new BytesArray("{\"limited role\": {\"cluster\": [\"all\"]}}"), + Map.of("principal", "test_user", "realm", "realm1", "realm_type", "realm_type1", "metadata", Map.of()) + ); + } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index db58c8ea9a8b0..d59edd277d7af 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -336,6 +336,7 @@ public void testAuthenticateSmartRealmOrdering() { assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); assertThat(result.getAuthenticatedBy().getType(), is(SECOND_REALM_TYPE)); assertThreadContextContainsAuthentication(result); + verify(auditTrail).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); assertTrue(completed.get()); @@ -352,13 +353,13 @@ public void testAuthenticateSmartRealmOrdering() { assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); assertThat(result.getAuthenticatedBy().getType(), is(SECOND_REALM_TYPE)); assertThreadContextContainsAuthentication(result); + verify(auditTrail, times(2)).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); verify(auditTrail).authenticationFailed(reqId, firstRealm.name(), token, "_action", transportRequest); - verify(auditTrail, times(2)).authenticationSuccess(reqId, secondRealm.name(), user, "_action", transportRequest); verify(firstRealm, times(2)).name(); // used above one time - verify(secondRealm, times(3)).name(); // used above one time + verify(secondRealm, times(2)).name(); verify(secondRealm, times(2)).type(); // used to create realm ref verify(firstRealm, times(2)).token(threadContext); verify(secondRealm, times(2)).token(threadContext); @@ -384,11 +385,11 @@ public void testAuthenticateSmartRealmOrdering() { assertThat(result.getAuthenticatedBy().getName(), is(FIRST_REALM_NAME)); assertThat(result.getAuthenticatedBy().getType(), is(FIRST_REALM_TYPE)); assertThreadContextContainsAuthentication(result); + verify(auditTrail).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); - verify(auditTrail, times(1)).authenticationFailed(reqId, SECOND_REALM_NAME, token, "_action", transportRequest); - verify(auditTrail, times(1)).authenticationSuccess(reqId, FIRST_REALM_NAME, user, "_action", transportRequest); + verify(auditTrail).authenticationFailed(reqId, SECOND_REALM_NAME, token, "_action", transportRequest); verify(secondRealm, times(3)).authenticate(eq(token), any(ActionListener.class)); // 2 from above + 1 more verify(firstRealm, times(2)).authenticate(eq(token), any(ActionListener.class)); // 1 from above + 1 more } @@ -449,8 +450,9 @@ public void testAuthenticateSmartRealmOrderingDisabled() { assertThat(result, notNullValue()); assertThat(result.getUser(), is(user)); assertThat(result.getLookedUpBy(), is(nullValue())); - assertThat(result.getAuthenticatedBy(), is(notNullValue())); // TODO implement equals + assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); // TODO implement equals assertThreadContextContainsAuthentication(result); + verify(auditTrail).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); assertTrue(completed.get()); @@ -460,14 +462,14 @@ public void testAuthenticateSmartRealmOrderingDisabled() { assertThat(result, notNullValue()); assertThat(result.getUser(), is(user)); assertThat(result.getLookedUpBy(), is(nullValue())); - assertThat(result.getAuthenticatedBy(), is(notNullValue())); // TODO implement equals + assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); // TODO implement equals assertThreadContextContainsAuthentication(result); + verify(auditTrail, times(2)).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); verify(auditTrail, times(2)).authenticationFailed(reqId, firstRealm.name(), token, "_action", transportRequest); - verify(auditTrail, times(2)).authenticationSuccess(reqId, secondRealm.name(), user, "_action", transportRequest); verify(firstRealm, times(3)).name(); // used above one time - verify(secondRealm, times(3)).name(); // used above one time + verify(secondRealm, times(2)).name(); verify(secondRealm, times(2)).type(); // used to create realm ref verify(firstRealm, times(2)).token(threadContext); verify(secondRealm, times(2)).token(threadContext); @@ -491,10 +493,12 @@ public void testAuthenticateFirstNotSupportingSecondSucceeds() throws Exception assertThat(result, notNullValue()); assertThat(result.getUser(), is(user)); assertThat(result.getAuthenticationType(), is(AuthenticationType.REALM)); + assertThat(result.getAuthenticatedBy().getName(), is(secondRealm.name())); // TODO implement equals + assertThat(result.getAuthenticationType(), is(AuthenticationType.REALM)); assertThreadContextContainsAuthentication(result); + verify(auditTrail).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); - verify(auditTrail).authenticationSuccess(reqId, secondRealm.name(), user, "_action", transportRequest); verifyNoMoreInteractions(auditTrail); verify(firstRealm, never()).authenticate(eq(token), any(ActionListener.class)); assertTrue(completed.get()); @@ -641,12 +645,13 @@ public void testAuthenticateTransportSuccess() throws Exception { authenticate.accept(ActionListener.wrap(result -> { assertThat(result, notNullValue()); assertThat(result.getUser(), sameInstance(user)); - assertThreadContextContainsAuthentication(result); assertThat(result.getAuthenticationType(), is(AuthenticationType.REALM)); + assertThat(result.getAuthenticatedBy().getName(), is(firstRealm.name())); // TODO implement equals + assertThreadContextContainsAuthentication(result); + verify(auditTrail).authenticationSuccess(reqId, result, "_action", transportRequest); setCompletedToTrue(completed); }, this::logAndFail)); - verify(auditTrail).authenticationSuccess(reqId, firstRealm.name(), user, "_action", transportRequest); verifyNoMoreInteractions(auditTrail); assertTrue(completed.get()); } @@ -662,11 +667,12 @@ public void testAuthenticateRestSuccess() throws Exception { assertThat(authentication, notNullValue()); assertThat(authentication.getUser(), sameInstance(user1)); assertThat(authentication.getAuthenticationType(), is(AuthenticationType.REALM)); + assertThat(authentication.getAuthenticatedBy().getName(), is(firstRealm.name())); // TODO implement equals assertThreadContextContainsAuthentication(authentication); + String reqId = expectAuditRequestId(); + verify(auditTrail).authenticationSuccess(reqId, authentication, restRequest); setCompletedToTrue(completed); }, this::logAndFail)); - String reqId = expectAuditRequestId(); - verify(auditTrail).authenticationSuccess(reqId, firstRealm.name(), user1, restRequest); verifyNoMoreInteractions(auditTrail); assertTrue(completed.get()); } @@ -840,7 +846,7 @@ public void testAnonymousUserRest() throws Exception { assertThat(result.getAuthenticationType(), is(AuthenticationType.ANONYMOUS)); assertThreadContextContainsAuthentication(result); String reqId = expectAuditRequestId(); - verify(auditTrail).authenticationSuccess(reqId, "__anonymous", new AnonymousUser(settings), request); + verify(auditTrail).authenticationSuccess(reqId, result, request); verifyNoMoreInteractions(auditTrail); } @@ -1290,12 +1296,13 @@ public void testAuthenticateWithToken() throws Exception { assertThat(result.getUser(), is(user)); assertThat(result.getLookedUpBy(), is(nullValue())); assertThat(result.getAuthenticatedBy(), is(notNullValue())); + assertThat(result.getAuthenticatedBy().getName(), is("realm")); // TODO implement equals assertThat(result.getAuthenticationType(), is(AuthenticationType.TOKEN)); setCompletedToTrue(completed); + verify(auditTrail).authenticationSuccess(anyString(), eq(result), eq("_action"), same(transportRequest)); }, this::logAndFail)); } assertTrue(completed.get()); - verify(auditTrail).authenticationSuccess(anyString(), eq("realm"), eq(user), eq("_action"), same(transportRequest)); verifyNoMoreInteractions(auditTrail); } @@ -1346,7 +1353,7 @@ public void testInvalidToken() throws Exception { latch.await(); if (success.get()) { final String realmName = firstRealm.name(); - verify(auditTrail).authenticationSuccess(anyString(), eq(realmName), eq(user), eq("_action"), same(transportRequest)); + verify(auditTrail).authenticationSuccess(anyString(), eq(expected), eq("_action"), same(transportRequest)); } verifyNoMoreInteractions(auditTrail); } @@ -1403,10 +1410,14 @@ public void testApiKeyAuth() { final Map source = new HashMap<>(); source.put("doc_type", "api_key"); source.put("creation_time", Instant.now().minus(5, ChronoUnit.MINUTES).toEpochMilli()); + source.put("expiration_time", null); source.put("api_key_invalidated", false); source.put("api_key_hash", new String(Hasher.BCRYPT4.hash(new SecureString(key.toCharArray())))); source.put("role_descriptors", Collections.singletonMap("api key role", Collections.singletonMap("cluster", "all"))); + source.put("limited_by_role_descriptors", + Collections.singletonMap("limited api key role", Collections.singletonMap("cluster", "all"))); source.put("name", "my api key for testApiKeyAuth"); + source.put("version", 0); Map creatorMap = new HashMap<>(); creatorMap.put("principal", "johndoe"); creatorMap.put("metadata", Collections.emptyMap()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index 6dc0ae4a26441..efd386c28807c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -32,6 +33,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.XPackLicenseState.Feature; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequest.Empty; @@ -90,6 +92,9 @@ import static org.elasticsearch.mock.orig.Mockito.times; import static org.elasticsearch.mock.orig.Mockito.verifyNoMoreInteractions; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY; +import static org.elasticsearch.xpack.security.authc.ApiKeyService.API_KEY_ID_KEY; import static org.elasticsearch.xpack.security.authc.ApiKeyServiceTests.Utils.createApiKeyAuthentication; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -100,12 +105,14 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anySetOf; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -1026,9 +1033,9 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception { }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class)); final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore()); ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS); - ApiKeyService apiKeyService = new ApiKeyService(SECURITY_ENABLED_SETTINGS, Clock.systemUTC(), mock(Client.class), + ApiKeyService apiKeyService = spy(new ApiKeyService(SECURITY_ENABLED_SETTINGS, Clock.systemUTC(), mock(Client.class), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), mock(SecurityIndexManager.class), mock(ClusterService.class), - mock(ThreadPool.class)); + mock(ThreadPool.class))); NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class); doAnswer(invocationOnMock -> { ActionListener> listener = @@ -1045,14 +1052,22 @@ public void testApiKeyAuthUsesApiKeyService() throws Exception { new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService, documentSubsetBitsetCache, rds -> effectiveRoleDescriptors.set(rds)); AuditUtil.getOrGenerateRequestId(threadContext); - + final Version version = randomFrom(Version.CURRENT, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1)); final Authentication authentication = createApiKeyAuthentication(apiKeyService, createAuthentication(), - Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[]{"manage"}, null, null)), null); + Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[]{"manage"}, null, null)), + null, + version); PlainActionFuture roleFuture = new PlainActionFuture<>(); compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture); Role role = roleFuture.actionGet(); assertThat(effectiveRoleDescriptors.get(), is(nullValue())); + + if (version == Version.CURRENT) { + verify(apiKeyService, times(2)).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + } else { + verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class)); + } assertThat(role.names().length, is(1)); assertThat(role.names()[0], containsString("user_role_")); } @@ -1071,9 +1086,9 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception { final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore()); ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS); - ApiKeyService apiKeyService = new ApiKeyService(SECURITY_ENABLED_SETTINGS, Clock.systemUTC(), mock(Client.class), + ApiKeyService apiKeyService = spy(new ApiKeyService(SECURITY_ENABLED_SETTINGS, Clock.systemUTC(), mock(Client.class), new XPackLicenseState(SECURITY_ENABLED_SETTINGS), mock(SecurityIndexManager.class), mock(ClusterService.class), - mock(ThreadPool.class)); + mock(ThreadPool.class))); NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class); doAnswer(invocationOnMock -> { ActionListener> listener = @@ -1090,16 +1105,23 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception { new XPackLicenseState(SECURITY_ENABLED_SETTINGS), cache, apiKeyService, documentSubsetBitsetCache, rds -> effectiveRoleDescriptors.set(rds)); AuditUtil.getOrGenerateRequestId(threadContext); - + final Version version = randomFrom(Version.CURRENT, VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_8_1)); final Authentication authentication = createApiKeyAuthentication(apiKeyService, createAuthentication(), - Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[]{"manage"}, null, null)), - Collections.singletonList(new RoleDescriptor("key_role_" + randomAlphaOfLength(8), new String[]{"monitor"}, null, null))); + Collections.singleton(new RoleDescriptor("user_role_" + randomAlphaOfLength(4), new String[]{"manage"}, null, null)), + Collections.singletonList(new RoleDescriptor("key_role_" + randomAlphaOfLength(8), new String[]{"monitor"}, null, null)), + version); PlainActionFuture roleFuture = new PlainActionFuture<>(); compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture); Role role = roleFuture.actionGet(); assertThat(role.checkClusterAction("cluster:admin/foo", Empty.INSTANCE, mock(Authentication.class)), is(false)); assertThat(effectiveRoleDescriptors.get(), is(nullValue())); + if (version == Version.CURRENT) { + verify(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), eq(false)); + verify(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), eq(true)); + } else { + verify(apiKeyService).getRoleForApiKey(eq(authentication), any(ActionListener.class)); + } assertThat(role.names().length, is(1)); assertThat(role.names()[0], containsString("user_role_")); } @@ -1181,6 +1203,108 @@ public void testLoggingOfDeprecatedRoles() { ); } + public void testCacheEntryIsReusedForIdenticalApiKeyRoles() { + final FileRolesStore fileRolesStore = mock(FileRolesStore.class); + doCallRealMethod().when(fileRolesStore).accept(any(Set.class), any(ActionListener.class)); + final NativeRolesStore nativeRolesStore = mock(NativeRolesStore.class); + doCallRealMethod().when(nativeRolesStore).accept(any(Set.class), any(ActionListener.class)); + when(fileRolesStore.roleDescriptors(anySetOf(String.class))).thenReturn(Collections.emptySet()); + doAnswer((invocationOnMock) -> { + ActionListener callback = (ActionListener) invocationOnMock.getArguments()[1]; + callback.onResponse(RoleRetrievalResult.failure(new RuntimeException("intentionally failed!"))); + return null; + }).when(nativeRolesStore).getRoleDescriptors(isA(Set.class), any(ActionListener.class)); + final ReservedRolesStore reservedRolesStore = spy(new ReservedRolesStore()); + ThreadContext threadContext = new ThreadContext(SECURITY_ENABLED_SETTINGS); + ApiKeyService apiKeyService = mock(ApiKeyService.class); + NativePrivilegeStore nativePrivStore = mock(NativePrivilegeStore.class); + doAnswer(invocationOnMock -> { + ActionListener> listener = + (ActionListener>) invocationOnMock.getArguments()[2]; + listener.onResponse(Collections.emptyList()); + return Void.TYPE; + }).when(nativePrivStore).getPrivileges(any(Collection.class), any(Collection.class), any(ActionListener.class)); + + final DocumentSubsetBitsetCache documentSubsetBitsetCache = buildBitsetCache(); + final AtomicReference> effectiveRoleDescriptors = new AtomicReference>(); + final CompositeRolesStore compositeRolesStore = new CompositeRolesStore(SECURITY_ENABLED_SETTINGS, + fileRolesStore, + nativeRolesStore, + reservedRolesStore, + nativePrivStore, + Collections.emptyList(), + new ThreadContext(SECURITY_ENABLED_SETTINGS), + new XPackLicenseState(SECURITY_ENABLED_SETTINGS), + cache, + apiKeyService, + documentSubsetBitsetCache, + rds -> effectiveRoleDescriptors.set(rds)); + AuditUtil.getOrGenerateRequestId(threadContext); + final BytesArray roleBytes = new BytesArray("{\"a role\": {\"cluster\": [\"all\"]}}"); + final BytesArray limitedByRoleBytes = new BytesArray("{\"limitedBy role\": {\"cluster\": [\"all\"]}}"); + Authentication authentication = new Authentication(new User("test api key user", "superuser"), + new RealmRef("_es_api_key", "_es_api_key", "node"), + null, + Version.CURRENT, + AuthenticationType.API_KEY, + Map.of(API_KEY_ID_KEY, + "key-id-1", + API_KEY_ROLE_DESCRIPTORS_KEY, + roleBytes, + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + limitedByRoleBytes)); + doCallRealMethod().when(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + + PlainActionFuture roleFuture = new PlainActionFuture<>(); + compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture); + roleFuture.actionGet(); + assertThat(effectiveRoleDescriptors.get(), is(nullValue())); + verify(apiKeyService, times(2)).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + verify(apiKeyService).parseRoleDescriptors("key-id-1", roleBytes); + verify(apiKeyService).parseRoleDescriptors("key-id-1", limitedByRoleBytes); + + // Different API key with the same roles should read from cache + authentication = new Authentication(new User("test api key user 2", "superuser"), + new RealmRef("_es_api_key", "_es_api_key", "node"), + null, + Version.CURRENT, + AuthenticationType.API_KEY, + Map.of(API_KEY_ID_KEY, + "key-id-2", + API_KEY_ROLE_DESCRIPTORS_KEY, + roleBytes, + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + limitedByRoleBytes)); + doCallRealMethod().when(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + roleFuture = new PlainActionFuture<>(); + compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture); + roleFuture.actionGet(); + assertThat(effectiveRoleDescriptors.get(), is(nullValue())); + verify(apiKeyService, times(2)).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + verify(apiKeyService, never()).parseRoleDescriptors(eq("key-id-2"), any(BytesReference.class)); + + // Different API key with the same limitedBy role should read from cache, new role should be built + final BytesArray anotherRoleBytes = new BytesArray("{\"b role\": {\"cluster\": [\"manage_security\"]}}"); + authentication = new Authentication(new User("test api key user 2", "superuser"), + new RealmRef("_es_api_key", "_es_api_key", "node"), + null, + Version.CURRENT, + AuthenticationType.API_KEY, + Map.of(API_KEY_ID_KEY, + "key-id-3", + API_KEY_ROLE_DESCRIPTORS_KEY, + anotherRoleBytes, + API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY, + limitedByRoleBytes)); + doCallRealMethod().when(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), anyBoolean()); + roleFuture = new PlainActionFuture<>(); + compositeRolesStore.getRoles(authentication.getUser(), authentication, roleFuture); + roleFuture.actionGet(); + assertThat(effectiveRoleDescriptors.get(), is(nullValue())); + verify(apiKeyService).getApiKeyIdAndRoleBytes(eq(authentication), eq(false)); + verify(apiKeyService).parseRoleDescriptors("key-id-3", anotherRoleBytes); + } + private Authentication createAuthentication() { final RealmRef lookedUpBy; final User user; diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/README.asciidoc b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/README.asciidoc deleted file mode 100644 index 5b2a6b737d779..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/README.asciidoc +++ /dev/null @@ -1,36 +0,0 @@ -= Keystore Details -This document details the steps used to create the certificate and keystore files in this directory. - -== Instructions on generating self-signed certificates -The certificates in this directory have been generated using the following openssl configuration and commands. - -OpenSSL Configuration File is located in this directory as `openssl_config.cnf`. - -NOTE: The `alt_names` section provides the Subject Alternative Names for each certificate. This is necessary for testing -with hostname verification enabled. - -[source,shell] ------------------------------------------------------------------------------------------------------------ -openssl req -new -x509 -extensions v3_req -out .cert -keyout .pem -days 1460 -config config.cnf ------------------------------------------------------------------------------------------------------------ - -When prompted the password is always set to the value of . - -Because we intend to import these certificates into a Java Keystore file, they certificate and private key must be combined -in a PKCS12 certificate. - -[source,shell] ------------------------------------------------------------------------------------------------------------ -openssl pkcs12 -export -name -in .cert -inkey .pem -out .p12 ------------------------------------------------------------------------------------------------------------ - -== Creating the Keystore -We need to create a keystore from the created PKCS12 certificate. - -[source,shell] ------------------------------------------------------------------------------------------------------------ -keytool -importkeystore -destkeystore .jks -srckeystore .p12 -srcstoretype pkcs12 -alias ------------------------------------------------------------------------------------------------------------ - -The keystore is now created and has the private/public key pair. You can import additional trusted certificates using -`keytool -importcert`. When doing so make sure to specify an alias so that others can recreate the keystore if necessary. diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/active-directory-ca.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/active-directory-ca.crt deleted file mode 100644 index 453d1361ce434..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/active-directory-ca.crt +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID1zCCAr+gAwIBAgIQWA24rVK7FopAgOHfEio/VjANBgkqhkiG9w0BAQsFADB+ -MRMwEQYKCZImiZPyLGQBGRYDY29tMR0wGwYKCZImiZPyLGQBGRYNZWxhc3RpY3Nl -YXJjaDEUMBIGCgmSJomT8ixkARkWBHRlc3QxEjAQBgoJkiaJk/IsZAEZFgJhZDEe -MBwGA1UEAxMVYWQtRUxBU1RJQ1NFQVJDSEFELUNBMB4XDTE0MDgyNzE2MjI0MloX -DTI5MDgyNzE2MzI0MlowfjETMBEGCgmSJomT8ixkARkWA2NvbTEdMBsGCgmSJomT -8ixkARkWDWVsYXN0aWNzZWFyY2gxFDASBgoJkiaJk/IsZAEZFgR0ZXN0MRIwEAYK -CZImiZPyLGQBGRYCYWQxHjAcBgNVBAMTFWFkLUVMQVNUSUNTRUFSQ0hBRC1DQTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNNZsDJ+lhsE/pCIkNlq6/F -xwv3PU2M+E1/SbWrLEtfbb1ATnn98DwxjpCj00wS0bt26/7zrhHKyX5LaxyS27ER -8bKpLSO4qcVWzDIQnVNk2XfBrYS/Og+6Pi/Lw/ylt/vE++kHWIJBc4O6i+pPByOM -oypM6bh71kTkpK8OTPqf+HiPp0qKhRah6XVtqTc+kOCOku2+wkELbCz8RNzF9ca6 -Uu3YxLi73pNdk0wDTmg6JVaUyVRpSkjJH4BAp9SVma6Rxy6tbh4e5P+8K8lY9ptM -TBzTsDS1EhNK/92xULfQbGT814Z294pF3ARMEJ89N+aegS++kz7CqjciZ1+bA6EC -AwEAAaNRME8wCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE -FIEKG0KdSVNknKcMZkbTlKo7N8MjMBAGCSsGAQQBgjcVAQQDAgEAMA0GCSqGSIb3 -DQEBCwUAA4IBAQBgbWBXPbEMTEsiVWzoxmTw1wJASBdPahx6CggutjGq3ASjby4p -nVCTwE4xdDEVyFGmeslSp9+23XjBuaiqVPtYw8P8hnG269J0q4cOF/VXOccRLeOw -HVDBv2a7xzgBSwc1KB50TLv07stcBmBYNu8anN6EwGksdgjb8IjRV6U3U+IvFNrI -rGifuIc/iRZD4Clhnpxw8tCsgcrcmz9CU7CN5RxKVEpZ6ou6ZjHO8l8H0t9zWrSI -PL+33iBGHNWlyU63N93XgJtxV1em1hHryLtTTtaVZJJ3R0OrLrUpG8SQ7zCUy62f -YtImFPClUMXY03yH+4DAhflueRvY/D1AKL12 ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.crt deleted file mode 100644 index bcabf51acb658..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.crt +++ /dev/null @@ -1,24 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEBzCCAu+gAwIBAgIBAjANBgkqhkiG9w0BAQUFADBGMQswCQYDVQQGEwJVUzEW -MBQGA1UEChMNZWxhc3RpY3NlYXJjaDENMAsGA1UECxMEdGVzdDEQMA4GA1UEAxMH -cm9vdC1jYTAeFw0xNDA4MjcxNTMyNTNaFw0xNTA4MjcxNTMyNTNaME4xCzAJBgNV -BAYTAlVTMRYwFAYDVQQKEw1lbGFzdGljc2VhcmNoMQ0wCwYDVQQLEwR0ZXN0MRgw -FgYDVQQDEw9pcC0xNzItMzAtMC0xNTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQC5iQ7DiQwT1YVoLycCGsm3p4Lp8V3uo/+sV9dA4fdpJ7i84Fq0V8g6 -0VjKMIhL8mrzbjCF8hDREO6fnuAwOM8IqladwFzEqFPnvLo/Uv5J7/BgqRN2H25x -REQrOOMRsvWZWslV4aAj3ivmYu3HU6gA0I1/OsI0MCU/kotdVRkAEcu3HL16AEGK -4CkQhw3JKt7iWm2vODQ7BZadrNvjKS6C7pqV/c4+r4T1aEQNbzvM7Br70NLqA9M4 -HeOJjyqEbbr9rHtIP9Cy0M6Lp52ho7sTA2D5b/bWCSog6DvClmj2NCySR2lR1QYF -S7qrsyE6ePq8TnyoReOAxqJdjm+tFvzzAgMBAAGjgfcwgfQwCQYDVR0TBAIwADAw -BglghkgBhvhCAQ0EIxYhWWFTVCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRl -MBEGCWCGSAGG+EIBAQQEAwIGQDALBgNVHQ8EBAMCBaAwHQYDVR0OBBYEFDH+zO6n -PQHyeqiKfo5kVYUQpSJDMHYGA1UdIwRvMG2AFO/jQiwne6+hZEs7afKnyoddejH8 -oUqkSDBGMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNZWxhc3RpY3NlYXJjaDENMAsG -A1UECxMEdGVzdDEQMA4GA1UEAxMHcm9vdC1jYYIJAOwOXHgWfvKEMA0GCSqGSIb3 -DQEBBQUAA4IBAQCk6wXIRaIQrhjD1p9wkv2tjEmnktoic8I0Z3CaHe80aLLcX5n/ -7Vlo4ccp3mWAhON7dCGSj9Qc7wj28h3CkDIIntQpErhEhEpSRU70P3SD7jn3tcrF -fu+SUwtP1YLLjU9S10Jx0eImufMj0AgwIL+axjxgCfQ+nxeLWTt35zZzQqizkSSy -PVGkY/YZy1tc4JdJl9TbwGsxWgLTHs7bD1WPAYovreblRuHNEwabwwgDK+F6G7Lh -BVYCygiuyG/MehQZSgb2LmX4O1QyVe2bZJUZQNMdZLORRdGQXf6grakdolqPIASZ -SpzwTZU5InUGvQG0HuOn//+wHZ0rih/qWV+3 ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.der b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.der deleted file mode 100644 index 8cdc214f0eb01..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openldap.der and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openssl_config.cnf b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openssl_config.cnf deleted file mode 100644 index 38adbbc776ef8..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/openssl_config.cnf +++ /dev/null @@ -1,35 +0,0 @@ -[ req ] -default_bits = 2048 # Size of keys -default_keyfile = key.pem # name of generated keys -default_md = sha256 # message digest algorithm -string_mask = nombstr # permitted characters -distinguished_name = req_distinguished_name -req_extensions = v3_req - -[ req_distinguished_name ] -0.organizationName = Organization Name (company) -organizationalUnitName = Organizational Unit Name (department, division) -emailAddress = Email Address -emailAddress_max = 40 -localityName = Locality Name (city, district) -stateOrProvinceName = State or Province Name (full name) -countryName = Country Name (2 letter code) -countryName_min = 2 -countryName_max = 2 -commonName = Common Name (hostname, IP, or your name) -commonName_max = 64 - -[ v3_req ] -basicConstraints = CA:FALSE -subjectKeyIdentifier = hash -subjectAltName = @alt_names - -[ alt_names ] -DNS.1 = localhost -DNS.2 = localhost.localdomain -DNS.3 = localhost4 -DNS.4 = localhost4.localdomain4 -DNS.5 = localhost6 -DNS.6 = localhost6.localdomain6 -IP.1 = 127.0.0.1 -IP.2 = ::1 diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-cert.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-cert.pem deleted file mode 100644 index b85edb30b02df..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-cert.pem +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICYjCCAgmgAwIBAgIJAPBTfsMrh6VTMAkGByqGSM49BAEwWDELMAkGA1UEBhMC -VVMxEzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdp -dHMgUHR5IEx0ZDERMA8GA1UEAxMISmF5cy1NQlAwHhcNMTcwNDExMTM1MDQ3WhcN -MTkwNDExMTM1MDQ3WjBYMQswCQYDVQQGEwJVUzETMBEGA1UECBMKU29tZS1TdGF0 -ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMREwDwYDVQQDEwhK -YXlzLU1CUDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABMMRTMV1HqtZPH7dwtWI -q/kLw8cYV8vK7bN7Mi09q9JQogbvwRkVir8b1/3DgUEvLv+8u8zgcIcx2iaWtaLz -rfmjgbwwgbkwHQYDVR0OBBYEFGgv2RGxkfH8fDYUMiAcLgSZ2el2MIGJBgNVHSME -gYEwf4AUaC/ZEbGR8fx8NhQyIBwuBJnZ6XahXKRaMFgxCzAJBgNVBAYTAlVTMRMw -EQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBXaWRnaXRzIFB0 -eSBMdGQxETAPBgNVBAMTCEpheXMtTUJQggkA8FN+wyuHpVMwDAYDVR0TBAUwAwEB -/zAJBgcqhkjOPQQBA0gAMEUCIBI2zkYo8aZImnlXxIS+7cILdx8AKo6VNvGykn3X -k/n1AiEAp5O/xswzb35GZbAnNCbXDYi2Ny2mv1S9WypHC6Y5/qk= ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key-noparam.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key-noparam.pem deleted file mode 100644 index 2cb86113d41fc..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key-noparam.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIN74E4fO1Pq89hh7NYjUBFu7akoHC36ZvlnHfcCASq5ToAoGCCqGSM49 -AwEHoUQDQgAEwxFMxXUeq1k8ft3C1Yir+QvDxxhXy8rts3syLT2r0lCiBu/BGRWK -vxvX/cOBQS8u/7y7zOBwhzHaJpa1ovOt+Q== ------END EC PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key.pem deleted file mode 100644 index 51a3f018558fe..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/prime256v1-key.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIN74E4fO1Pq89hh7NYjUBFu7akoHC36ZvlnHfcCASq5ToAoGCCqGSM49 -AwEHoUQDQgAEwxFMxXUeq1k8ft3C1Yir+QvDxxhXy8rts3syLT2r0lCiBu/BGRWK -vxvX/cOBQS8u/7y7zOBwhzHaJpa1ovOt+Q== ------END EC PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt deleted file mode 100644 index 59ecbd22e8b23..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/samba4.crt +++ /dev/null @@ -1,22 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDoDCCAoigAwIBAgIUMVGoHuyNTjTFaoRmqFELz75jzDEwDQYJKoZIhvcNAQEL -BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l -cmF0ZWQgQ0EwHhcNMTgwMjE1MTc0OTExWhcNMjEwMjE0MTc0OTExWjARMQ8wDQYD -VQQDEwZzYW1iYTQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtGBwa -n+7JN2vweSUsYh4zPmh8RPIE+nEVjK1lx/rADUBY7UVjfTYC+MVKKiezZe7gYCNT -7JNKazPpgVI9e3ZFKw/UxomLqRuuvn5bTh+1tMs3afY5+GGzi7oPmEbBO3ceg0Hi -rNSTDa1rfroZnRYK8uIeSZacQnAW90plITI7rBBt9jq+W9albFbDybfDgNv+yS/C -rzIsofm4rbFC3SMRYfrT6HvwDhjOmmYKZci5x7tsn0T+3tSiR44Bw5/DgiN5kX3m -/kl9qg1eoYWbCUy1dKmQlb4Nb4uNcxrIugLB3zjBkfhMZ0OHoveKh/lJASTWik9k -xQ9rEYbpsRbuXpsHAgMBAAGjgcwwgckwHQYDVR0OBBYEFJOLa7UXKtLPibgKeFh7 -Kq1+rS0/MG8GA1UdIwRoMGaAFGaNmN5mi9jaMW25MEWYgt+5OkDBoTikNjA0MTIw -MAYDVQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBD -QYIUdwsnIxjgSneHNVKT6JNCCsrQ3T0wLAYDVR0RBCUwI4IJbG9jYWxob3N0hwR/ -AAABhxAAAAAAAAAAAAAAAAAAAAABMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQAD -ggEBAEHqT1WHkcF8DuOgyIBx7wKcUVQ5H1qYYlJ1xgMGrKFFZLUzouLcON7oadEu -HLIJ4Z3AKD3bqWpcls5XJ9MTECGR48tou67x9cXqTV7jR3Rh0H/VGwzwhR85vbpu -o8ielOPL8XAQOfnAFESJii5sfCU4ZwLg+3evmGZdKfhU6rqQtLimgG/Gm96vOJne -y0a/TZTWrfAarithkOHHXSSAhEI5SdW5SlZAytF4AmYqFvafwxe1+NyFwfCRy0Xl -H40WgVsq+z84psU+WyORb3THX5rgB4au9nuMXOqFKAtrJSI/uApncYraaqU28rqB -gYd8XrtjhKOLw+6viqAKu8l7/cs= ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt deleted file mode 100644 index d14805ddc89d4..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDizCCAnOgAwIBAgIJANif+/9AeRBYMA0GCSqGSIb3DQEBCwUAMCQxIjAgBgNV -BAMTGXRlc3RjbGllbnQtY2xpZW50LXByb2ZpbGUwHhcNMTUwOTIzMTg1MjU0WhcN -MTkwOTIyMTg1MjU0WjAkMSIwIAYDVQQDExl0ZXN0Y2xpZW50LWNsaWVudC1wcm9m -aWxlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2g9jTDeAZMjXygm4 -Cb9CkBhSGN738ZOBiTGDH0q/3inE/qKJj/SF6RjHrVx5v9QUlqVTBX/NPd0VSFy7 -jbYtab8mlaYEAhatYQhHlEjcre1hjK3MW2oHhk+T9ScnIBeArBjUSUzwPAUM8FOj -0qGSK6QifrCf/5dQyq9D1WynwhGwUqp3WcZEvvaEl66QnBugViY7JRsuWOfIiJ7j -xkFv74Qm3yfCpQR+5TPAWd2ipRZZinwn9M0u0txiffZo7jemdpK5aiNAgJCGn/Sw -jlOzFU5pKxIIHeaR3rLWZ4P0WgMu+kqVPoy66UKBA6kXK4eMk1Wla7dG1gtp+OmQ -TQGqhwIDAQABo4G/MIG8MAkGA1UdEwQCMAAwHQYDVR0OBBYEFGgAgQUFrcvEOeed -swWf+D5DRoj9MIGPBgNVHREEgYcwgYSCCWxvY2FsaG9zdIIVbG9jYWxob3N0Lmxv -Y2FsZG9tYWluggpsb2NhbGhvc3Q0ghdsb2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIK -bG9jYWxob3N0NoIXbG9jYWxob3N0Ni5sb2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAA -AAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggEBAKrKuUTmZ3WymiG6jRhLA5DA -36k7Hey8rvHZ6wbdnUseO+JJbItGEoEd0Js2syjghOnMf+M686prv0tWpaqA3ySh -uEcKSwy063eMmmTPXR9HfEK/QkdfC6otOvR/elgs1t0R45JFbHWSS1tJLOU/HU2i -3jwqy8Gwp+XJG9yrwC4OsJ1pAp7vCIoXrcq6glueiq9vjdAMYCtlrNIlDhmB+rYD -K8mf4S1AygSsZNUmh04oj8kBRacH4vdoKYkxvixPvo1F5VaoJcr+hh2xL4AyuqLc -gdoJmOv6Fk2UCUI2aI4nn0pXwkyOlx0t8B3Hp/TZY2Yn0x+uIIkKGJRm0CuKzNQ= ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks deleted file mode 100644 index 2e0923de67a37..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.p12 deleted file mode 100644 index cece3d1243208..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.p12 and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem deleted file mode 100644 index f5ea25da6967f..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient-client-profile.pem +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,02BA9C3E433F0711 - -M3Ds1K8/cp5v9BKj9SstoXWlyj+CmasLfvPlz6O5Ootdai/wTirhxyAcYcsgjuQA -FIMuQiAaKC+W/DOP52+ilY3FZKw09DptdkaU7IFvSQvJ5SeiJIGLZ9REUWIPwRKV -qCHpKFCS4//mitnykjSu2HhTM+BjbXuytLJtxFJuodr8YhJGgtwfETAmbVlwWG3M -mhEsfdr0kOJGwT/z3dZbCxnGkjOY1FZCdMU/ggeYx93NP0c4yzNU6/A0idZRMlHE -BuSIkDhn3BXxgxYxTOgHzCkxaZ0T7kNv0MvTXrrcQf2WrMaG1FfTzCzF78YDBn7+ -kg5HI8wfxjGjDTnHH9KSuN5gi14DiEEUN45Ba3ChH75SjGwBbUjnHsGIp6Ch8mh8 -5ZgldgUOff0atEBaMpvoZAzxGbY5PhC5/Pfiq+enzvmICxszFNIRMhhLxWWBWuH9 -v04OOKZJ4v+ygZHvPJEFNc/XznGb39ELn4Y6P26fqNfPmv5/yHcenQPNZgGkKQEJ -SYTPxct+yDi3lul1WBPrvZwCOAjUtcCf4tBC6uOqlRjd7VREk5hKUlXTtxJLbaoB -FH1VKn9xZITBqk3g292KGbpZsGP5HtYAbB/gJUcOXZZbr8blMmpeIXXFWfPbvUcv -d1fokqHSrlclNt+h2pENqjJTc2iKKKlkNY0ol+cDej7fOgNH0sNOjuESrofdaXfa -4CPuvMxeQFKnjQFBLM14CFUyseZsAqnLjneUnk5qTPE5twCN0pkJx0y095pEGdrF -2UQ4HfdghnKxWQK50iGUhCgBfqeI3O3cfi76ZdeJYM+7CluGNb/0sybjF6kNiAME -m5PfaSXk7A+a/FteFThW52E8W8Sekf2/lOMdbdQa26/oUfcMWzBj+aMcJBwcOaq4 -lIY6IPHySQIakLGCLLOkYHAG+Kp8AldjcwfpOZ3Jk1HrjOmx9CfonXvdVQmS2BQZ -8+V2ECd0uw9EbD+i4VIIHy6H1a57W9gikshXsv4K5vQsl0s6GjbioN1JFFC78XKS -XohDMDmR2Ty3LU9TLoS9K2fLcgtoJmnEtJ1cs5Cw9+T1s+ILFgoBTXzePnDKK78T -4uNw7a8CRAKW++dHtwHEnHOIDJ0rx+4Y+V7e5Cr0nT4Idv5P9xe9knTghb34vwBw -SBfEn0sCQoly4DGsKlPz8xhLRCX7euXsbPcTy3owpTFywJsfjR5e/jtxa6C5NmNI -LmGJb6TpsAl9mULfSRlGjynqU9T0EzOdcbPs8z1cLFQl7QRi63tlZ9jWh+7eV+dR -NaN6f0jY0me0D7OzvoD19e3ESFARV7MGQ5U0OaZOAcLdx3CYjs777AQ1S2q6ra+p -gr4oqenWqYkbY8/z8AALiWPO59Rl00GcUJqwYXlJANCtK4ikwQPG2+YIYrclBW/d -ZQplHShy0AayYPguLI9ts0XLulIrcsJ0zlEDPf3b8+Gr6yONiYpOtA0ZtwE3w+Q5 -5byhoYoBQPtLXyLff31ozRs/MhJd8+gIIReKSE5sGMwbVX2ZfjWOXzDHp6+CQIKD -OhIfgnyG6tPQ2N4DhSrE/cEITPlSOv55NUnLQUCgfAHvAslgWwmVDk1ikznij/Zj ------END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt deleted file mode 100644 index 18221208c162e..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.crt +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID1zCCAr+gAwIBAgIJALnUl/KSS74pMA0GCSqGSIb3DQEBCwUAMEoxDDAKBgNV -BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEiMCAGA1UEAxMZRWxhc3Rp -Y3NlYXJjaCBUZXN0IENsaWVudDAeFw0xNTA5MjMxODUyNTVaFw0xOTA5MjIxODUy -NTVaMEoxDDAKBgNVBAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEiMCAG -A1UEAxMZRWxhc3RpY3NlYXJjaCBUZXN0IENsaWVudDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAMKm+P6vDAff0c6BWKGdhnYoNl9HijLIgfU3d9CQcqKt -wT+yUW3DPSVjIfaLmDIGj6Hl8jTHWPB7ZP4fzhrPi6m4qlRGclJMECBuNASZFiPD -tEDv3msoeqOKQet6n7PZvgpWM7hxYZO4P1aMKJtRsFAdvBAdZUnv0spR5G4UZTHz -SKmMeanIKFkLaD0XVKiLQu9/z9M6roDQeAEoCJ/8JsanG8ih2ymfPHIZuNyYIOrV -ekHN2zU6bnVn8/PCeZSjS6h5xYw+Jl5gzGI/n+F5CZ+THoH8pM4pGp6xRVzpiH12 -gvERGwgSIDXdn/+uZZj+4lE7n2ENRSOt5KcOGG99r60CAwEAAaOBvzCBvDAJBgNV -HRMEAjAAMB0GA1UdDgQWBBSSFhBXNp7AaNrHdlgCV0mCEzt7ajCBjwYDVR0RBIGH -MIGEgglsb2NhbGhvc3SCFWxvY2FsaG9zdC5sb2NhbGRvbWFpboIKbG9jYWxob3N0 -NIIXbG9jYWxob3N0NC5sb2NhbGRvbWFpbjSCCmxvY2FsaG9zdDaCF2xvY2FsaG9z -dDYubG9jYWxkb21haW42hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMA0GCSqGSIb3 -DQEBCwUAA4IBAQANvAkddfLxn4/BCY4LY/1ET3d7ZRldjFTyjjHRYJ3CYBXWVahM -skLxIcFNca8YjKfXoX8mcK+NQK/dAbGHXqk76yMlkrKjh1OQiZ1YAX5ryYerGrZ9 -9N3E9wnbn72bW3iumoLlqmTWlHEpMI0Ql6J75BQLTgKHxCPupVA5sTbWkKwGjXXA -i84rUlzhDJOR8jk3/7ct0iZO8Hk6AWMcNix5Wka3IDGUXuEVevYRlxgVyCxcnZWC -7JWREpar5aIPQFkY6VCEglxwUyXbHZw5T/u6XaKKnS7gz8RiwRh68ddSQJeEHi5e -4onUD7bOCJgfsiUwdiCkDbfN9Yum8OIpmBRs ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks deleted file mode 100644 index b4cf27dfd26a8..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.p12 deleted file mode 100644 index cbe7195f285b1..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.p12 and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem deleted file mode 100644 index 7268c55dba977..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testclient.pem +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,C98A45E4AFC263C2 - -wLuUEXldYc54r4ryWd6jw6UMGYwn6+ibGKHp4sD92l42lmI2UrCT/Mb/E0O+KMMy -pHgc5/dBWkXgMiqDyLIhHk4kgT40rdw5W5lZkAA4Qt/Yzd+rbscTvzp09zrF6Fll -czgoE7FrvhOKiEOakerTit4pIPYosdX606cpVQE2lq9oZs9HVMcLzdAZj8A/P/4g -fo4X3+zqVYC/LH4n00bhNoeeej2o1lEJ+l9u9hptT2ATXle6pANa83Ldg4OxJyj8 -dkR9ahnAMCvYTSjEU7nwmGNPeFX0PIUjJKQivr410cYG104DC30Yy+XrIUfjTVUi -agwlMpHoBq79/ZRUJR3xPLkIGgw4g+RPt45D9eKsEsV4vqy8SFlgaoJ2mKUKleZy -i7D9ouzMKQ3sYE4eQVQ5o3K8ZPn5eozCwCVIp7jGSsuvDpLA9peZSwWPfc5y8JFD -/64usCt1J8Mv/e9NVllC8ZA+ZmDitTiwLZysczpMOaFqqeUbk9EJst38n4nBzRV2 -quxvg9W/iveQIydFyftCtNfRkpbp0NCsLz293dBYwZacHsPcY27IBCwXHiICjiAW -q7bnisXsgSaQMhMNRGW9YElZGb7ZWxoIzcyNBisGI8zxn48ObERVOmkOFxY/gs9T -YmpVMliWtmRG6hb6iCh9b7z8THRquxgTGE9ZFBwtLUKg33aubtgAfnUh/Xq2Ue5K -l+ZCqDGEi/FSIjVENUNNntAx/vXeNPbkoGLb/HSJwAh+sjpaLGQ54xixCtE9l3NY -o2QAiZ804KLPaGtbbOv7wPumxQ+8mxG5FN0hTRrsMW9t8pBXw47iMy/T2H21TD5D -E5XbM6kFeBrnsWnZJ2/ieXqDE4SX0tm3WEvZlDg7N7jV8QDM/D3Xdkb/sqJRabMG -tQRgwkLiB+mZ5MAfGLogI2/lOEayrBVz4qYdXojewxY4LtaZ5HiUIlyA9CJelMvD -nS52I6+FpaFhvuZC10qaM9Ph9TNyx+XKRUsPILuDiBRnYiHUKs1qASl5tjn2yyjM -71WSo7A7btOckzhDZdMVf1T472f0LGsRYoQebMhotqCuR7yArZHzTeWB0CjL3tOz -j3QlhKt2E1jx43bSK5tBasd9Bpmn2onvdwu1RRP8cyQBsXJSDy4/8t/g63+C3wod -8VPrlKhK+TenK9EoEqJ2mNuNq+duOjTXfK/7GM5s0BFKv+i2ckpDi1NPckd2gXjF -yUFZhmK6k0WC4jjWloMt+WQpi1rXMEXwCypgTrqWbvD0p6+X3uQmP57L4yHQcZoW -Qcs5GnihJ0DIhw9vYDhBhNo0WY1oBO20nVCN3R/JIpp3uDtg64WvfvMSXzJIPBCY -s+/GM5TtuD6mERDu3+qXxWwiy4PMQRcgjRTMEZ3A4Iv77YfQRkcd6S9qjUUuR/5D -xs+J4ryb1biz9ofW7I+Dbz4SArWSgwcuh14AV9RBv6Rh9m83rjT2K0yvbe/+7hHW -R8nzRMqJcGNGCHmRjA/cwoiv6+k2J/RbCJqnR3RmNex/85XaXBfZwRfHXVbzZQfa -SrFaaNLf1hMwGLAJjIcQRxa3yZbjFXVx1Bp4hh8rKNWaOItjavNtNg== ------END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.crt deleted file mode 100644 index 47e5b37c28b35..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDhzCCAm+gAwIBAgIJAMDqQhbo/w/YMA0GCSqGSIb3DQEBCwUAMCIxIDAeBgNV -BAMTF3Rlc3Rub2RlLWNsaWVudC1wcm9maWxlMB4XDTE1MDkyMzE4NTI1NloXDTE5 -MDkyMjE4NTI1NlowIjEgMB4GA1UEAxMXdGVzdG5vZGUtY2xpZW50LXByb2ZpbGUw -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDh9VWS5kI7Sf1tJSCP3F+l -F1Ve2w9GgoRYjOz7jTWm8h8WPoMSGoCIEQqxm+Eo+AGCd58tacKO4ANxZbjh33hP -7bh7jYmj3qDZlwnTaM4RIvzPSgYzlgx7iBWXJZoxWi9jlEBehg/bB6+2tK0/4CJJ -27Xi2cjJDTcctyLEhuQEH6oLf0pBfK2UXv9LkoOkrnRzY+x4YBOlrPmho+5jU2iw -5DnFNKw4Cxdd9XnDpAprU2E02pCXN3mpcvDk2MdnzWvDh14j1eisxmAl7wbU5fdF -P50v5m8cOPreWUlS2WYIJ+nHLnbnmVE8F2FpXDupF5WK5BU/QHVnE6i5kwvMITsJ -AgMBAAGjgb8wgbwwCQYDVR0TBAIwADAdBgNVHQ4EFgQU08G5lM8hL/8f61uAkl3n -vJzyEv4wgY8GA1UdEQSBhzCBhIIJbG9jYWxob3N0ghVsb2NhbGhvc3QubG9jYWxk -b21haW6CCmxvY2FsaG9zdDSCF2xvY2FsaG9zdDQubG9jYWxkb21haW40ggpsb2Nh -bGhvc3Q2ghdsb2NhbGhvc3Q2LmxvY2FsZG9tYWluNocEfwAAAYcQAAAAAAAAAAAA -AAAAAAAAATANBgkqhkiG9w0BAQsFAAOCAQEAq84Eku8yLOcaxfBb7pHWaHHsaxXZ -k1lK3SZm49VTuwtMrGCYY7TBt4Kz1mnpQZd9KVyE4BzhpvUmD2ybSsyK/w1nYGVw -VyFEnutlIsYWs4rWwrvYoX1/B86WvNMBa+XFBnlO0HR5yTc/m5LKkNsZ3p2CGPfN -PCaok0NwS7cKIsWIgBaFlIYMYjZM+cL5qAWeOOgQoxPKi5uzZ/YqJbWf0jwpA/8s -M9QuqJzJMSGvwn0i74ulbubjnit2dMbrUUm6W65snw5qLDSqZzDH02EYYG4yMkmP -/S/SZFvw7fuOHFeJBoO3oej+ilEuYPkjkysr3Ys6xmprqExdDHUMElEycA== ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.jks deleted file mode 100644 index aed185d975567..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.p12 deleted file mode 100644 index 111d6a79f9b0e..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.p12 and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.pem deleted file mode 100644 index 4f5b0e56782ee..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-client-profile.pem +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,AA1E05F81F24B5E3 - -Axn5ritl59GJQ4PLJDIk/IRvYSPsk+5JygqjZc4oeewhpbWqOjzVYROMD65DqN4k -sZw9vhDSm8TjhmuMWP+n1oJWZtAEKBcXbAhoG8lErsJPYx5xq9SYrbK6LWXYlVN2 -fbf5o77vQDqLiZf2QeXyufgLbEZ7yslYPBSMiOQfp2jwpHOQrlWIg7awheTpd8Dv -PQ/Jy45mGJ2gl5n1Yn+nZbIO/S+3Wy0AX0MmS7nBU4ogOkIMq4ZBFALnfmcdGZHJ -pfi2riNF6lTeMgHrGtCRY3yp9Hyu0Ve/2fYdfECcG2BUQ0TweZynMqE3kJokwwC6 -ua3T3WBnQws1MYxvYjy8WaSriuAfNvnu80rdiJRRcp8wBjbSYrxHq2BtYMrVuekt -lFdDCBAnmbTzYVFCez4K5l1Tbd0ilc7D8dFhHP07dkkcm57P2cm3u76O0pIqjzEx -DvYs33vdxiFq4Z4QPDbFh6Xz9uaDhy1gxlm8/giHd+JBPJlcoHD5C2Wtd0pI0tz/ -5n1bdFeejqItmfy45IFFrRn/jb7Q59olVNSbWH9RojLx9HQct4w6MScmv8n8J/r9 -PUrapSwwLEV6lYDBMsO5RXXDDUTJHcXVbr1nuU6BTCTTe1nMIK1+8pS1m4Cizjcg -9J3FWLktCqiCx8NI9O/QzIsbyA6R/NjmvYPOx0/nqtMCPNYJHuZyExMiVMMaPNly -Ic0HfnZyiwJPj52l67xzm4KwJH57piwldRGjVCj7IkVbHs2MRVsT0+j/QKGSU0uY -DTHIOSToCrzorh0j01fmG4zpMFkU/rtCmT3STkSFdsEsy9EORGx2C9t4mVth3V6e -Rc0y9TUcKGdtW13OiZN4o12aToC7yhg0aIpseWm8I0nnRg/TVtzoltOv85I0Lelr -vcQ8/jZ5Go8JZbTrMFmCdIPYvx+m7lL7kGx9Zt3hDpj+EBm0g5Gp44KUkxtI7Bs8 -1sUvSTyNKW8mixkiS4Bq4dGDqHIuvwW1x9PVoJODbHzTip5aAzJWmXA5rOxVp2kz -keA73TUpJo2UZkME8SiDGVlcnZAGx0odxe1tMBkc2GPpQVmz8Xmi75aEuvzBlAEh -2LWGnBhgN1T6cwvkJKgdbVHUlt3fUIPHzZhItMKrPbATMl5Rtf8Fkj4TYGvoxJZh -QPLuD7ngTORv84W0xnxlt2KO8hEkpKjTImu20n4FIb/YPlVxbYULnt9xRl/QaEol -7jjpZJ7zmuZBAEevOlQk/Jc05TbdttPoWU6zVsSswjuAn3lVRIK9Ad0iXzPDWkWU -Du41QXpkCXCoIdky1WZjmZPqLd/nnSdKSBt52ZtMimFix7uer/u9f+933lETSK7U -ImUFAP/pnlmCzll1Rmublb+WDyFV2lIZ2rZURIQUuQMCCcAFo9XmWgdTjJAOUdwX -qLvYUNIVb2B5uAUgm8Wn/vxUW04Jn1aQrPSPGBCo9KSy16DbkslSWpJeCHwS+FBP -Z8CedsnajbUnAa3HebykjlhOkob7wTCogzLjaPj07XYa/O2Dwmiv1GOnXrdie9Wi -ul0/CAm53ZtLOrgHmi1cPq9FtYAYJLboLu7eW2jZz2MjcugGG+0eGdzSoKSfWfOX ------END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-different-passwords.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-different-passwords.jks deleted file mode 100644 index cec32ed75a930..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-different-passwords.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.crt deleted file mode 100644 index 6ff84ba6674c5..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDYjCCAkqgAwIBAgIJAKP3WbDN1WxEMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV -BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp -Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTUwMTE5MTQwNDE3WhcNMTkwMTE4MTQwNDE3 -WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV -BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3a5+yBl2rVEGwlOjw6Ji43+iqvaAbmVhnCk6laEa3GpzothX -7HhtGGDfjdhaLzWF5PWP8SvMM8g4f1PLN0hGSR7vrWjlnpvUDXIHsoIRqWfYdwDA -RNiUvOI4FBjN4pZ4sXyUYsTpw80l6W0r3zopyycE4+4HJv55U1Yy2/3qzv1IITqD -LwRt6VpbPGVyzDSBMQXEgfT7sfaJB9Ru+A/onIpEicrWhgCPHrBnSUkKCKNj9AX/ -RV6/yQYnS/KhLx/eQTP7NVcbrC2J4fFOLX9oZAj6dir/tYQ6rDAMqBTnbhGygDqP -0RgCVf82n6mA23n7l5DaZ4RZl+ssN3fNqDyDpQIDAQABo08wTTAJBgNVHRMEAjAA -MB0GA1UdDgQWBBSDFYaN/Od9ad7Kztv6cGjd2X4w1TAhBgNVHREEGjAYhwR/AAAB -hxAAAAAAAAAAAAAAAAAAAAACMA0GCSqGSIb3DQEBCwUAA4IBAQCbxk4VHMcdD2yU -VpLSBxHBdWY/Gn3f7k0WWhQAmRPR+S6vSr89hVO8UIkqEFzc+D19s9h0XvAmo2QO -G80OLTcHIjxiVqAVWoGUPH4D7FdG7sSBbrJbweIBMW8Ba4kefWGcI0KlUWTssFyE -YLJQIUCIdKtVf/qmcItvrEXw8ucSYaExvizMClTvf2fZxRws8Omo3dxn+ifUH5nn -evQW8Tawlx2ql5P6wHTSUclGLv1CXtMAnuOlyaYY/UbslNhSXigxYOZCI3ole3qL -Z0dBah+eCspOit14mi7jHPoS1Yji/CSh33KZD6ZESuv/V1B7oy6zZWei/b4vllW5 -RwZx1Y/r ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.jks deleted file mode 100644 index 7e757a3704939..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-ip-only.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.cert b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.cert deleted file mode 100644 index ced9d81d96fa6..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.cert +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDTTCCAjWgAwIBAgIJALL7dwEsWamvMA0GCSqGSIb3DQEBCwUAME8xDDAKBgNV -BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEnMCUGA1UEAxMeRWxhc3Rp -Y3NlYXJjaCBUZXN0IE5vZGUgTm8gU0FOMB4XDTE0MTIxNjE5NTcyNloXDTE4MTIx -NTE5NTcyNlowTzEMMAoGA1UEChMDb3JnMRYwFAYDVQQLEw1lbGFzdGljc2VhcmNo -MScwJQYDVQQDEx5FbGFzdGljc2VhcmNoIFRlc3QgTm9kZSBObyBTQU4wggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCkIGS7A/V6TesR34ajMyNYL3tB1OjW -Raq4KtF8FfW1H6nHGrWa/qXjZWPirczy1k2n6ZL7YOCcv/YeY8xAqC9mGQxvEuqo -EaqXq2cjRdAs/7zqzRkdPPi3Jw/p/RHrDfOAzOsMnBGc0G2Hrsj//aP44vp85pek -fM3t2kNAYZWYCzXUqWAIUoxBDK4DcQdsN8H4KTMIwQEEiRtcKnL/b8QGKsyGLfLq -36ZABHZ4kY2SmcP3bWxZtbFN4hamdwoAtYe+lS0/ee8/fOTLyZ3Ey+X6EEmGO1lk -WR4XLli15k1L2HBzWGG7zwxVEC5r2h3Sx1njYh/Jq3khIdSvDbiMmM+VAgMBAAGj -LDAqMAkGA1UdEwQCMAAwHQYDVR0OBBYEFGm8wrYF9mJweJ1vloDw19e0PUuIMA0G -CSqGSIb3DQEBCwUAA4IBAQBbEZ73weDphNIcmvN25v6NIfjBebqgm0/2grDFwmZe -Z1DibzRoVfoQ7WeUqbPS7SHUQ+KzIN1GdfHXhW9r6mmLbtzPv90Q/8zBcNv5HNZZ -YK+T2r9hoAWEY6nB1fiOJ4udkFMYfAi6LiSxave4IPWp/WIqd0IWtPtkPl+MmG41 -TfRom8TnO+o+VsjgDkY5Q1JDsNQKy1BrtxzIZyz7d1zYKTQ+HXZ4yeYJoVoc3k4y -6w9eX2zAUZ6Z3d4an6CLr6Hew9Dj2VX1vqCj1a5/VvHZVyVxyh4hg8sHYm7tZOJX -wN3B5GcKwbbFjaMVBLaMlP62OdGg7tCh61evWm+l06S0 ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks deleted file mode 100644 index ec482775bd055..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode-no-subjaltname.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt deleted file mode 100644 index 08c160bcea5ff..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt +++ /dev/null @@ -1,23 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID0zCCArugAwIBAgIJALi5bDfjMszLMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV -BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp -Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTUwOTIzMTg1MjU3WhcNMTkwOTIyMTg1MjU3 -WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV -BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA3rGZ1QbsW0+MuyrSLmMfDFKtLBkIFW8V0gRuurFg1PUKKNR1 -Mq2tMVwjjYETAU/UY0iKZOzjgvYPKhDTYBTte/WHR1ZK4CYVv7TQX/gtFQG/ge/c -7u0sLch9p7fbd+/HZiLS/rBEZDIohvgUvzvnA8+OIYnw4kuxKo/5iboAIS41klMg -/lATm8V71LMY68inht71/ZkQoAHKgcR9z4yNYvQ1WqKG8DG8KROXltll3sTrKbl5 -zJhn660es/1ZnR6nvwt6xnSTl/mNHMjkfv1bs4rJ/py3qPxicdoSIn/KyojUcgHV -F38fuAy2CQTdjVG5fWj9iz+mQvLm3+qsIYQdFwIDAQABo4G/MIG8MAkGA1UdEwQC -MAAwHQYDVR0OBBYEFEMMWLWQi/g83PzlHYqAVnty5L7HMIGPBgNVHREEgYcwgYSC -CWxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghds -b2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5s -b2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL -BQADggEBAMjGGXT8Nt1tbl2GkiKtmiuGE2Ej66YuZ37WSJViaRNDVHLlg87TCcHe -k2rdO+6sFqQbbzEfwQ05T7xGmVu7tm54HwKMRugoQ3wct0bQC5wEWYN+oMDvSyO6 -M28mZwWb4VtR2IRyWP+ve5DHwTM9mxWa6rBlGzsQqH6YkJpZojzqk/mQTug+Y8aE -mVoqRIPMHq9ob+S9qd5lp09+MtYpwPfTPx/NN+xMEooXWW/ARfpGhWPkg/FuCu4z -1tFmCqHgNcWirzMm3dQpF78muE9ng6OB2MXQwL4VgnVkxmlZNHbkR2v/t8MyZJxC -y4g6cTMM3S/UMt5/+aIB2JAuMKyuD+A= ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks deleted file mode 100644 index ebe6146124e8f..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 deleted file mode 100644 index c61c5d99574fa..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.p12 and /dev/null differ diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem deleted file mode 100644 index 5a67e1033440d..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,9D867F7E0C94D013 - -dVoVCjPeg1wgS7rVtOvGfQcrZyLkx393aWRnFq45tbjKBVuITtJ9vI7o4QXOV/15 -Gnb6WhXGIdWrzsxEAd46K6hIuNSISd4Emsx6c2Q5hTqWXXfexbOZBNfTtXtdJPnJ -1jAaikhtztLo3JSLTKNY5sNxd+XbaQyYVUWvueK6zOaIIMETvB+VPVFd9i1ROibk -Sgdtyj01KjkoalifqK/tA0CIYNKL0S6/eoK3UhAlpIprlpV+cnXa940C6bjLeJPt -PMAGGp5RrplxSgrSerw3I9DOWkHGtpqzIka3XneNUXJP8k4HUJ+aZkGH2ZILKS8d -4KMIb+KZSpHEGn+6uGccWLtZZmAjWJrDw56JbQtSHdRYLBRSOjLbTvQoPu/2Hpli -7HOxbotlvjptMunncq5aqK57SHA1dh0cwF7J3LUmGFJ67eoz+VV3b5qMn4MopSeI -mS16Ydd3nGpjSrln/elM0CQxqWfcOAXRZpDpFUQoXcBrLVzvz2DBl/0CrTRLhgzi -CO+5/IVcBWRlYpRNGgjjP7q0j6URID3jk5J06fYQXmBiwQT5j+GZqqzpMCJ9mIy2 -1O9SN1hebJnIcEU+E0njn/MGjlYdPywhaCy8pqElp6Q8TUEJpwLRFO/owCoBet/n -ZmCXUjfCGhc1pWHufFcDEQ6xMgEWWY/tdwCZeSU7EhErTjCbfupg+55A5fpDml0m -3wH4CFcuRjlqyx6Ywixm1ATeitDtJl5HQTw6b8OtEXwSgRmZ0eSqSRVk9QbVS7gu -IpQe09/Zimb5HzjZqZ3fdqHlcW4xax8hyJeyIvF5ZJ57eY8CBvu/wP2GDn26QnvF -xQqdfDbq1H4JmpwUHpbFwBoQK4Q6WFd1z4EA9bRQeo3H9PoqoOwMDjzajwLRF7b7 -q6tYH/n9PyHwdf1c4fFwgSmL1toXGfKlA9hjIaLsRSDD6srT5EdUk78bsnddwI51 -tu7C7P4JG+h1VdRNMNTlqtileWsIE7Nn2A1OkcUxZdF5mamENpDpJcHePLto6c8q -FKiwyFMsxhgsj6HK2HqO+UA4sX5Ni4oHwiPmb//EZLn045M5i1AN26KosJmb8++D -sgR5reWRy+UqJCTYblVg+7Dx++ggUnfxVyQEsWmw5r5f4KU5wXBkvoVMGtPNa9DE -n/uLtObD1qkNL38pRsr2OGRchYCgEoKGqEISBP4knfGXLOlWiW/246j9QzI97r1u -tvy7fKg28G7AUz9l6bpewsPHefBUeRQeieP9eJINaEpxkF/w2RpKDLpQjWxwDDOM -s+D0mrBMJve17AmJ8rMw6dIQPZYNZ88/jz1uQuUwQ2YlbmtZbCG81k9YMFGEU9XS -cyhJxj8hvYnt2PR5Z9/cJPyWOs0m/ufOeeQQ8SnU/lzmrQnpzUd2Z6p5i/B7LdRP -n1kX+l1qynuPnjvBz4nJQE0p6nzW8RyCDSniC9mtYtZmhgC8icqxgbvS7uEOBIYJ -NbK+0bEETTO34iY/JVTIqLOw3iQZYMeUpxpj6Phgx/oooxMTquMecPKNgeVtaBst -qjTNPX0ti1/HYpZqzYi8SV8YjHSJWCVMsZjKPr3W/HIcCKqYoIfgzi83Ha2KMQx6 ------END RSA PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt deleted file mode 100644 index f4c9a6a7aaca8..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.crt +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIB7zCCAZOgAwIBAgIEcmggOzAMBggqhkjOPQQDAgUAMCIxIDAeBgNVBAMTF0Vs -YXN0aWNzZWFyY2ggVGVzdCBOb2RlMB4XDTE4MDUxNzA5MzYxMFoXDTQ1MTAwMjA5 -MzYxMFowIjEgMB4GA1UEAxMXRWxhc3RpY3NlYXJjaCBUZXN0IE5vZGUwWTATBgcq -hkjOPQIBBggqhkjOPQMBBwNCAATuZRlXGn/ROcO7yFJJ50b20YvgV3U+FpRx0nx/ -yigWj6xiEMKnWbbUnM0mKF8c3GHGk5g8OXPnbK96uj6tpMB5o4G0MIGxMB0GA1Ud -DgQWBBRNAGO77mUhG6SQvIXQTbpcFwlf2TCBjwYDVR0RBIGHMIGEgglsb2NhbGhv -c3SCFWxvY2FsaG9zdC5sb2NhbGRvbWFpboIKbG9jYWxob3N0NIIXbG9jYWxob3N0 -NC5sb2NhbGRvbWFpbjSCCmxvY2FsaG9zdDaCF2xvY2FsaG9zdDYubG9jYWxkb21h -aW42hwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMAwGCCqGSM49BAMCBQADSAAwRQIg -Z3IvdmY5LFdbxoVSs6pV2tJ5+U833Chu0+ZzPo77IVUCIQDRx1FVitVuzBpqwhSW -+Zprt2RLPllC4s4BCApGDh8i1g== ------END CERTIFICATE----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.pem b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.pem deleted file mode 100644 index f8f29e47e0d35..0000000000000 --- a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode_ec.pem +++ /dev/null @@ -1,8 +0,0 @@ -Bag Attributes - friendlyName: testnode_ec - localKeyID: 54 69 6D 65 20 31 35 35 36 30 33 32 36 30 37 32 33 30 -Key Attributes: ------BEGIN PRIVATE KEY----- -MEECAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcEJzAlAgEBBCCxFwoKqfcGailZhuh0 -xEj3gssdjuEw6BasiC8+zhqHBA== ------END PRIVATE KEY----- diff --git a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks b/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks deleted file mode 100644 index f18b9288b10b8..0000000000000 Binary files a/x-pack/plugin/security/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/truststore-testnode-only.jks and /dev/null differ diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index b1f4da8c521ce..5aa7cc16bd7bf 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -130,19 +130,10 @@ public GeoShapeWithDocValuesFieldType(String name, boolean indexed, boolean hasD super(name, indexed, hasDocValues, meta); } - protected GeoShapeWithDocValuesFieldType(GeoShapeWithDocValuesFieldType ref) { - super(ref); - } - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { failIfNoDocValues(); return new AbstractLatLonShapeIndexFieldData.Builder(GeoShapeValuesSourceType.instance()); } - - @Override - public GeoShapeWithDocValuesFieldType clone() { - return new GeoShapeWithDocValuesFieldType(this); - } } public static final class TypeParser extends AbstractShapeGeometryFieldMapper.TypeParser { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 0a3e5ed0ec3e6..3278911094f90 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -131,19 +131,10 @@ public PointFieldType(String name, boolean indexed, boolean hasDocValues, Map { - - @Before - public void addModifiers() { - addModifier(t -> { - GeoShapeWithDocValuesFieldType copy = t.clone(); - if (copy.orientation() == ShapeBuilder.Orientation.RIGHT) { - copy.setOrientation(ShapeBuilder.Orientation.LEFT); - } else { - copy.setOrientation(ShapeBuilder.Orientation.RIGHT); - } - return copy; - }); - } - - @Override - protected GeoShapeWithDocValuesFieldType createDefaultFieldType(String name, Map meta) { - return new GeoShapeWithDocValuesFieldType(name, true, true, meta); - } -} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java deleted file mode 100644 index 9c851988e35f9..0000000000000 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldTypeTests.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.spatial.index.mapper; - -import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; - -import java.util.Map; - -public class PointFieldTypeTests extends FieldTypeTestCase { - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new PointFieldMapper.PointFieldType(name, true, true, meta); - } -} diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec index d4fd02ec782e6..fbc8fbae6d73a 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/agg.csv-spec @@ -1160,33 +1160,37 @@ GROUP BY gender ORDER BY gender; ; extendedStatsAggregateFunctionsWithScalars -schema::stddev_pop:d|sum_of_squares:d|var_pop:d|gender:s +schema::stddev_pop:d|stddev_samp:d|sum_of_squares:d|var_pop:d|var_samp:d|gender:s SELECT STDDEV_POP(CASE WHEN (salary / 2) > 10000 THEN (salary + 12345) * 1.2 ELSE (salary - 12345) * 2.7 END) AS "stddev_pop", +STDDEV_SAMP(CASE WHEN (salary / 2) > 10000 THEN (salary + 12345) * 1.2 ELSE (salary - 12345) * 2.7 END) AS "stddev_samp", SUM_OF_SQUARES(CASE WHEN (salary - 20) > 50000 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "sum_of_squares", VAR_POP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "var_pop", +VAR_SAMP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "var_samp", gender FROM test_emp GROUP BY gender ORDER BY gender; - stddev_pop | sum_of_squares | var_pop | gender -------------------+---------------------+--------------------+--------------- -16752.73244172422 |3.06310583829007E10 |3.460331137445282E8 |null -17427.462400181845|1.148127725047658E11 |3.1723426960671306E8|F -15702.798665784752|1.5882243113919238E11|2.529402043805585E8 |M + stddev_pop | stddev_samp | sum_of_squares | var_pop | var_samp | gender +------------------+------------------+---------------------+--------------------+--------------------+--------------- +16752.73244172422 |17658.930515747525|3.06310583829007E10 |3.460331137445282E8 |3.844812374939202E8 |null +17427.462400181845|17697.67172930331 |1.148127725047658E11 |3.1723426960671306E8|3.271478405319228E8 |F +15702.798665784752|15842.381843421828|1.5882243113919238E11|2.529402043805585E8 |2.5745699374449703E8|M ; extendedStatsAggregateFunctionsWithScalarAndSameArg -schema::stddev_pop:d|sum_of_squares:d|var_pop:d|gender:s +schema::stddev_pop:d|stddev_samp:d|sum_of_squares:d|var_pop:d|var_samp:d|gender:s SELECT STDDEV_POP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "stddev_pop", +STDDEV_SAMP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "stddev_samp", SUM_OF_SQUARES(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "sum_of_squares", VAR_POP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "var_pop", +VAR_SAMP(CASE WHEN (salary - 20) % 1000 > 200 THEN (salary * 1.2) - 1234 ELSE (salary - 20) * 0.93 END) AS "var_samp", gender FROM test_emp GROUP BY gender ORDER BY gender; - stddev_pop | sum_of_squares | var_pop | gender -------------------+---------------------+--------------------+--------------- -18601.965319409886|3.4461553130896095E10|3.460331137445282E8 |null -17811.071545718776|1.2151168881502939E11|3.1723426960671306E8|F -15904.093950318531|1.699198993070239E11 |2.529402043805585E8 |M + stddev_pop | stddev_samp | sum_of_squares | var_pop | var_samp | gender +------------------+------------------+---------------------+--------------------+--------------------+--------------- +18601.965319409886|19608.193121598946|3.4461553130896095E10|3.460331137445282E8 |3.844812374939202E8 |null +17811.071545718776|18087.228658142263|1.2151168881502939E11|3.1723426960671306E8|3.271478405319228E8 |F +15904.093950318531|16045.466454562704|1.699198993070239E11 |2.529402043805585E8 |2.5745699374449703E8|M ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec index 74314ccfb8448..4c89b2313eae2 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/command.csv-spec @@ -19,14 +19,16 @@ MAX |AGGREGATE MIN |AGGREGATE SUM |AGGREGATE KURTOSIS |AGGREGATE -MAD |AGGREGATE -PERCENTILE |AGGREGATE -PERCENTILE_RANK |AGGREGATE -SKEWNESS |AGGREGATE -STDDEV_POP |AGGREGATE -SUM_OF_SQUARES |AGGREGATE -VAR_POP |AGGREGATE -HISTOGRAM |GROUPING +MAD |AGGREGATE +PERCENTILE |AGGREGATE +PERCENTILE_RANK |AGGREGATE +SKEWNESS |AGGREGATE +STDDEV_POP |AGGREGATE +STDDEV_SAMP |AGGREGATE +SUM_OF_SQUARES |AGGREGATE +VAR_POP |AGGREGATE +VAR_SAMP |AGGREGATE +HISTOGRAM |GROUPING CASE |CONDITIONAL COALESCE |CONDITIONAL GREATEST |CONDITIONAL diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec index 20be529b399c7..244123606cf2c 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/docs.csv-spec @@ -215,13 +215,15 @@ MAX |AGGREGATE MIN |AGGREGATE SUM |AGGREGATE KURTOSIS |AGGREGATE -MAD |AGGREGATE -PERCENTILE |AGGREGATE -PERCENTILE_RANK |AGGREGATE -SKEWNESS |AGGREGATE -STDDEV_POP |AGGREGATE -SUM_OF_SQUARES |AGGREGATE -VAR_POP |AGGREGATE +MAD |AGGREGATE +PERCENTILE |AGGREGATE +PERCENTILE_RANK |AGGREGATE +SKEWNESS |AGGREGATE +STDDEV_POP |AGGREGATE +STDDEV_SAMP |AGGREGATE +SUM_OF_SQUARES |AGGREGATE +VAR_POP |AGGREGATE +VAR_SAMP |AGGREGATE HISTOGRAM |GROUPING CASE |CONDITIONAL COALESCE |CONDITIONAL @@ -1550,10 +1552,9 @@ SELECT MIN(salary) AS min, MAX(salary) AS max, SKEWNESS(salary) AS s FROM emp; aggStddevPop // tag::aggStddevPop -SELECT MIN(salary) AS min, MAX(salary) AS max, STDDEV_POP(salary) AS stddev - FROM emp; +SELECT MIN(salary) AS min, MAX(salary) AS max, STDDEV_POP(salary) AS stddev FROM emp; - min | max | stddev + min | max | stddev ---------------+---------------+------------------ 25324 |74999 |13765.125502787832 // end::aggStddevPop @@ -1570,6 +1571,27 @@ SELECT MIN(salary / 12.0) AS min, MAX(salary / 12.0) AS max, STDDEV_POP(salary / // end::aggStddevPopScalars ; +aggStddevSamp +// tag::aggStddevSamp +SELECT MIN(salary) AS min, MAX(salary) AS max, STDDEV_SAMP(salary) AS stddev FROM emp; + + min | max | stddev +---------------+---------------+------------------ +25324 |74999 |13834.471662090747 +// end::aggStddevSamp +; + +aggStddevSampScalars +schema::min:d|max:d|stddev:d +// tag::aggStddevSampScalars +SELECT MIN(salary / 12.0) AS min, MAX(salary / 12.0) AS max, STDDEV_SAMP(salary / 12.0) AS stddev FROM emp; + + min | max | stddev +------------------+-----------------+----------------- +2110.3333333333335|6249.916666666667|1152.872638507562 +// end::aggStddevSampScalars +; + aggSumOfSquares // tag::aggSumOfSquares @@ -1615,6 +1637,27 @@ SELECT MIN(salary / 24.0) AS min, MAX(salary / 24.0) AS max, VAR_POP(salary / 24 // end::aggVarPopScalars ; +aggVarSamp +// tag::aggVarSamp +SELECT MIN(salary) AS min, MAX(salary) AS max, VAR_SAMP(salary) AS varsamp FROM emp; + + min | max | varsamp +---------------+---------------+---------------- +25324 |74999 |1.913926061691E8 +// end::aggVarSamp +; + +aggVarSampScalars +schema::min:d|max:d|varsamp:d +// tag::aggVarSampScalars +SELECT MIN(salary / 24.0) AS min, MAX(salary / 24.0) AS max, VAR_SAMP(salary / 24.0) AS varsamp FROM emp; + + min | max | varsamp +------------------+------------------+---------------- +1055.1666666666667|3124.9583333333335|332278.830154847 +// end::aggVarSampScalars +; + /////////////////////////////// // // String diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java index 62a49da1efba6..3f552a0c25e1c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/SqlFunctionRegistry.java @@ -20,9 +20,11 @@ import org.elasticsearch.xpack.sql.expression.function.aggregate.PercentileRank; import org.elasticsearch.xpack.sql.expression.function.aggregate.Skewness; import org.elasticsearch.xpack.sql.expression.function.aggregate.StddevPop; +import org.elasticsearch.xpack.sql.expression.function.aggregate.StddevSamp; import org.elasticsearch.xpack.sql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.sql.expression.function.aggregate.SumOfSquares; import org.elasticsearch.xpack.sql.expression.function.aggregate.VarPop; +import org.elasticsearch.xpack.sql.expression.function.aggregate.VarSamp; import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.function.scalar.Database; @@ -143,8 +145,10 @@ private static FunctionDefinition[][] functions() { def(PercentileRank.class, PercentileRank::new, "PERCENTILE_RANK"), def(Skewness.class, Skewness::new, "SKEWNESS"), def(StddevPop.class, StddevPop::new, "STDDEV_POP"), + def(StddevSamp.class, StddevSamp::new, "STDDEV_SAMP"), def(SumOfSquares.class, SumOfSquares::new, "SUM_OF_SQUARES"), - def(VarPop.class, VarPop::new, "VAR_POP") + def(VarPop.class, VarPop::new, "VAR_POP"), + def(VarSamp.class, VarSamp::new, "VAR_SAMP") }, // histogram new FunctionDefinition[] { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/StddevSamp.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/StddevSamp.java new file mode 100644 index 0000000000000..8c568c19550c7 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/StddevSamp.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.aggregate; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; + +public class StddevSamp extends NumericAggregate implements ExtendedStatsEnclosed { + public StddevSamp(Source source, Expression field) { + super(source, field); + } + + @Override + public String innerName() { + return "std_deviation_sampling"; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 1) { + throw new IllegalArgumentException("expected [1] child but received [" + newChildren.size() + "]"); + } + return new StddevSamp(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StddevSamp::new, field()); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/VarSamp.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/VarSamp.java new file mode 100644 index 0000000000000..9f2954e4f0c10 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/VarSamp.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.aggregate; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.List; + +public class VarSamp extends NumericAggregate implements ExtendedStatsEnclosed { + public VarSamp(Source source, Expression field) { + super(source, field); + } + + @Override + public String innerName() { + return "variance_sampling"; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 1) { + throw new IllegalArgumentException("expected [1] child but received [" + newChildren.size() + "]"); + } + return new VarSamp(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, VarSamp::new, field()); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index f92526d5814ca..0233083ac83d2 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.execution.search.FieldExtraction; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expression; @@ -68,6 +69,7 @@ import org.elasticsearch.xpack.sql.planner.QueryTranslator.QueryTranslation; import org.elasticsearch.xpack.sql.querydsl.agg.AggFilter; import org.elasticsearch.xpack.sql.querydsl.agg.GroupByDateHistogram; +import org.elasticsearch.xpack.sql.querydsl.container.MetricAggRef; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.types.SqlTypesTests; import org.elasticsearch.xpack.sql.util.DateUtils; @@ -1895,6 +1897,33 @@ public void testTopHitsAggregationWithTwoArgs() { } } + public void testExtendedStatsAggsStddevAndVar() { + final Map metricToAgg = Map.of( + "STDDEV_POP", "std_deviation", + "STDDEV_SAMP", "std_deviation_sampling", + "VAR_POP", "variance", + "VAR_SAMP", "variance_sampling" + ); + for (String funcName: metricToAgg.keySet()) { + PhysicalPlan p = optimizeAndPlan("SELECT " + funcName + "(int) FROM test"); + assertEquals(EsQueryExec.class, p.getClass()); + EsQueryExec eqe = (EsQueryExec) p; + assertEquals(1, eqe.output().size()); + + assertEquals(funcName + "(int)", eqe.output().get(0).qualifiedName()); + assertEquals(DOUBLE, eqe.output().get(0).dataType()); + + FieldExtraction fe = eqe.queryContainer().fields().get(0).v1(); + assertEquals(MetricAggRef.class, fe.getClass()); + assertEquals(((MetricAggRef) fe).property(), metricToAgg.get(funcName)); + + String aggName = eqe.queryContainer().aggs().asAggBuilder().getSubAggregations().iterator().next().getName(); + assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""), + endsWith("\"aggregations\":{\"" + aggName + "\":{\"extended_stats\":{\"field\":\"int\",\"sigma\":2.0}}}}}")); + } + + } + public void testGlobalCountInImplicitGroupByForcesTrackHits() { PhysicalPlan p = optimizeAndPlan("SELECT COUNT(*) FROM test"); assertEquals(EsQueryExec.class, p.getClass()); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_data_frame_analytics.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_data_frame_analytics.json new file mode 100644 index 0000000000000..9086a655833ca --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_data_frame_analytics.json @@ -0,0 +1,29 @@ +{ + "ml.update_data_frame_analytics":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/update-dfanalytics.html", + "description":"Updates certain properties of a data frame analytics job." + }, + "stability":"experimental", + "url":{ + "paths":[ + { + "path":"/_ml/data_frame/analytics/{id}/_update", + "methods":[ + "POST" + ], + "parts":{ + "id":{ + "type":"string", + "description":"The ID of the data frame analytics to update" + } + } + } + ] + }, + "body":{ + "description":"The data frame analytics settings to update", + "required":true + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/histogram.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/histogram.yml index d1fb34c8b8149..168f66e498f02 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/histogram.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/histogram.yml @@ -46,4 +46,24 @@ setup: - match: { aggregations.histo_avg.value: 0.3125} +--- +"Histogram over histograms": + - do: + search: + index: "test" + body: + size: 0 + aggs: + histo: + histogram: + field: "latency" + interval: 0.3 + + + - match: { hits.total.value: 2 } + - length: { aggregations.histo.buckets: 2 } + - match: { aggregations.histo.buckets.0.key: 0.0 } + - match: { aggregations.histo.buckets.0.doc_count: 20 } + - match: { aggregations.histo.buckets.1.key: 0.3 } + - match: { aggregations.histo.buckets.1.doc_count: 60 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 457d94c74680c..7dbca2c62f5da 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -409,3 +409,76 @@ indices.delete_data_stream: name: simple-data-stream1 - is_true: acknowledged + +--- +"Verify data stream resolvability in rollup search": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + partition: + type: keyword + price: + type: integer + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + index: + index: simple-data-stream1 + body: { partition: a, price: 1, '@timestamp': '2020-12-12T00:00:00.000Z' } + + - do: + index: + index: simple-data-stream1 + body: { partition: a, price: 2, '@timestamp': '2020-12-12T01:00:00.000Z' } + + - do: + index: + index: simple-data-stream1 + body: { partition: b, price: 3, '@timestamp': '2020-12-12T01:00:00.000Z' } + + - do: + indices.refresh: + index: simple-data-stream1 + + - do: + rollup.rollup_search: + index: "simple-data-stream1" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "@timestamp" + calendar_interval: "1h" + time_zone: "UTC" + + - length: { aggregations.histo.buckets: 2 } + - match: { aggregations.histo.buckets.0.key_as_string: "2020-12-12T00:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 1 } + - match: { aggregations.histo.buckets.1.key_as_string: "2020-12-12T01:00:00.000Z" } + - match: { aggregations.histo.buckets.1.doc_count: 2 } + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml index ebe78698c88ef..83d8d92b7641c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml @@ -81,6 +81,6 @@ setup: - match: $body: | /^ id \s+ t \s+ s \s+ p \s+ source_index \s+ dest_index \n - (dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-c \n)+ + (dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0,inference:0 \s+ index-source \s+ index-dest-c \n)+ (dfa\-outlier\-detection\-job \s+ outlier_detection \s+ stopped \s+ reindexing:0,loading_data:0,computing_outliers:0,writing_results:0 \s+ index-source \s+ index-dest-od \n)+ - (dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-r \n)+ $/ + (dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0,inference:0 \s+ index-source \s+ index-dest-r \n)+ $/ diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml index 43d20cc02b33a..f75f61f37c284 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_crud.yml @@ -2100,3 +2100,57 @@ setup: "includes": ["excluded"] } } + +--- +"Test update given all updatable settings": + + - do: + ml.put_data_frame_analytics: + id: "update-test-job" + body: > + { + "source": { + "index": "index-source" + }, + "dest": { + "index": "index-dest" + }, + "analysis": {"outlier_detection":{}}, + "description": "before update", + "model_memory_limit": "20mb", + "allow_lazy_start": false, + "max_num_threads": 1 + } + - match: { id: "update-test-job" } + - match: { description: "before update" } + - match: { model_memory_limit: "20mb" } + - match: { allow_lazy_start: false } + - match: { max_num_threads: 1 } + + - do: + ml.update_data_frame_analytics: + id: "update-test-job" + body: > + { + "description": "after update", + "model_memory_limit": "30mb", + "allow_lazy_start": true, + "max_num_threads": 2 + } + - match: { id: "update-test-job" } + - match: { description: "after update" } + - match: { model_memory_limit: "30mb" } + - match: { allow_lazy_start: true } + - match: { max_num_threads: 2 } + +--- +"Test update given missing analytics": + + - do: + catch: missing + ml.update_data_frame_analytics: + id: "missing-analytics" + body: > + { + "description": "blah" + } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml index 2dca2483aaf2d..a4dc3ceecca09 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/privileges/11_builtin.yml @@ -16,4 +16,4 @@ setup: # I would much prefer we could just check that specific entries are in the array, but we don't have # an assertion for that - length: { "cluster" : 36 } - - length: { "index" : 18 } + - length: { "index" : 19 } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml index 7c0c780dc5bfd..971e383124ee0 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/50_data_streams.yml @@ -15,7 +15,17 @@ setup: body: > { "indices": [ - { "names": ["simple*"], "privileges": ["read", "write", "view_index_metadata"] } + { "names": ["simple*"], "privileges": ["read", "write", "create_index", "view_index_metadata", "delete_index"] } + ] + } + + - do: + security.put_role: + name: "data_stream_role2" + body: > + { + "indices": [ + { "names": ["matches_none"], "privileges": ["read", "write", "create_index", "view_index_metadata", "delete_index"] } ] } @@ -26,16 +36,26 @@ setup: { "password" : "x-pack-test-password", "roles" : [ "data_stream_role" ], - "full_name" : "user with privileges on data streams but not backing indices" + "full_name" : "user with privileges on some data streams" + } + + - do: + security.put_user: + username: "no_authz_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "data_stream_role2" ], + "full_name" : "user with privileges on no data streams" } - do: allowed_warnings: - - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + - "index template [my-template1] has index patterns [s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" indices.put_index_template: name: my-template1 body: - index_patterns: [simple-data-stream1] + index_patterns: [s*] template: mappings: properties: @@ -51,16 +71,27 @@ teardown: username: "test_user" ignore: 404 + - do: + security.delete_user: + username: "test_user2" + ignore: 404 + - do: security.delete_role: name: "data_stream_role" ignore: 404 + - do: + security.delete_role: + name: "data_stream_role2" + ignore: 404 + --- "Test backing indices inherit parent data stream privileges": - skip: version: " - 7.99.99" reason: "change to 7.8.99 after backport" + features: ["headers"] - do: # superuser indices.create_data_stream: @@ -122,6 +153,7 @@ teardown: - skip: version: " - 7.99.99" reason: "change to 7.8.99 after backport" + features: ["headers"] - do: # superuser indices.create_data_stream: @@ -147,3 +179,191 @@ teardown: indices.delete_data_stream: name: simple-data-stream1 - is_true: acknowledged + +--- +"Test that create data stream is limited to authorized namespace": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: # superuser + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.create_data_stream: + name: outside_of_namespace + +--- +"Test that get data stream is limited to authorized namespace": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: # superuser + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: # superuser + indices.create_data_stream: + name: s-outside-of-authed-namespace + - is_true: acknowledged + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get_data_stream: + name: simple-data-stream1 + + - length: { data_streams: 1 } + - match: { data_streams.0.name: simple-data-stream1 } + + - do: # superuser + indices.get_data_stream: + name: "*" + + # superuser should be authorized for both data streams + - length: { data_streams: 2 } + - match: { data_streams.0.name: s-outside-of-authed-namespace } + - match: { data_streams.1.name: simple-data-stream1 } + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get_data_stream: + name: "*" + + # test_user should be authorized for only one data stream + - length: { data_streams: 1 } + - match: { data_streams.0.name: simple-data-stream1 } + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.get_data_stream: + name: outside_of_namespace + + - do: + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + indices.get_data_stream: {} + + # no_authz_user should not be authorized for any data streams + - length: { data_streams: 0 } + + - do: # superuser + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: # superuser + indices.delete_data_stream: + name: s-outside-of-authed-namespace + - is_true: acknowledged + +--- +"Test that delete data stream is limited to authorized namespace": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: # superuser + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: # superuser + indices.create_data_stream: + name: s-outside-of-authed-namespace + - is_true: acknowledged + + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + indices.delete_data_stream: + name: s-outside-of-authed-namespace + + - do: + catch: forbidden + headers: { Authorization: "Basic bm9fYXV0aHpfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # no_authz_user + indices.delete_data_stream: + name: simple-data-stream1 + + - do: # superuser + indices.delete_data_stream: + name: s-outside-of-authed-namespace + - is_true: acknowledged + +--- +"auto_configure privilege permits auto-create of data streams": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + features: ["headers", "allowed_warnings"] + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple*] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + security.put_role: + name: "data_stream_role" + body: > + { + "indices": [ + { "names": ["simple-allows-auto-configure"], "privileges": ["create_doc", "auto_configure"] }, + { "names": ["simple-data-stream1"], "privileges": ["create_doc"] } + ] + } + + - do: + security.clear_cached_roles: + name: "data_stream_role" + + # should succeed because test_user is authorized for auto_configure on simple-allows-auto-configure + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + index: + index: simple-allows-auto-configure + id: 1 + op_type: create + body: { foo: bar, "@timestamp": "2020-12-12" } + + # should fail because test_user is not authorized for auto_configure on simple-data-stream1 + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + index: + index: simple-data-stream1 + id: 1 + op_type: create + body: { foo: bar, "@timestamp": "2020-12-12" } + + - do: # superuser + indices.delete_data_stream: + name: simple-allows-auto-configure + - is_true: acknowledged diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/55_auto_configure.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/55_auto_configure.yml new file mode 100644 index 0000000000000..4ee0e6619230c --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/security/authz/55_auto_configure.yml @@ -0,0 +1,82 @@ +--- +setup: + - skip: + features: ["headers", "allowed_warnings"] + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "ingest_role" + body: > + { + "indices": [ + { "names": ["index-auto-configure"], "privileges": ["create_doc", "auto_configure"] }, + { "names": ["index-limited"], "privileges": ["create_doc"] } + ] + } + + - do: + security.put_user: + username: "test_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "ingest_role" ], + "full_name" : "user with privileges on data streams but not backing indices" + } + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [index*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [index*] + template: + mappings: + properties: + '@timestamp': + type: date + +--- +teardown: + - do: + security.delete_user: + username: "test_user" + ignore: 404 + + - do: + security.delete_role: + name: "ingest_role" + ignore: 404 + +--- +"auto_configure privilege permits auto-create of indices": + - skip: + version: " - 7.99.99" + reason: "change to 7.8.99 after backport" + features: ["headers", "allowed_warnings"] + + # should succeed because test_user is authorized for auto_configure on index-auto-configure + - do: + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + index: + index: index-auto-configure + id: 1 + op_type: create + body: { foo: bar, "@timestamp": "2020-12-12" } + + # should fail because test_user is not authorized for auto_configure on index-limited + - do: + catch: forbidden + headers: { Authorization: "Basic dGVzdF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # test_user + index: + index: index-limited + id: 1 + op_type: create + body: { "@timestamp": "2020-12-12" } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 7ee3393de3aab..7e45286e28737 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -104,15 +104,6 @@ public DenseVectorFieldType(String name, int dims, Map meta) { this.dims = dims; } - protected DenseVectorFieldType(DenseVectorFieldType ref) { - super(ref); - this.dims = ref.dims; - } - - public DenseVectorFieldType clone() { - return new DenseVectorFieldType(this); - } - int dims() { return dims; } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 66904e26eafb3..51cafa01ecdcc 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -87,14 +87,6 @@ public SparseVectorFieldType(String name, Map meta) { super(name, false, false, TextSearchInfo.NONE, meta); } - protected SparseVectorFieldType(SparseVectorFieldType ref) { - super(ref); - } - - public SparseVectorFieldType clone() { - return new SparseVectorFieldType(this); - } - @Override public String typeName() { return CONTENT_TYPE; diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java deleted file mode 100644 index 8a82208fb36cc..0000000000000 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - -package org.elasticsearch.xpack.vectors.mapper; - -import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; - -import java.util.Map; - -public class DenseVectorFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new DenseVectorFieldMapper.DenseVectorFieldType(name, 4, meta); - } -} diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java index 7420e5174e5b6..403ec3f38e2cb 100644 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java @@ -11,17 +11,11 @@ import org.elasticsearch.index.mapper.MappedFieldType; import java.util.Collections; -import java.util.Map; -public class SparseVectorFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new SparseVectorFieldMapper.SparseVectorFieldType(name, meta); - } +public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { - MappedFieldType fieldType = createDefaultFieldType("field", Collections.emptyMap()); + MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index")); } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 8ee10677ca5da..8f0ef4b13f705 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -217,16 +217,6 @@ public WildcardFieldType(String name, FieldType fieldType, Map m setIndexAnalyzer(WILDCARD_ANALYZER); } - protected WildcardFieldType(WildcardFieldType ref) { - super(ref); - } - - public WildcardFieldType clone() { - WildcardFieldType result = new WildcardFieldType(this); - return result; - } - - @Override public Query wildcardQuery(String wildcardPattern, RewriteMethod method, QueryShardContext context) { @@ -849,12 +839,12 @@ public Query fuzzyQuery( public String typeName() { return CONTENT_TYPE; } - + @Override public String familyTypeName() { return KeywordFieldMapper.CONTENT_TYPE; } - + @Override public Query existsQuery(QueryShardContext context) { diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java deleted file mode 100644 index 8f2b5b00b156a..0000000000000 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldTypeTests.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - -package org.elasticsearch.xpack.wildcard.mapper; - -import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.index.mapper.MappedFieldType; - -import java.util.Map; - -public class WildcardFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType(String name, Map meta) { - return new WildcardFieldMapper.WildcardFieldType(name, WildcardFieldMapper.Defaults.FIELD_TYPE, meta); - } -} diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 80e442d98f590..29fc766b888bc 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -5,17 +5,24 @@ */ package org.elasticsearch.xpack.restart; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ObjectPath; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -35,6 +42,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; @@ -42,6 +50,8 @@ import java.util.stream.Collectors; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.upgrades.FullClusterRestartIT.assertNumHits; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -627,4 +637,56 @@ public void testFrozenIndexAfterRestarted() throws Exception { assertNoFileBasedRecovery(index, n -> true); } } + + @SuppressWarnings("unchecked") + public void testDataStreams() throws Exception { + assumeTrue("no data streams in versions before " + Version.V_7_9_0, getOldClusterVersion().onOrAfter(Version.V_7_9_0)); + if (isRunningAgainstOldCluster()) { + String mapping = "{\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + " }"; + Template template = new Template(null, new CompressedXContent(mapping), null); + createComposableTemplate(client(), "dst", "ds", template); + + Request indexRequest = new Request("POST", "/ds/_doc/1?op_type=create&refresh"); + XContentBuilder builder = JsonXContent.contentBuilder().startObject() + .field("f", "v") + .field("@timestamp", new Date()) + .endObject(); + indexRequest.setJsonEntity(Strings.toString(builder)); + assertOK(client().performRequest(indexRequest)); + } + + Request getDataStream = new Request("GET", "/_data_stream/ds"); + Response response = client().performRequest(getDataStream); + assertOK(response); + List dataStreams = (List) entityAsMap(response).get("data_streams"); + assertEquals(1, dataStreams.size()); + Map ds = (Map) dataStreams.get(0); + List> indices = (List>) ds.get("indices"); + assertEquals("ds", ds.get("name")); + assertEquals(1, indices.size()); + assertEquals(DataStream.getDefaultBackingIndexName("ds", 1), indices.get(0).get("index_name")); + assertNumHits("ds", 1, 1); + } + + private static void createComposableTemplate(RestClient client, String templateName, String indexPattern, Template template) + throws IOException { + XContentBuilder builder = jsonBuilder(); + template.toXContent(builder, ToXContent.EMPTY_PARAMS); + StringEntity templateJSON = new StringEntity( + String.format(Locale.ROOT, "{\n" + + " \"index_patterns\": \"%s\",\n" + + " \"data_stream\": { \"timestamp_field\": \"@timestamp\" },\n" + + " \"template\": %s\n" + + "}", indexPattern, Strings.toString(builder)), + ContentType.APPLICATION_JSON); + Request createIndexTemplateRequest = new Request("PUT", "_index_template/" + templateName); + createIndexTemplateRequest.setEntity(templateJSON); + client.performRequest(createIndexTemplateRequest); + } } diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml similarity index 60% rename from qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml rename to x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml index a5c3bcf111cbc..f777187efdec4 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/100_resolve_index.yml @@ -21,30 +21,30 @@ - match: {indices.2.attributes.0: hidden} - match: {indices.2.attributes.1: open} - match: {indices.2.data_stream: simple-data-stream2} - - match: {indices.3.name: my_remote_cluster:ccs_duel_index} + - match: {indices.3.name: my_remote_cluster:.security-7} - match: {indices.3.attributes.0: open} - - match: {indices.4.name: my_remote_cluster:ccs_duel_index_empty} - - match: {indices.4.attributes.0: open} - - match: {indices.5.name: my_remote_cluster:ccs_duel_index_err} + - match: {indices.4.name: my_remote_cluster:closed_index} + - match: {indices.4.aliases.0: aliased_closed_index} + - match: {indices.4.attributes.0: closed} + - match: {indices.5.name: my_remote_cluster:field_caps_index_1} - match: {indices.5.attributes.0: open} - - match: {indices.6.name: my_remote_cluster:closed_index} - - match: {indices.6.aliases.0: aliased_closed_index} - - match: {indices.6.attributes.0: closed} - - match: {indices.7.name: my_remote_cluster:field_caps_empty_index} + - match: {indices.6.name: my_remote_cluster:field_caps_index_3} + - match: {indices.6.attributes.0: open} + - match: {indices.7.name: my_remote_cluster:secured_via_alias} - match: {indices.7.attributes.0: open} - - match: {indices.8.name: my_remote_cluster:field_caps_index_1} + - match: {indices.8.name: my_remote_cluster:single_doc_index} - match: {indices.8.attributes.0: open} - - match: {indices.9.name: my_remote_cluster:field_caps_index_3} + - match: {indices.9.name: my_remote_cluster:test_index} + - match: {indices.9.aliases.0: aliased_test_index} - match: {indices.9.attributes.0: open} - - match: {indices.10.name: my_remote_cluster:single_doc_index} - - match: {indices.10.attributes.0: open} - - match: {indices.11.name: my_remote_cluster:test_index} - - match: {indices.11.aliases.0: aliased_test_index} - - match: {indices.11.attributes.0: open} - - match: {aliases.0.name: my_remote_cluster:aliased_closed_index} - - match: {aliases.0.indices.0: closed_index} - - match: {aliases.1.name: my_remote_cluster:aliased_test_index} - - match: {aliases.1.indices.0: test_index} + - match: {aliases.0.name: my_remote_cluster:.security} + - match: {aliases.0.indices.0: .security-7} + - match: {aliases.1.name: my_remote_cluster:aliased_closed_index} + - match: {aliases.1.indices.0: closed_index} + - match: {aliases.2.name: my_remote_cluster:aliased_test_index} + - match: {aliases.2.indices.0: test_index} + - match: {aliases.3.name: my_remote_cluster:secure_alias} + - match: {aliases.3.indices.0: secured_via_alias} - match: {data_streams.0.name: my_remote_cluster:simple-data-stream1} - match: {data_streams.0.backing_indices.0: .ds-simple-data-stream1-000001} - match: {data_streams.0.timestamp_field: "@timestamp"} diff --git a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml index 4cd5a8e434473..01cb27150846c 100644 --- a/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml +++ b/x-pack/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml @@ -53,6 +53,60 @@ setup: } --- "Index data and search on the remote cluster": + - skip: + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + - do: + allowed_warnings: + - "index template [my-template2] has index patterns [simple-data-stream2] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation" + indices.put_index_template: + name: my-template2 + body: + index_patterns: [simple-data-stream2] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: + timestamp_field: '@timestamp' + + - do: + indices.create_data_stream: + name: simple-data-stream1 + + - do: + indices.create_data_stream: + name: simple-data-stream2 + + - do: + indices.rollover: + alias: "simple-data-stream2" + + - do: + indices.create: + index: closed_index + body: + aliases: + aliased_closed_index: {} + + - do: + indices.close: + index: closed_index - do: indices.create: diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 943287170626d..d7a947bfa42e6 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -48,6 +48,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.token.timeout', '60m' + setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.audit.enabled', 'true' setting 'xpack.security.transport.ssl.key', 'testnode.pem' setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index 7e88c327127a3..7aee77a482b91 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -51,6 +51,11 @@ protected boolean preserveILMPoliciesUponCompletion() { return true; } + @Override + protected boolean preserveDataStreamsUponCompletion() { + return true; + } + enum ClusterType { OLD, MIXED, diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java new file mode 100644 index 0000000000000..b41229ec3ffba --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.upgrades; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.Booleans; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.elasticsearch.upgrades.IndexingIT.assertCount; + +public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase { + + public void testDataStreams() throws IOException { + if (CLUSTER_TYPE == ClusterType.OLD) { + String requestBody = "{\n" + + " \"index_patterns\":[\"logs-*\"],\n" + + " \"template\": {\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"@timestamp\": {\n" + + " \"type\": \"date\"\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"data_stream\":{\n" + + " \"timestamp_field\":\"@timestamp\"" + + " }\n" + + " }"; + Request request = new Request("PUT", "/_index_template/1"); + request.setJsonEntity(requestBody); + client().performRequest(request); + + StringBuilder b = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + b.append("{\"create\":{\"_index\":\"").append("logs-foobar").append("\"}}\n"); + b.append("{\"@timestamp\":\"2020-12-12\",\"test\":\"value").append(i).append("\"}\n"); + } + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + bulk.addParameter("filter_path", "errors"); + bulk.setJsonEntity(b.toString()); + Response response = client().performRequest(bulk); + assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + } else if (CLUSTER_TYPE == ClusterType.MIXED) { + Request rolloverRequest = new Request("POST", "/logs-foobar/_rollover"); + client().performRequest(rolloverRequest); + + Request index = new Request("POST", "/logs-foobar/_doc"); + index.addParameter("refresh", "true"); + index.addParameter("filter_path", "_index"); + if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) { + index.setJsonEntity("{\"@timestamp\":\"2020-12-12\",\"test\":\"value1000\"}"); + Response response = client().performRequest(index); + assertEquals("{\"_index\":\".ds-logs-foobar-000002\"}", + EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + } else { + index.setJsonEntity("{\"@timestamp\":\"2020-12-12\",\"test\":\"value1001\"}"); + Response response = client().performRequest(index); + assertEquals("{\"_index\":\".ds-logs-foobar-000003\"}", + EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + } + } + + final int expectedCount; + if (CLUSTER_TYPE.equals(ClusterType.OLD)) { + expectedCount = 1000; + } else if (CLUSTER_TYPE.equals(ClusterType.MIXED)) { + if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) { + expectedCount = 1001; + } else { + expectedCount = 1002; + } + } else if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) { + expectedCount = 1002; + } else { + throw new AssertionError("unexpected cluster type"); + } + assertCount("logs-foobar", expectedCount); + } + +} diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index ad5a8b68a5ea8..174d808ae6a8f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -117,7 +117,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio client().performRequest(bulk); } - private void assertCount(String index, int count) throws IOException { + static void assertCount(String index, int count) throws IOException { Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchTestIndexRequest.addParameter("filter_path", "hits.total"); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/120_api_key_auth.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/120_api_key_auth.yml new file mode 100644 index 0000000000000..c317402ddfa9a --- /dev/null +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/120_api_key_auth.yml @@ -0,0 +1,25 @@ +--- +"Test API key authentication will work in a mixed cluster": + + - skip: + features: headers + version: " - 7.99.99" + reason: "7.9.0 backport pending" + + - do: + security.create_api_key: + body: > + { + "name": "my-api-key" + } + - match: { name: "my-api-key" } + - is_true: id + - is_true: api_key + - transform_and_set: { login_creds: "#base64EncodeCredentials(id,api_key)" } + + - do: + headers: + Authorization: ApiKey ${login_creds} + nodes.info: {} + - match: { _nodes.failed: 0 } + diff --git a/x-pack/qa/third-party/active-directory/build.gradle b/x-pack/qa/third-party/active-directory/build.gradle index 6f4f4d9239b78..c85c392c09c85 100644 --- a/x-pack/qa/third-party/active-directory/build.gradle +++ b/x-pack/qa/third-party/active-directory/build.gradle @@ -10,6 +10,7 @@ testFixtures.useFixture ":x-pack:test:smb-fixture" // add test resources from security, so tests can use example certs processTestResources { + from(project(xpackModule('core')).sourceSets.test.resources.srcDirs) from(project(xpackModule('security')).sourceSets.test.resources.srcDirs) }