Compat/mapping

.ci/os.ps1

@@ -1,3 +1,5 @@
+param($GradleTasks='destructiveDistroTest')
+
 If (-NOT ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator))
 {
     # Relaunch as an elevated process:
@@ -25,7 +27,6 @@ Remove-Item -Recurse -Force \tmp -ErrorAction Ignore
 New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
-# TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain destructiveDistroTest
+& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
 
 exit $LastExitCode

benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/bucket/terms/LongKeyedBucketOrdsBenchmark.java

@@ -0,0 +1,172 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.benchmark.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
+import org.elasticsearch.search.aggregations.CardinalityUpperBound;
+import org.elasticsearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.LongKeyedBucketOrds;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+@Fork(2)
+@Warmup(iterations = 10)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@OperationsPerInvocation(1_000_000)
+@State(Scope.Benchmark)
+public class LongKeyedBucketOrdsBenchmark {
+    private static final long LIMIT = 1_000_000;
+    /**
+     * The number of distinct values to add to the buckets.
+     */
+    private static final long DISTINCT_VALUES = 10;
+    /**
+     * The number of buckets to create in the {@link #multiBucket} case.
+     * <p>
+     * If this is not relatively prime to {@link #DISTINCT_VALUES} then the
+     * values won't be scattered evenly across the buckets.
+     */
+    private static final long DISTINCT_BUCKETS = 21;
+
+    private final PageCacheRecycler recycler = new PageCacheRecycler(Settings.EMPTY);
+    private final BigArrays bigArrays = new BigArrays(recycler, null, "REQUEST");
+
+    /**
+     * Force loading all of the implementations just for extra paranoia's sake.
+     * We really don't want the JVM to be able to eliminate one of them just
+     * because we don't use it in the particular benchmark. That is totally a
+     * thing it'd do. It is sneaky.
+     */
+    @Setup
+    public void forceLoadClasses(Blackhole bh) {
+        bh.consume(LongKeyedBucketOrds.FromSingle.class);
+        bh.consume(LongKeyedBucketOrds.FromMany.class);
+    }
+
+    /**
+     * Emulates a way that we do <strong>not</strong> use {@link LongKeyedBucketOrds}
+     * because it is not needed.
+     */
+    @Benchmark
+    public void singleBucketIntoSingleImmutableMonmorphicInvocation(Blackhole bh) {
+        try (LongKeyedBucketOrds.FromSingle ords = new LongKeyedBucketOrds.FromSingle(bigArrays)) {
+            for (long i = 0; i < LIMIT; i++) {
+                ords.add(0, i % DISTINCT_VALUES);
+            }
+            bh.consume(ords);
+        }
+    }
+
+    /**
+     * Emulates the way that most aggregations use {@link LongKeyedBucketOrds}.
+     */
+    @Benchmark
+    public void singleBucketIntoSingleImmutableBimorphicInvocation(Blackhole bh) {
+        try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE)) {
+            for (long i = 0; i < LIMIT; i++) {
+                ords.add(0, i % DISTINCT_VALUES);
+            }
+            bh.consume(ords);
+        }
+    }
+
+    /**
+     * Emulates the way that {@link AutoDateHistogramAggregationBuilder} uses {@link LongKeyedBucketOrds}.
+     */
+    @Benchmark
+    public void singleBucketIntoSingleMutableMonmorphicInvocation(Blackhole bh) {
+        LongKeyedBucketOrds.FromSingle ords = new LongKeyedBucketOrds.FromSingle(bigArrays);
+        for (long i = 0; i < LIMIT; i++) {
+            if (i % 100_000 == 0) {
+                ords.close();
+                bh.consume(ords);
+                ords = new LongKeyedBucketOrds.FromSingle(bigArrays);
+            }
+            ords.add(0, i % DISTINCT_VALUES);
+        }
+        bh.consume(ords);
+        ords.close();
+    }
+
+    /**
+     * Emulates a way that we do <strong>not</strong> use {@link LongKeyedBucketOrds}
+     * because it is significantly slower than the
+     * {@link #singleBucketIntoSingleMutableMonmorphicInvocation monomorphic invocation}.
+     */
+    @Benchmark
+    public void singleBucketIntoSingleMutableBimorphicInvocation(Blackhole bh) {
+        LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE);
+        for (long i = 0; i < LIMIT; i++) {
+            if (i % 100_000 == 0) {
+                ords.close();
+                bh.consume(ords);
+                ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.ONE);
+            }
+            ords.add(0, i % DISTINCT_VALUES);
+
+        }
+        bh.consume(ords);
+        ords.close();
+    }
+
+    /**
+     * Emulates an aggregation that collects from a single bucket "by accident".
+     * This can happen if an aggregation is under, say, a {@code terms}
+     * aggregation and there is only a single value for that term in the index.
+     */
+    @Benchmark
+    public void singleBucketIntoMulti(Blackhole bh) {
+        try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.MANY)) {
+            for (long i = 0; i < LIMIT; i++) {
+                ords.add(0, i % DISTINCT_VALUES);
+            }
+            bh.consume(ords);
+        }
+    }
+
+    /**
+     * Emulates an aggregation that collects from many buckets.
+     */
+    @Benchmark
+    public void multiBucket(Blackhole bh) {
+        try (LongKeyedBucketOrds ords = LongKeyedBucketOrds.build(bigArrays, CardinalityUpperBound.MANY)) {
+            for (long i = 0; i < LIMIT; i++) {
+                ords.add(i % DISTINCT_BUCKETS, i % DISTINCT_VALUES);
+            }
+            bh.consume(ords);
+        }
+    }
+}

build.gradle

@@ -174,8 +174,8 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
-final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/59076" /* place a PR link here when committing bwc changes */
+boolean bwc_tests_enabled = true
+final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -24,6 +24,7 @@ import groovy.transform.CompileStatic
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.ElasticsearchJavaPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.elasticsearch.gradle.RepositoriesSetupPlugin
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
@@ -59,9 +60,9 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
         project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(TestClustersPlugin)
+        project.pluginManager.apply(RepositoriesSetupPlugin)
 
         project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
-        ElasticsearchJavaPlugin.configureRepositories(project)
         ElasticsearchJavaPlugin.configureTestTasks(project)
         ElasticsearchJavaPlugin.configureInputNormalization(project)
         ElasticsearchJavaPlugin.configureCompile(project)

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java

@@ -36,9 +36,6 @@
 import org.gradle.api.artifacts.ModuleDependency;
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.ResolutionStrategy;
-import org.gradle.api.artifacts.dsl.RepositoryHandler;
-import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
-import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
 import org.gradle.api.execution.TaskActionListener;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.BasePlugin;
@@ -59,16 +56,10 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.util.Arrays;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
 import static org.elasticsearch.gradle.util.Util.toStringable;
@@ -83,11 +74,11 @@ public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         // apply global test task failure listener
         project.getRootProject().getPluginManager().apply(TestFailureReportingPlugin.class);
-
+        // common repositories setup
+        project.getPluginManager().apply(RepositoriesSetupPlugin.class);
         project.getPluginManager().apply(JavaLibraryPlugin.class);
 
         configureConfigurations(project);
-        configureRepositories(project);
         configureCompile(project);
         configureInputNormalization(project);
         configureTestTasks(project);
@@ -149,77 +140,6 @@ public static void configureConfigurations(Project project) {
         disableTransitiveDeps.accept(JavaPlugin.TEST_IMPLEMENTATION_CONFIGURATION_NAME);
     }
 
-    private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
-
-    /**
-     * Adds repositories used by ES dependencies
-     */
-    public static void configureRepositories(Project project) {
-        // ensure all repositories use secure urls
-        // TODO: remove this with gradle 7.0, which no longer allows insecure urls
-        project.getRepositories().all(repository -> {
-            if (repository instanceof MavenArtifactRepository) {
-                final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
-                assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
-                for (URI uri : maven.getArtifactUrls()) {
-                    assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
-                }
-            } else if (repository instanceof IvyArtifactRepository) {
-                final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
-                assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
-            }
-        });
-        RepositoryHandler repos = project.getRepositories();
-        if (System.getProperty("repos.mavenLocal") != null) {
-            // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
-            // useful for development ie. bwc tests where we install stuff in the local repository
-            // such that we don't have to pass hardcoded files to gradle
-            repos.mavenLocal();
-        }
-        repos.jcenter();
-
-        String luceneVersion = VersionProperties.getLucene();
-        if (luceneVersion.contains("-snapshot")) {
-            // extract the revision number from the version with a regex matcher
-            Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
-            if (matcher.find() == false) {
-                throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
-            }
-            String revision = matcher.group(1);
-            MavenArtifactRepository luceneRepo = repos.maven(repo -> {
-                repo.setName("lucene-snapshots");
-                repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
-            });
-            repos.exclusiveContent(exclusiveRepo -> {
-                exclusiveRepo.filter(
-                    descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
-                );
-                exclusiveRepo.forRepositories(luceneRepo);
-            });
-        }
-    }
-
-    private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
-
-    private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
-        if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
-            String url;
-            try {
-                url = uri.toURL().toString();
-            } catch (MalformedURLException e) {
-                throw new IllegalStateException(e);
-            }
-            final String message = String.format(
-                Locale.ROOT,
-                "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
-                repositoryName,
-                projectPath,
-                url
-            );
-            throw new GradleException(message);
-        }
-    }
-
     /**
      * Adds compiler settings to the project
      */

buildSrc/src/main/java/org/elasticsearch/gradle/LazyPropertyList.java

@@ -218,6 +218,7 @@ private class PropertyListEntry<T> {
             this.normalization = normalization;
         }
 
+        @Input
         public PropertyNormalization getNormalization() {
             return normalization;
         }