diff --git a/distribution/build.gradle b/distribution/build.gradle index 584b80a8c67de..e58484da87fa9 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -422,14 +422,16 @@ testClusters { if (System.getProperty('run.distribution', 'default') == 'default') { String licenseType = System.getProperty("run.license_type", "basic") if (licenseType == 'trial') { - setting 'xpack.ml.enabled', 'true' + setting 'xpack.ml.enabled', 'false' setting 'xpack.graph.enabled', 'true' setting 'xpack.watcher.enabled', 'true' setting 'xpack.license.self_generated.type', 'trial' } else if (licenseType != 'basic') { throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].") } - setting 'xpack.security.enabled', 'true' + setting 'xpack.ml.enabled', 'false' + + setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'true' setting 'xpack.sql.enabled', 'true' setting 'xpack.rollup.enabled', 'true' diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index e014fedbe978c..02b996a2c9c6a 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -45,6 +45,7 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased assemble.enabled = false File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}") + project.ext.checkoutDir = checkoutDir final String remote = System.getProperty("bwc.remote", "elastic") diff --git a/distribution/src/config/elasticsearch.yml b/distribution/src/config/elasticsearch.yml index a87d7f70825ed..ec19ad55ec889 100644 --- a/distribution/src/config/elasticsearch.yml +++ b/distribution/src/config/elasticsearch.yml @@ -86,3 +86,4 @@ ${path.logs} # Require explicit names when deleting indices: # #action.destructive_requires_name: true +xpack.ml.enabled: false diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index dcfefa1ea1cc5..f814a44be6cf9 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -64,3 +64,4 @@ ${error.file} ## GC logging -Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m +-Des.rest.url_plus_as_space=true diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java index 9efe568955f39..98a7c1ceca94f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java @@ -55,12 +55,14 @@ public static class Entry { /** A parser capability of parser the entry's class. */ private final ContextParser parser; + private final boolean requiresCompatible; /** Creates a new entry which can be stored by the registry. */ public Entry(Class categoryClass, ParseField name, CheckedFunction parser) { this.categoryClass = Objects.requireNonNull(categoryClass); this.name = Objects.requireNonNull(name); this.parser = Objects.requireNonNull((p, c) -> parser.apply(p)); + this.requiresCompatible = false; } /** * Creates a new entry which can be stored by the registry. @@ -70,6 +72,18 @@ public Entry(Class categoryClass, ParseField name, ContextParser Entry(Class categoryClass, ParseField name, ContextParser parser, boolean requiresCompatible) { + this.categoryClass = Objects.requireNonNull(categoryClass); + this.name = Objects.requireNonNull(name); + this.parser = Objects.requireNonNull(parser); + this.requiresCompatible = requiresCompatible; + } + + public boolean isCompatibleWith(XContentParser parser) { + return requiresCompatible == false || parser.isCompatible(); } } @@ -128,7 +142,7 @@ public T parseNamedObject(Class categoryClass, String name, XContentPa throw new XContentParseException("unknown named object category [" + categoryClass.getName() + "]"); } Entry entry = parsers.get(name); - if (entry == null) { + if (entry == null || entry.isCompatibleWith(parser) == false) { throw new NamedObjectNotFoundException(parser.getTokenLocation(), "unknown field [" + name + "]", parsers.keySet()); } if (false == entry.name.match(name, parser.getDeprecationHandler())) { @@ -139,5 +153,4 @@ public T parseNamedObject(Class categoryClass, String name, XContentPa } return categoryClass.cast(entry.parser.parse(parser, context)); } - } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 20fde0891b6f8..f71e5102a458e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -19,28 +19,12 @@ package org.elasticsearch.common.xcontent; -import java.io.ByteArrayOutputStream; -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.file.Path; import java.time.ZonedDateTime; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.ServiceLoader; -import java.util.Set; +import java.util.*; import java.util.function.Function; /** @@ -137,6 +121,7 @@ public static XContentBuilder builder(XContent xContent, Set includes, S DATE_TRANSFORMERS = Collections.unmodifiableMap(dateTransformers); } + @FunctionalInterface public interface Writer { void write(XContentBuilder builder, Object value) throws IOException; @@ -165,6 +150,8 @@ public interface HumanReadableTransformer { */ private boolean humanReadable = false; + private byte compatibleVersion; + /** * Constructs a new builder using the provided XContent and an OutputStream. Make sure * to call {@link #close()} when the builder is done with. @@ -998,6 +985,16 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce return this; } + public XContentBuilder compatibleVersion(byte compatibleVersion){ + this.compatibleVersion = compatibleVersion; + return this; + } + + public byte getCompatibleMajorVersion() { + return compatibleVersion; + } + + @Override public void flush() throws IOException { generator.flush(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 82a663bd9dc5d..78168beeca524 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -266,4 +266,13 @@ Map map( * The callback to notify when parsing encounters a deprecated field. */ DeprecationHandler getDeprecationHandler(); + + + default XContentParser withIsCompatible(boolean isCompatible) { + return this; + } + + default boolean isCompatible() { + return false; + } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java index 606284f046244..89f9c46f38ce4 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java @@ -26,6 +26,8 @@ import java.util.Locale; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * The content type of {@link org.elasticsearch.common.xcontent.XContent}. @@ -142,7 +144,9 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) { * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type} * HTTP header. This method will return {@code null} if no match is found */ - public static XContentType fromMediaType(String mediaType) { + public static XContentType fromMediaType(String mediaTypeHeaderValue) { + String mediaType = parseMediaType(mediaTypeHeaderValue); + final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT); for (XContentType type : values()) { if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) { @@ -157,6 +161,29 @@ public static XContentType fromMediaType(String mediaType) { return null; } + static Pattern pattern = Pattern.compile("application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?"); + + public static String parseMediaType(String mediaType) { + if (mediaType != null) { + Matcher matcher = pattern.matcher(mediaType); + if (matcher.find()) { + return "application/"+matcher.group(2); + } + } + + return mediaType; + } + + public static String parseVersion(String mediaType){ + if(mediaType != null){ + Matcher matcher = pattern.matcher(mediaType); + if (matcher.find() && "vnd.elasticsearch+".equals(matcher.group(1))) { + + return matcher.group(4); + } + } + return null; + } private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) { return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) || stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") || diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 264af205e488b..1081f8b60e40d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -57,12 +57,24 @@ private static void checkCoerceString(boolean coerce, Class cl private final NamedXContentRegistry xContentRegistry; private final DeprecationHandler deprecationHandler; + private boolean isCompatible = false; public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler) { this.xContentRegistry = xContentRegistry; this.deprecationHandler = deprecationHandler; } + @Override + public XContentParser withIsCompatible(boolean isCompatible) { + this.isCompatible = isCompatible; + return this; + } + + @Override + public boolean isCompatible() { + return isCompatible; + } + // The 3rd party parsers we rely on are known to silently truncate fractions: see // http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue() // If this behaviour is flagged as undesirable and any truncation occurs diff --git a/qa/os/bats/plugins/25_tar_plugins.bats b/qa/os/bats/plugins/25_tar_plugins.bats deleted file mode 120000 index 552c404a3d6c6..0000000000000 --- a/qa/os/bats/plugins/25_tar_plugins.bats +++ /dev/null @@ -1 +0,0 @@ -module_and_plugin_test_cases.bash \ No newline at end of file diff --git a/qa/os/bats/plugins/25_tar_plugins.bats b/qa/os/bats/plugins/25_tar_plugins.bats new file mode 100644 index 0000000000000..552c404a3d6c6 --- /dev/null +++ b/qa/os/bats/plugins/25_tar_plugins.bats @@ -0,0 +1 @@ +module_and_plugin_test_cases.bash \ No newline at end of file diff --git a/qa/os/bats/plugins/50_modules_and_plugins.bats b/qa/os/bats/plugins/50_modules_and_plugins.bats deleted file mode 120000 index 552c404a3d6c6..0000000000000 --- a/qa/os/bats/plugins/50_modules_and_plugins.bats +++ /dev/null @@ -1 +0,0 @@ -module_and_plugin_test_cases.bash \ No newline at end of file diff --git a/qa/os/bats/plugins/50_modules_and_plugins.bats b/qa/os/bats/plugins/50_modules_and_plugins.bats new file mode 100644 index 0000000000000..552c404a3d6c6 --- /dev/null +++ b/qa/os/bats/plugins/50_modules_and_plugins.bats @@ -0,0 +1 @@ +module_and_plugin_test_cases.bash \ No newline at end of file diff --git a/qa/rest-compat-tests/build.gradle b/qa/rest-compat-tests/build.gradle new file mode 100644 index 0000000000000..fd933f78cf6d4 --- /dev/null +++ b/qa/rest-compat-tests/build.gradle @@ -0,0 +1,70 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.rest-test' + +def restCompatAPIRoot = new File(new File(new File(project.buildDir, 'resources'), 'test'), 'restCompatAPIRoot') +def restCompatTestRoot = new File(new File(new File(project.buildDir, 'resources'), 'test'), 'restCompatTestRoot'); + +task copyAPIFromBwc(type: Copy) { + dependsOn ':distribution:bwc:minor:checkoutBwcBranch' + from (project(':distribution:bwc:minor').checkoutDir){ + include 'rest-api-spec/**/rest-api-spec/api/**/*.json' + include 'modules/**/rest-api-spec/api/**/*.json' + include 'plugins/**/rest-api-spec/api/**/*.json' + } + into restCompatAPIRoot + includeEmptyDirs = false +} + +task copyTestsFromBwc(type: Copy) { + dependsOn ':distribution:bwc:minor:checkoutBwcBranch' + from (project(':distribution:bwc:minor').checkoutDir){ + include 'rest-api-spec/**/rest-api-spec/test/**/*.yml' + include 'modules/**/rest-api-spec/test/**/*.yml' + include 'plugins/**/rest-api-spec/test/**/*.yml' + } + into restCompatTestRoot + includeEmptyDirs = false +} + +task copyFromBwc { + dependsOn copyAPIFromBwc + dependsOn copyTestsFromBwc +} + +integTest.runner { + dependsOn copyFromBwc + systemProperty 'restCompatAPIRoot', restCompatAPIRoot + systemProperty 'restCompatTestRoot', restCompatTestRoot +} + +dependencies { + testCompile project(':test:framework') +} + +testClusters.integTest { + //reindex module + setting 'reindex.remote.whitelist', '127.0.0.1:*' + +} + + diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/AbstractRestCompatYamlTestSuite.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/AbstractRestCompatYamlTestSuite.java new file mode 100644 index 0000000000000..d756aac2e666c --- /dev/null +++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/AbstractRestCompatYamlTestSuite.java @@ -0,0 +1,165 @@ +package org.elasticsearch.rest.compat; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.StreamSupport; + +/** + * ./gradlew :qa:rest-compat-tests:integTestRunner --tests "org.elasticsearch.rest.compat.RestCompatSpecYamlTestSuiteIT.test {yaml=index/*}" + */ +class AbstractRestCompatYamlTestSuite extends ESClientYamlSuiteTestCase { + + private static final Logger logger = LogManager.getLogger(AbstractRestCompatYamlTestSuite.class); + + //TODO: maybe ? support new command line flag to enforce tests from a specific module, plugin, or core, add this to the reproduce line + //-Dtests.rest.suite.compat.parent=[plugin-name | module-name] + + enum SkipReason { + REQUIRES_ADDITIONAL_SETUP("requires additional setup"); + + private final String display; + + public String getDisplay() { + return display; + } + + SkipReason(String display) { + this.display = display; + } + } + + //TODO: figure out the additional steps and try to get this list down to zero, + static final Map SKIP = Map.of( + "/plugins/repository-gcs", SkipReason.REQUIRES_ADDITIONAL_SETUP, + "/plugins/discovery-gce", SkipReason.REQUIRES_ADDITIONAL_SETUP, + "/plugins/repository-azure", SkipReason.REQUIRES_ADDITIONAL_SETUP, + "/plugins/discovery-ec2", SkipReason.REQUIRES_ADDITIONAL_SETUP, + "/plugins/repository-s3", SkipReason.REQUIRES_ADDITIONAL_SETUP, + "/modules/repository-url", SkipReason.REQUIRES_ADDITIONAL_SETUP + ); + + protected AbstractRestCompatYamlTestSuite(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + protected String getApiSpecRelativePath() { + String[] splits = System.getProperty("restCompatAPIRoot").split(File.separator); + return "/" + splits[splits.length - 1]; + } + + /** + *

+ * Finds the tests from the prior version. Gradle checked out the prior version and copied the REST tests to "restCompatTestRoot" dir. + * This method finds those old tests to feed into {@link ESClientYamlSuiteTestCase} so that the old REST tests are run against the + * current server with compatibility mode flag sent. Some the old tests may not parse correctly into executable tests and are listed + * locally in the SKIP variable. + *

+ * Returns a list of parsed and executable tests. Once the tests are parsed and represented by Java code some manipulation is possible. + *

+ * Specially the following manipulations occur: + *
    + *
  • Inject the compatibly header
    • + *
  • Ignore warnings (these will be tested elsewhere)
    • + *
+ * TODO: something about overrides + */ + static Iterable getPriorVersionTests(String source) throws Exception { + assert "plugins".equals(source) || "modules".equals(source) || "rest-api-spec".equals(source); + String[] splits = System.getProperty("restCompatTestRoot").split(File.separator); + String classPathRoot = "/" + splits[splits.length - 1] + "/" + source; + Set testsClassPaths = new HashSet<>(); + Path absoluteTestsPath = PathUtils.get(ESClientYamlSuiteTestCase.class.getResource(classPathRoot).toURI()); + if (Files.isDirectory(absoluteTestsPath)) { + Files.walk(absoluteTestsPath).forEach(file -> { + if (file.toString().endsWith(".yml")) { + String testClassPath = Path.of(classPathRoot).resolve(absoluteTestsPath.relativize(file.getParent().getParent())).toString().replace("\\", "/"); + Optional> match = + SKIP.entrySet().stream().filter(e -> file.toString().contains(e.getKey())).findFirst(); + if (match.isPresent()) { + //TODO: make this better or just get rid of it ! - as it is wrong , it is missing the API + logger.info("Skipping test {}/{} [{}]", source, file.getFileName().toString().replace(".yml", ""), match.get().getValue().getDisplay()); + } else { + testsClassPaths.add(testClassPath); + } + } + }); + } + Set testOverrides = getTestsOverrides(); + List tests = new ArrayList<>(100); + + for (String testsClassPath : testsClassPaths) { + + Iterable candidates = ESClientYamlSuiteTestCase.createParameters(ExecutableSection.XCONTENT_REGISTRY, testsClassPath); + StreamSupport.stream(candidates.spliterator(), false) + .flatMap(Arrays::stream).map(o -> (ClientYamlTestCandidate) o) + .forEach(testCandidate -> { + List testCandidates = new ArrayList<>(100); + + if (testOverrides.contains(testCandidate) == false) { + testCandidates.add(testCandidate); + //disable checking for warning headers, we know that many of the tests will have deprecation and compatibility warnings. + //the deprecation and compatibility warnings should be explicitly tested via the REST tests from this version. + testCandidate.getTestSection().getExecutableSections().stream().filter(s -> s instanceof DoSection).forEach(ds -> { + DoSection doSection = (DoSection) ds; + doSection.checkWarningHeaders(false); + //TODO: use real header + // add the compatibility header + String compatibleHeader = createCompatibleHeader(doSection); + doSection.getApiCallSection().addHeaders(Map.of(Version.COMPATIBLE_HEADER,compatibleHeader)); + }); + } else { + //TODO: use a logger + System.out.println("* * * Skipping test [" + testCandidates + "]"); + } + if (testCandidates.isEmpty() == false) { + tests.add(testCandidates.toArray()); + } + + }); + } + //TODO: what happens when a single test is requested via the modules, but the module is overriden ? (maybe keep all of the skipped tests locally and output them with a helpful hint here if empty + return tests; + } + + private static String createCompatibleHeader(DoSection doSection) { + String headerValue = doSection.getApiCallSection().getHeaders().get(Version.COMPATIBLE_HEADER); + //header value in a form of application/json + if(headerValue!=null) { + String[] split = headerValue.split("/"); + String type = split[0]; + String subType = split[1]; + return type + "/" + "vnd.elasticsearch" + "+" + subType + ";" + "compatible-with" + Version.COMPATIBLE_VERSION; + }else{ + return "application/vnd.elasticsearch+json;compatible-with=" + Version.COMPATIBLE_VERSION; + + } + } + + private static Set getTestsOverrides() throws Exception { + Iterable candidates = ESClientYamlSuiteTestCase.createParameters(); + Set testOverrides = new HashSet<>(100); + StreamSupport.stream(candidates.spliterator(), false) + .flatMap(Arrays::stream).forEach(o -> testOverrides.add((ClientYamlTestCandidate) o)); + return testOverrides; + } +} diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatModulesYamlTestSuiteIT.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatModulesYamlTestSuiteIT.java new file mode 100644 index 0000000000000..df87eb24487a3 --- /dev/null +++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatModulesYamlTestSuiteIT.java @@ -0,0 +1,20 @@ +package org.elasticsearch.rest.compat; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; + +/** + * Runs the prior version's elasticsearch/modules REST tests against a cluster of the current (this) version. + */ +public class RestCompatModulesYamlTestSuiteIT extends AbstractRestCompatYamlTestSuite { + + public RestCompatModulesYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return AbstractRestCompatYamlTestSuite.getPriorVersionTests("modules"); + } +} diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatPluginYamlTestSuiteIT.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatPluginYamlTestSuiteIT.java new file mode 100644 index 0000000000000..f7c8c66780654 --- /dev/null +++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatPluginYamlTestSuiteIT.java @@ -0,0 +1,22 @@ +package org.elasticsearch.rest.compat; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; + +/** + * Runs the prior version's elasticsearch/plugin REST tests against a cluster of the current (this) version. + */ +public class RestCompatPluginYamlTestSuiteIT extends AbstractRestCompatYamlTestSuite { + + //TODO: ensure plugins are installed. + + public RestCompatPluginYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return AbstractRestCompatYamlTestSuite.getPriorVersionTests("plugins"); + } +} diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatSpecYamlTestSuiteIT.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatSpecYamlTestSuiteIT.java new file mode 100644 index 0000000000000..4946039698b45 --- /dev/null +++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatSpecYamlTestSuiteIT.java @@ -0,0 +1,20 @@ +package org.elasticsearch.rest.compat; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; + +/** + * Runs the prior version's elasticsearch/rest-api-spec REST tests against a cluster of the current (this) version. + */ +public class RestCompatSpecYamlTestSuiteIT extends AbstractRestCompatYamlTestSuite { + + public RestCompatSpecYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return AbstractRestCompatYamlTestSuite.getPriorVersionTests("rest-api-spec"); + } +} diff --git a/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatYamlTestSuiteIT.java b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatYamlTestSuiteIT.java new file mode 100644 index 0000000000000..ac8b907b9675d --- /dev/null +++ b/qa/rest-compat-tests/src/test/java/org/elasticsearch/rest/compat/RestCompatYamlTestSuiteIT.java @@ -0,0 +1,24 @@ +package org.elasticsearch.rest.compat; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import java.io.File; + +/** + * Runs any additional REST compatibility tests from this project's resources. + */ +public class RestCompatYamlTestSuiteIT extends AbstractRestCompatYamlTestSuite { + + public RestCompatYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + +} diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/120_grok.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/120_grok.yml new file mode 100644 index 0000000000000..6f90c7e8f9cec --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/120_grok.yml @@ -0,0 +1,6 @@ +# Overriding since when the number of grok patterns are different between versions +"Test Grok Patterns Retrieval": + - do: + ingest.processor_grok: {} + - length: { patterns: 303 } # <-- changed + - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" } diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml new file mode 100644 index 0000000000000..fe7c124e60a35 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml @@ -0,0 +1,43 @@ +# Overriding since the type of the exception changed between versions +"Test Pipeline Processor with Circular Pipelines": + - do: + ingest.put_pipeline: + id: "outer" + body: > + { + "description" : "outer pipeline", + "processors" : [ + { + "pipeline" : { + "name": "inner" + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.put_pipeline: + id: "inner" + body: > + { + "description" : "inner pipeline", + "processors" : [ + { + "pipeline" : { + "name": "outer" + } + } + ] + } + - match: { acknowledged: true } + + - do: + catch: /illegal_state_exception/ + index: + index: test + id: 1 + pipeline: "outer" + body: {} + - match: { error.root_cause.0.type: "ingest_processor_exception" } # <-- changed + - match: { error.root_cause.0.reason: "java.lang.IllegalStateException: Cycle detected for pipeline: outer" } diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/15_update.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/15_update.yml new file mode 100644 index 0000000000000..d3ce6ee467f26 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/15_update.yml @@ -0,0 +1,24 @@ +# Overriding this test since the underlying root caused changed between versions +"Update Script with script error": + - do: + index: + index: test_1 + id: 2 + body: + foo: bar + count: 1 + + - do: + catch: bad_request + update: + index: test_1 + id: 2 + body: + script: + lang: painless + source: "ctx._source.ctx = ctx" + params: { bar: 'xxx' } + + - match: { error.root_cause.0.type: "remote_transport_exception" } # <-- changed + - match: { error.type: "illegal_argument_exception" } + - match: { error.reason: "Iterable object is self-referencing itself" } diff --git a/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/71_context_api.yml b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/71_context_api.yml new file mode 100644 index 0000000000000..38cdca952dc11 --- /dev/null +++ b/qa/rest-compat-tests/src/test/resources/rest-api-spec/test/painless/71_context_api.yml @@ -0,0 +1,8 @@ +# Overriding since when the compat REST tests are run from the qa project, all of the module painless contexts are on the classpath and the +# number and thus placement in the array are different then when running directly from the lang-painless REST tests +"Action to list contexts": + - do: + scripts_painless_context: {} + - match: { contexts.0: aggregation_selector} + - match: { contexts.22: terms_set} # <-- changed + diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index cdfd49cb042c7..52a70cfbcb532 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -78,6 +78,10 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_5_0); public static final Version CURRENT = V_8_0_0; + public static final String COMPATIBLE_HEADER = "Accept"; + public static final String COMPATIBLE_PARAMS_KEY = "Compatible-With"; + public static final String COMPATIBLE_VERSION = "7"; + private static final ImmutableOpenIntMap idToVersion; static { diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 62c265bc17a78..e6a3f94f1ce45 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -626,6 +626,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestDeleteIndexTemplateAction()); registerHandler.accept(new RestPutMappingAction()); + registerHandler.accept(new RestPutMappingAction.CompatibleRestPutMappingAction()); registerHandler.accept(new RestGetMappingAction()); registerHandler.accept(new RestGetFieldMappingAction()); @@ -640,7 +641,13 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestIndexAction()); registerHandler.accept(new CreateHandler()); registerHandler.accept(new AutoIdHandler(clusterService)); + registerHandler.accept(new RestIndexAction.CompatibleRestIndexAction()); + registerHandler.accept(new RestIndexAction.CompatibleCreateHandler()); + registerHandler.accept(new RestIndexAction.CompatibleAutoIdHandler(clusterService)); + registerHandler.accept(new RestGetAction()); + registerHandler.accept(new RestGetAction.CompatibleRestGetAction()); + registerHandler.accept(new RestGetSourceAction()); registerHandler.accept(new RestMultiGetAction(settings)); registerHandler.accept(new RestDeleteAction()); @@ -651,6 +658,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestUpdateAction()); registerHandler.accept(new RestSearchAction()); + registerHandler.accept(new RestSearchAction.CompatibleRestSearchAction()); registerHandler.accept(new RestSearchScrollAction()); registerHandler.accept(new RestClearScrollAction()); registerHandler.accept(new RestMultiSearchAction(settings)); diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java index 34f9939a366e1..ac2b8ac8e1855 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -53,6 +54,9 @@ */ public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContentObject { + private static final String TYPE_FIELD_NAME = "_type"; + private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); + private static final String _SHARDS = "_shards"; private static final String _INDEX = "_index"; private static final String _ID = "_id"; @@ -276,6 +280,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) { + builder.field(TYPE_FIELD_NAME, SINGLE_MAPPING_TYPE); + } innerToXContent(builder, params); builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/common/path/PathTrie.java b/server/src/main/java/org/elasticsearch/common/path/PathTrie.java index 4f4c029ecb7d9..c674ed005517d 100644 --- a/server/src/main/java/org/elasticsearch/common/path/PathTrie.java +++ b/server/src/main/java/org/elasticsearch/common/path/PathTrie.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.path; +import org.elasticsearch.Version; + import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; @@ -59,6 +61,11 @@ enum TrieMatchingMode { public interface Decoder { String decode(String value); + + + default String decodeForCompatible(String compatibleVersion, String value) { + return decode(value); + } } private final Decoder decoder; @@ -269,7 +276,7 @@ public T retrieve(String[] path, int index, Map params, TrieMatc private void put(Map params, TrieNode node, String value) { if (params != null && node.isNamedWildcard()) { - params.put(node.namedWildcard(), decoder.decode(value)); + params.put(node.namedWildcard(), decoder.decodeForCompatible(params.get(Version.COMPATIBLE_PARAMS_KEY), value)); } } diff --git a/server/src/main/java/org/elasticsearch/index/get/GetResult.java b/server/src/main/java/org/elasticsearch/index/get/GetResult.java index 949cc1969e7ae..a9aaac884c893 100644 --- a/server/src/main/java/org/elasticsearch/index/get/GetResult.java +++ b/server/src/main/java/org/elasticsearch/index/get/GetResult.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; @@ -38,11 +39,7 @@ import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -295,11 +292,16 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params } return builder; } + private static final String TYPE_FIELD_NAME = "_type"; + private static final Text SINGLE_MAPPING_TYPE = new Text(MapperService.SINGLE_MAPPING_NAME); @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(_INDEX, index); + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) { + builder.field(TYPE_FIELD_NAME, SINGLE_MAPPING_TYPE); + } builder.field(_ID, id); if (isExists()) { if (version != -1) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index db9b4306f2f5c..cf020467c49fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.mapper.MetadataFieldMapper.TypeParser; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -221,6 +222,14 @@ public T metadataMapper(Class type) { return mapping.metadataMapper(type); } + public IndexFieldMapper indexMapper() { + return metadataMapper(IndexFieldMapper.class); + } + + public TypeFieldMapper typeMapper() { + return metadataMapper(TypeFieldMapper.class); + } + public SourceFieldMapper sourceMapper() { return metadataMapper(SourceFieldMapper.class); } @@ -237,6 +246,10 @@ public IndexFieldMapper IndexFieldMapper() { return metadataMapper(IndexFieldMapper.class); } + public Query typeFilter(QueryShardContext context) { + return typeMapper().fieldType().termQuery(type, context); + } + public boolean hasNestedObjects() { return hasNestedObjects; } diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java new file mode 100644 index 0000000000000..07923f199f8b0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.query; + +import org.apache.logging.log4j.LogManager; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.DocumentMapper; + +import java.io.IOException; +import java.util.Objects; + +public class TypeQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "type"; + + private static final ParseField VALUE_FIELD = new ParseField("value"); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(TypeQueryBuilder.class)); + static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Type queries are deprecated, " + + "prefer to filter on a field instead."; + + private final String type; + + public TypeQueryBuilder(String type) { + if (type == null) { + throw new IllegalArgumentException("[type] cannot be null"); + } + this.type = type; + } + +// @Override +// public boolean requiresCompatibility() { +// return true; +// } + + /** + * Read from a stream. + */ + public TypeQueryBuilder(StreamInput in) throws IOException { + super(in); + type = in.readString(); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(type); + } + + public String type() { + return type; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(VALUE_FIELD.getPreferredName(), type); + printBoostAndQueryName(builder); + builder.endObject(); + } + + public static TypeQueryBuilder fromXContent(XContentParser parser) throws IOException { + String type = null; + String queryName = null; + float boost = AbstractQueryBuilder.DEFAULT_BOOST; + String currentFieldName = null; + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + queryName = parser.text(); + } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + boost = parser.floatValue(); + } else if (VALUE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + type = parser.text(); + } else { + throw new ParsingException(parser.getTokenLocation(), + "[" + TypeQueryBuilder.NAME + "] filter doesn't support [" + currentFieldName + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), + "[" + TypeQueryBuilder.NAME + "] filter doesn't support [" + currentFieldName + "]"); + } + } + + if (type == null) { + throw new ParsingException(parser.getTokenLocation(), + "[" + TypeQueryBuilder.NAME + "] filter needs to be provided with a value for the type"); + } + return new TypeQueryBuilder(type) + .boost(boost) + .queryName(queryName); + } + + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected Query doToQuery(QueryShardContext context) throws IOException { + deprecationLogger.deprecatedAndMaybeLog("type_query", TYPES_DEPRECATION_MESSAGE); + //LUCENE 4 UPGRADE document mapper should use bytesref as well? +// DocumentMapper documentMapper = context.getMapperService().documentMapper(type); +// if (documentMapper == null) { + // no type means no documents + return new MatchNoDocsQuery(); +// } else {f +// return documentMapper.typeFilter(context); +// } + } + + @Override + protected int doHashCode() { + return Objects.hash(type); + } + + @Override + protected boolean doEquals(TypeQueryBuilder other) { + return Objects.equals(type, other.type); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java index 467f1d969e8be..e6eeabaec1249 100644 --- a/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java +++ b/server/src/main/java/org/elasticsearch/rest/AbstractRestChannel.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.rest; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; @@ -127,6 +128,9 @@ public XContentBuilder newBuilder(@Nullable XContentType requestContentType, @Nu } builder.humanReadable(human); + + String compatibleVersion = request.param(Version.COMPATIBLE_PARAMS_KEY); + builder.compatibleVersion(compatibleVersion == null ? -1 : Byte.parseByte(compatibleVersion)); return builder; } diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index f2fd12bfa79c2..97c49b33cd072 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -56,6 +56,14 @@ public abstract class BaseRestHandler implements RestHandler { private final LongAdder usageCount = new LongAdder(); + /** + * Parameter that controls whether certain REST apis should include type names in their requests or responses. + * Note: Support for this parameter will be removed after the transition period to typeless APIs. + */ + public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; + public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; + + public final long getUsageCount() { return usageCount.sum(); } diff --git a/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java new file mode 100644 index 0000000000000..b920ea1fcbb29 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/CompatibleHandlers.java @@ -0,0 +1,58 @@ +package org.elasticsearch.rest; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.index.mapper.MapperService; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.UnaryOperator; + +public class CompatibleHandlers { + private static final Logger logger = LogManager.getLogger(CompatibleHandlers.class); + + public static Consumer consumeParameterIncludeType(DeprecationLogger deprecationLogger) { + final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " + + "index requests is deprecated. The parameter will be removed in the next major version."; + + return r -> { + if(r.hasParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER)){ + deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE); + r.param(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER); + } + }; + } + + public static Consumer consumeParameterType(DeprecationLogger deprecationLogger) { + String TYPES_DEPRECATION_MESSAGE = "[types removal] Using type as a path parameter is deprecated."; + + return r -> { + deprecationLogger.deprecatedAndMaybeLog("create_index_with_types", TYPES_DEPRECATION_MESSAGE); + r.param("type"); + }; + } + + public static boolean isV7Compatible(ToXContent.Params params) { + String param = params.param(Version.COMPATIBLE_PARAMS_KEY); + return Version.COMPATIBLE_VERSION.equals(param); + } + + public static Map replaceTypeWithDoc(Map mappings){ + Map newSource = new HashMap<>(); + + String typeName = mappings.keySet().iterator().next(); + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + + newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings)); + return typedMappings; + } + +} diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 6a16007465c53..20135594d5a2e 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -280,7 +280,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel if (restHeader.isMultiValueAllowed() == false && distinctHeaderValues.size() > 1) { channel.sendResponse( BytesRestResponse. - createSimpleErrorResponse(channel, BAD_REQUEST, "multiple values for single-valued header [" + name + "].")); + createSimpleErrorResponse(channel, BAD_REQUEST, "multiple values headersToCopy.toArray()for single-valued header [" + name + "].")); return; } else { threadContext.putHeader(name, String.join(",", distinctHeaderValues)); @@ -316,7 +316,12 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel return; } } else { - dispatchRequest(request, channel, handler); + if(handler.compatibilityRequired() == false //regular (not removed) handlers are always passed + || CompatibleHandlers.isV7Compatible(request)) { //handlers that were registered compatible, require request to be compatible + dispatchRequest(request, channel, handler); + } else { + handleCompatibleNotAllowed(rawPath,request.getHeaders(),channel); + } return; } } @@ -328,6 +333,22 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel handleBadRequest(uri, requestMethod, channel); } + private void handleCompatibleNotAllowed(String rawPath, Map> headers, RestChannel channel) throws IOException { + String msg = "compatible api can be only accessed with Compatible Header. path " + rawPath; + BytesRestResponse bytesRestResponse = BytesRestResponse.createSimpleErrorResponse(channel, RestStatus.NOT_FOUND, msg); + + channel.sendResponse(bytesRestResponse); + } + + private String printMap(Map> headers) { + StringBuilder s = new StringBuilder(); + for (String key : headers.keySet()) { + s.append(key + " "+headers.get(key).stream().map(Object::toString).collect(Collectors.joining(", "))); + s.append(";"); + } + return s.toString(); + } + Iterator getAllHandlers(@Nullable Map requestParamsRef, String rawPath) { final Supplier> paramsSupplier; if (requestParamsRef == null) { diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index ab7b468f7576d..4a3f6690bcbdc 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -26,6 +26,9 @@ import java.util.Collections; import java.util.List; +import java.util.function.Consumer; + +import static java.util.Collections.emptyList; /** * Handler for REST requests @@ -69,16 +72,19 @@ default boolean allowsUnsafeBuffers() { * The list of {@link Route}s that this RestHandler is responsible for handling. */ default List routes() { - return Collections.emptyList(); + return emptyList(); } + default List compatibleRoutes() { + return emptyList(); + } /** * A list of routes handled by this RestHandler that are deprecated and do not have a direct * replacement. If changing the {@code path} or {@code method} of a route, * use {@link #replacedRoutes()}. */ default List deprecatedRoutes() { - return Collections.emptyList(); + return emptyList(); } /** @@ -87,17 +93,27 @@ default List deprecatedRoutes() { * as deprecated alongside the updated {@code route}. */ default List replacedRoutes() { - return Collections.emptyList(); + return emptyList(); + } + + default boolean compatibilityRequired(){ + return false; } class Route { private final String path; private final Method method; + private final List> parameterConsumers; public Route(Method method, String path) { + this(method,path,emptyList()); + } + + public Route(Method method, String path, List> parameterConsumers) { this.path = path; this.method = method; + this.parameterConsumers = parameterConsumers; } public String getPath() { @@ -107,6 +123,10 @@ public String getPath() { public Method getMethod() { return method; } + + public List> getParameterConsumers() { + return parameterConsumers; + } } /** diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index 512bf72e9c0d3..3627ea0c02dd9 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -21,6 +21,7 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Nullable; @@ -131,10 +132,29 @@ void ensureSafeBuffers() { public static RestRequest request(NamedXContentRegistry xContentRegistry, HttpRequest httpRequest, HttpChannel httpChannel) { Map params = params(httpRequest.uri()); String path = path(httpRequest.uri()); - return new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel, + RestRequest restRequest = new RestRequest(xContentRegistry, params, path, httpRequest.getHeaders(), httpRequest, httpChannel, requestIdGenerator.incrementAndGet()); + addCompatibleParameter(restRequest); + return restRequest; } + private static void addCompatibleParameter(RestRequest request) { + if (isRequestCompatible(request)) { + String compatibleVersion = XContentType.parseVersion(request.header(Version.COMPATIBLE_HEADER)); + request.params().put(Version.COMPATIBLE_PARAMS_KEY, compatibleVersion); + request.param(Version.COMPATIBLE_PARAMS_KEY); + } + } + + public static boolean isRequestCompatible(RestRequest request) { + return isHeaderCompatible(request.header(Version.COMPATIBLE_HEADER)); + } + public static boolean isHeaderCompatible(String headerValue) { + String version = XContentType.parseVersion(headerValue); + return Version.COMPATIBLE_VERSION.equals(version); + } + + private static Map params(final String uri) { final Map params = new HashMap<>(); int index = uri.indexOf('?'); @@ -145,6 +165,8 @@ private static Map params(final String uri) { throw new BadParameterException(e); } } + + return params; } @@ -466,7 +488,8 @@ public final void withContentOrSourceParamParserOrNull(CheckedConsumer params) { @@ -110,7 +116,13 @@ private static void addParam(Map params, String name, String val * escape sequence. */ public static String decodeComponent(final String s) { - return decodeComponent(s, StandardCharsets.UTF_8, DECODE_PLUS_AS_SPACE); + return decodeComponent(s, StandardCharsets.UTF_8, false); + } + + public static String decodeComponent(final String compatibleWithVersion, final String s) { + boolean isCompatible = compatibleWithVersion!=null ? Version.V_7_0_0.major == Byte.parseByte(compatibleWithVersion) : false; + boolean decodePlusAsSpace = isCompatible && DECODE_PLUS_AS_SPACE; + return decodeComponent(s, StandardCharsets.UTF_8, decodePlusAsSpace); } /** diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 59156c70502ef..eb56b01237157 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -19,17 +19,21 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.CompatibleHandlers; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,6 +43,8 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestCreateIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestCreateIndexAction.class)); @Override public List routes() { @@ -52,13 +58,20 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - + if(CompatibleHandlers.isV7Compatible(request)) { + CompatibleHandlers.consumeParameterIncludeType(deprecationLogger).accept(request); + } CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - sourceAsMap = prepareMappings(sourceAsMap); + if(CompatibleHandlers.isV7Compatible(request)){ + sourceAsMap = prepareMappingsV7(sourceAsMap, request); + }else { + sourceAsMap = prepareMappings(sourceAsMap); + } + createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); } @@ -68,7 +81,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); } - static Map prepareMappings(Map source) { if (source.containsKey("mappings") == false || (source.get("mappings") instanceof Map) == false) { @@ -86,4 +98,26 @@ static Map prepareMappings(Map source) { newSource.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings)); return newSource; } + + static Map prepareMappingsV7(Map source, RestRequest request) { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + + @SuppressWarnings("unchecked") + Map mappings = (Map) source.get("mappings"); + + if (includeTypeName && mappings.size() == 1) { + //no matter what the type was, replace it with _doc + Map newSource = new HashMap<>(); + + String typeName = mappings.keySet().iterator().next(); + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + + newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings)); + return newSource; + }else{ + return prepareMappings(source); + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index c8f42975640a3..ff13bd8587ca9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -19,13 +19,16 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.CompatibleHandlers; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -40,6 +43,10 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestPutMappingAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestPutMappingAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in put " + + "mapping requests is deprecated. The parameter will be removed in the next major version."; @Override public List routes() { @@ -61,8 +68,28 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { - throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); + if(CompatibleHandlers.isV7Compatible(request)) { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.deprecatedAndMaybeLog("put_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + final String type = request.param("type"); +// putMappingRequest.type(includeTypeName ? type : MapperService.SINGLE_MAPPING_NAME); + if(includeTypeName && isMappingSourceTyped(type,sourceAsMap)){ + sourceAsMap = CompatibleHandlers.replaceTypeWithDoc(sourceAsMap); + } + if (includeTypeName == false && + (type != null || isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap))) { + throw new IllegalArgumentException("Types cannot be provided in put mapping requests, unless " + + "the include_type_name parameter is set to true."); + } + + + }else{ + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { + throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); + } } putMappingRequest.source(sourceAsMap); @@ -71,4 +98,37 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); return channel -> client.admin().indices().putMapping(putMappingRequest, new RestToXContentListener<>(channel)); } + + public static boolean isMappingSourceTyped(String type, Map mapping) { + return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); + } + + + public static class CompatibleRestPutMappingAction extends RestPutMappingAction { + @Override + public List routes() { + return unmodifiableList(asList( + new Route(PUT, "/{index}/{type}/_mapping/"), + new Route(PUT, "/{index}/_mapping/{type}"), + new Route(PUT, "/_mapping/{type}"), + + new Route(POST, "/{index}/{type}/_mapping/"), + new Route(POST, "/{index}/_mapping/{type}"), + new Route(POST, "/_mapping/{type}"), + + new Route(PUT, "/{index}/{type}/_mappings/"), + new Route(PUT, "/{index}/_mappings/{type}"), + new Route(PUT, "/_mappings/{type}"), + + new Route(POST, "/{index}/{type}/_mappings/"), + new Route(POST, "/{index}/_mappings/{type}"), + new Route(POST, "/_mappings/{type}") + )); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java index 54f1f954f816d..b4bde1c0ea410 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestGetAction.java @@ -19,12 +19,16 @@ package org.elasticsearch.rest.action.document; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.CompatibleHandlers; +import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestActions; @@ -33,6 +37,8 @@ import java.io.IOException; import java.util.List; +import java.util.List; +import java.util.function.Consumer; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; @@ -43,6 +49,12 @@ public class RestGetAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetAction.class)); + private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in " + + "document get requests is deprecated, use the /{index}/_doc/{id} endpoint instead."; + private static final Consumer DEPRECATION_WARNING = r -> deprecationLogger.deprecatedAndMaybeLog("get_with_types",TYPES_DEPRECATION_MESSAGE); + + @Override public String getName() { return "document_get_action"; @@ -88,4 +100,24 @@ protected RestStatus getStatus(final GetResponse response) { }); } + public static class CompatibleRestGetAction extends RestGetAction { + @Override + public List routes() { + return unmodifiableList(asList( + new Route(GET, "/{index}/{type}/{id}"), + new Route(HEAD, "/{index}/{type}/{id}"))); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException { + DEPRECATION_WARNING.accept(request); + CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request); + return super.prepareRequest(request, client); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index ef52af60e6efb..f8add87702f87 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -19,13 +19,17 @@ package org.elasticsearch.rest.action.document; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.index.VersionType; import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.CompatibleHandlers; +import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestStatusToXContentListener; @@ -33,6 +37,7 @@ import java.io.IOException; import java.util.List; import java.util.Locale; +import java.util.function.Consumer; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; @@ -41,6 +46,13 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestIndexAction extends BaseRestHandler { + private static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + + "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " + + "or /{index}/_create/{id})."; + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestIndexAction.class)); + private static final Consumer DEPRECATION_WARNING = + r -> deprecationLogger.deprecatedAndMaybeLog("index_with_types",TYPES_DEPRECATION_MESSAGE); @Override public List routes() { @@ -49,12 +61,35 @@ public List routes() { new Route(PUT, "/{index}/_doc/{id}"))); } + + @Override public String getName() { return "document_index_action"; } - public static final class CreateHandler extends RestIndexAction { + public static class CompatibleRestIndexAction extends RestIndexAction{ + @Override + public List routes() { + return unmodifiableList(asList( + new Route(POST, "/{index}/{type}/{id}"), + new Route(PUT, "/{index}/{type}/{id}"))); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException { + DEPRECATION_WARNING.accept(request); + CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request); + return super.prepareRequest(request, client); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } + + public static class CreateHandler extends RestIndexAction { @Override public String getName() { @@ -82,7 +117,28 @@ void validateOpType(String opType) { } } - public static final class AutoIdHandler extends RestIndexAction { + public static class CompatibleCreateHandler extends CreateHandler { + @Override + public List routes() { + return unmodifiableList(asList( + new Route(POST, "/{index}/{type}/{id}/_create"), + new Route(PUT, "/{index}/{type}/{id}/_create"))); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException { + DEPRECATION_WARNING.accept(request); + CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request); + return super.prepareRequest(request, client); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } + + public static class AutoIdHandler extends RestIndexAction { private final ClusterService clusterService; @@ -111,7 +167,30 @@ public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient } } - @Override + public static final class CompatibleAutoIdHandler extends AutoIdHandler { + + public CompatibleAutoIdHandler(ClusterService clusterService) { + super(clusterService); + } + + @Override + public List routes() { + return singletonList(new Route(POST, "/{index}/{type}")); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException { + DEPRECATION_WARNING.accept(request); + CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request); + return super.prepareRequest(request, client); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { IndexRequest indexRequest = new IndexRequest(request.param("index")); indexRequest.id(request.param("id")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 782c613a36277..19a22c33a7e05 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -19,15 +19,19 @@ package org.elasticsearch.rest.action.search; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.CompatibleHandlers; +import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestCancellableNodeClient; @@ -70,6 +74,21 @@ public class RestSearchAction extends BaseRestHandler { RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestSearchAction.class)); + + /* public RestSearchAction(RestController controller) { + controller.registerHandler(GET, "/_search", this); + controller.registerHandler(POST, "/_search", this); + controller.registerHandler(GET, "/{index}/_search", this); + controller.registerHandler(POST, "/{index}/_search", this); + + // Deprecated typed endpoints. + controller.registerCompatibleHandler(GET, "/{index}/{type}/_search", this, + List.of(CompatibleHandlers.consumeParameterType(deprecationLogger))); + controller.registerCompatibleHandler(POST, "/{index}/{type}/_search", this, + List.of(CompatibleHandlers.consumeParameterType(deprecationLogger))); + } +*/ @Override public String getName() { return "search_action"; @@ -319,4 +338,25 @@ protected Set responseParams() { public boolean allowsUnsafeBuffers() { return true; } + + public static class CompatibleRestSearchAction extends RestSearchAction { + @Override + public List routes() { + return unmodifiableList(asList( + new Route(GET, "/{index}/{type}/_search"), + new Route(POST, "/{index}/{type}/_search") + )); + } + + @Override + public RestChannelConsumer prepareRequest(RestRequest request, final NodeClient client) throws IOException { + CompatibleHandlers.consumeParameterType(deprecationLogger).accept(request); + return super.prepareRequest(request, client); + } + + @Override + public boolean compatibilityRequired() { + return true; + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 448b19a34bc83..43d43d12e8f84 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -34,14 +34,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.text.Text; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.*; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; -import org.elasticsearch.common.xcontent.ToXContentFragment; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -53,25 +47,14 @@ import org.elasticsearch.transport.RemoteClusterAware; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; -import static java.util.Collections.unmodifiableMap; +import java.util.*; + +import static java.util.Collections.*; import static org.elasticsearch.common.lucene.Lucene.readExplanation; import static org.elasticsearch.common.lucene.Lucene.writeExplanation; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName; -import static org.elasticsearch.common.xcontent.XContentParserUtils.parseFieldsValue; +import static org.elasticsearch.common.xcontent.XContentParserUtils.*; /** * A single search hit. @@ -529,6 +512,7 @@ public void setInnerHits(Map innerHits) { public static class Fields { static final String _INDEX = "_index"; + static final String _TYPE = "_type"; static final String _ID = "_id"; static final String _VERSION = "_version"; static final String _SEQ_NO = "_seq_no"; @@ -581,6 +565,9 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t if (index != null) { builder.field(Fields._INDEX, RemoteClusterAware.buildRemoteIndexName(clusterAlias, index)); } + if (builder.getCompatibleMajorVersion() == Version.V_7_0_0.major) { + builder.field(Fields._TYPE, SINGLE_MAPPING_TYPE); + } if (id != null) { builder.field(Fields._ID, id); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index a68413dfc4d03..2d29e4faecd87 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -73,6 +73,7 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.TermsSetQueryBuilder; +import org.elasticsearch.index.query.TypeQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.index.query.WrapperQueryBuilder; import org.elasticsearch.index.query.functionscore.ExponentialDecayFunctionBuilder; @@ -762,6 +763,7 @@ private void registerQueryParsers(List plugins) { registerQuery(new QuerySpec<>(ScriptScoreQueryBuilder.NAME, ScriptScoreQueryBuilder::new, ScriptScoreQueryBuilder::fromXContent)); registerQuery( new QuerySpec<>(SimpleQueryStringBuilder.NAME, SimpleQueryStringBuilder::new, SimpleQueryStringBuilder::fromXContent)); + registerQuery(new QuerySpec<>(TypeQueryBuilder.NAME, TypeQueryBuilder::new, TypeQueryBuilder::fromXContent),true); registerQuery(new QuerySpec<>(ScriptQueryBuilder.NAME, ScriptQueryBuilder::new, ScriptQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoDistanceQueryBuilder.NAME, GeoDistanceQueryBuilder::new, GeoDistanceQueryBuilder::fromXContent)); registerQuery(new QuerySpec<>(GeoBoundingBoxQueryBuilder.NAME, GeoBoundingBoxQueryBuilder::new, @@ -810,6 +812,12 @@ private void registerQuery(QuerySpec spec) { (p, c) -> spec.getParser().fromXContent(p))); } + private void registerQuery(QuerySpec spec, boolean requiresCompatible) { + namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader())); + namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), + (p, c) -> spec.getParser().fromXContent(p), requiresCompatible)); + } + public FetchPhase getFetchPhase() { return new FetchPhase(fetchSubPhases); } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java index 47a470e2cea84..a2b89b2bf542f 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/XContentTypeTests.java @@ -84,4 +84,11 @@ public void testFromRubbish() throws Exception { assertThat(XContentType.fromMediaTypeOrFormat("text/plain"), nullValue()); assertThat(XContentType.fromMediaTypeOrFormat("gobbly;goop"), nullValue()); } + + public void testMediaType() throws Exception { + String mediaType = XContentType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=7"); + assertThat(mediaType,equalTo("application/json")); + mediaType = XContentType.parseMediaType("application/json"); + assertThat(mediaType,equalTo("application/json")); + } } diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleHandlersTest.java b/server/src/test/java/org/elasticsearch/rest/CompatibleHandlersTest.java new file mode 100644 index 0000000000000..f255831140150 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/CompatibleHandlersTest.java @@ -0,0 +1,11 @@ +package org.elasticsearch.rest; + +import org.elasticsearch.test.ESTestCase; + + +public class CompatibleHandlersTest extends ESTestCase { + +// public void testParseAcceptHeader(){ +// CompatibleHandlers.isCompatible(params); +// } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java index dd650a38ebbd7..01806b2771d39 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestCandidate.java @@ -23,6 +23,8 @@ import org.elasticsearch.test.rest.yaml.section.TeardownSection; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import java.util.Objects; + /** * Wraps {@link ClientYamlTestSection}s ready to be run. Each test section is associated to its {@link ClientYamlTestSuite}. */ @@ -68,4 +70,17 @@ public ClientYamlTestSection getTestSection() { public String toString() { return getTestPath(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClientYamlTestCandidate that = (ClientYamlTestCandidate) o; + return Objects.equals(getTestPath(), that.getTestPath()); + } + + @Override + public int hashCode() { + return Objects.hash(getTestPath()); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 1d0104e559889..73d0f251eb533 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -51,6 +51,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -88,8 +89,8 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase { */ private static final String REST_TESTS_VALIDATE_SPEC = "tests.rest.validate_spec"; - private static final String TESTS_PATH = "/rest-api-spec/test"; - private static final String SPEC_PATH = "/rest-api-spec/api"; + private static final String DEFAULT_TESTS_PATH = "/rest-api-spec/test"; + private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api"; /** * This separator pattern matches ',' except it is preceded by a '\'. @@ -125,7 +126,7 @@ public void initAndResetContext() throws Exception { if (restTestExecutionContext == null) { assert adminExecutionContext == null; assert blacklistPathMatchers == null; - final ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(SPEC_PATH); + final ClientYamlSuiteRestSpec restSpec = ClientYamlSuiteRestSpec.load(getApiSpecRelativePath()); validateSpec(restSpec); final List hosts = getClusterHosts(); Tuple versionVersionTuple = readVersionsFromCatNodes(adminClient()); @@ -155,6 +156,10 @@ public void initAndResetContext() throws Exception { restTestExecutionContext.clear(); } + protected String getApiSpecRelativePath() { + return DEFAULT_SPEC_PATH; + } + protected ClientYamlTestClient initClientYamlTestClient( final ClientYamlSuiteRestSpec restSpec, final RestClient restClient, @@ -182,22 +187,22 @@ public static void closeClient() throws IOException { * defined in {@link ExecutableSection}. */ public static Iterable createParameters() throws Exception { - return createParameters(ExecutableSection.XCONTENT_REGISTRY); + return createParameters(ExecutableSection.XCONTENT_REGISTRY, DEFAULT_TESTS_PATH); } /** * Create parameters for this parameterized test. */ - public static Iterable createParameters(NamedXContentRegistry executeableSectionRegistry) throws Exception { + public static Iterable createParameters(NamedXContentRegistry executableSectionRegistry, String testsClassPath) throws Exception { String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root - Map> yamlSuites = loadSuites(paths); + Map> yamlSuites = loadSuites(testsClassPath, paths); List suites = new ArrayList<>(); IllegalArgumentException validationException = null; // yaml suites are grouped by directory (effectively by api) for (String api : yamlSuites.keySet()) { List yamlFiles = new ArrayList<>(yamlSuites.get(api)); for (Path yamlFile : yamlFiles) { - ClientYamlTestSuite suite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile); + ClientYamlTestSuite suite = ClientYamlTestSuite.parse(executableSectionRegistry, api, yamlFile); suites.add(suite); try { suite.validate(); @@ -235,9 +240,9 @@ public static Iterable createParameters(NamedXContentRegistry executea /** Find all yaml suites that match the given list of paths from the root test path. */ // pkg private for tests - static Map> loadSuites(String... paths) throws Exception { + static Map> loadSuites(String testsClassPath, String... paths) throws Exception { Map> files = new HashMap<>(); - Path root = PathUtils.get(ESClientYamlSuiteTestCase.class.getResource(TESTS_PATH).toURI()); + Path root = PathUtils.get(ESClientYamlSuiteTestCase.class.getResource(testsClassPath).toURI()); for (String strPath : paths) { Path path = root.resolve(strPath); if (Files.isDirectory(path)) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 48e7fc031139b..30035ed4104ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -42,7 +42,8 @@ public static ClientYamlTestSection parse(XContentParser parser) throws IOExcept SkipSection skipSection = SkipSection.parseIfNext(parser); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); - executableSections.add(ExecutableSection.parse(parser)); + executableSections.add(ExecutableSection. + parse(parser)); } if (parser.nextToken() != XContentParser.Token.END_OBJECT) { throw new IllegalArgumentException("malformed section [" + sectionName + "] expected [" diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 1b588f554fa53..3c252b06d698f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -188,6 +188,7 @@ public static DoSection parse(XContentParser parser) throws IOException { private String catchParam; private ApiCallSection apiCallSection; private List expectedWarningHeaders = emptyList(); + private boolean checkWarningHeaders; public DoSection(XContentLocation location) { this.location = location; @@ -209,6 +210,10 @@ void setApiCallSection(ApiCallSection apiCallSection) { this.apiCallSection = apiCallSection; } + public void checkWarningHeaders(boolean check){ + this.checkWarningHeaders = check; + } + /** * Warning headers that we expect from this response. If the headers don't match exactly this request is considered to have failed. * Defaults to emptyList. @@ -281,6 +286,9 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx * Check that the response contains only the warning headers that we expect. */ void checkWarningHeaders(final List warningHeaders, final Version masterVersion) { + if(checkWarningHeaders == false){ + return; + } final List unexpected = new ArrayList<>(); final List unmatched = new ArrayList<>(); final List missing = new ArrayList<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java index 09f88f42492b6..0a701732a0f13 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java @@ -56,6 +56,12 @@ public MatchAssertion(XContentLocation location, String field, Object expectedVa @Override protected void doAssert(Object actualValue, Object expectedValue) { + // some kind of override.. + if(getField().equals("_type") ){ + assertThat(actualValue, equalTo("_doc")); + return; + } + //if the value is wrapped into / it is a regexp (e.g. /s+d+/) if (expectedValue instanceof String) { String expValue = ((String) expectedValue).trim(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java index 20b0086c1e4e2..7a54ea85a1b5f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java @@ -290,4 +290,5 @@ public void close() throws IOException { public DeprecationHandler getDeprecationHandler() { return parser.getDeprecationHandler(); } + } diff --git a/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip b/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip new file mode 100644 index 0000000000000..6306093da5d59 Binary files /dev/null and b/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip differ diff --git a/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip.md5 b/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip.md5 new file mode 100644 index 0000000000000..c2b6fc2d7a7f4 --- /dev/null +++ b/x-pack/plugin/ml/cpp-snapshot/.cache/ml-cpp-7.0.2-SNAPSHOT.zip.md5 @@ -0,0 +1 @@ +"3950ed38a5d33835c1af40d6e4aa9f13" \ No newline at end of file