Version/xcontent

distribution/build.gradle

@@ -422,14 +422,16 @@ testClusters {
     if (System.getProperty('run.distribution', 'default') == 'default') {
       String licenseType = System.getProperty("run.license_type", "basic")
       if (licenseType == 'trial') {
-        setting 'xpack.ml.enabled', 'true'
+        setting 'xpack.ml.enabled', 'false'
         setting 'xpack.graph.enabled', 'true'
         setting 'xpack.watcher.enabled', 'true'
         setting 'xpack.license.self_generated.type', 'trial'
       } else if (licenseType != 'basic') {
         throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
       }
-      setting 'xpack.security.enabled', 'true'
+      setting 'xpack.ml.enabled', 'false'
+
+      setting 'xpack.security.enabled', 'false'
       setting 'xpack.monitoring.enabled', 'true'
       setting 'xpack.sql.enabled', 'true'
       setting 'xpack.rollup.enabled', 'true'

distribution/bwc/build.gradle

@@ -45,6 +45,7 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased
     assemble.enabled = false
 
     File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
+    project.ext.checkoutDir = checkoutDir
 
     final String remote = System.getProperty("bwc.remote", "elastic")
 

distribution/src/config/elasticsearch.yml

@@ -86,3 +86,4 @@ ${path.logs}
 # Require explicit names when deleting indices:
 #
 #action.destructive_requires_name: true
+xpack.ml.enabled: false

distribution/src/config/jvm.options

@@ -64,3 +64,4 @@ ${error.file}
 
 ## GC logging
 -Xlog:gc*,gc+age=trace,safepoint:file=${loggc}:utctime,pid,tags:filecount=32,filesize=64m
+-Des.rest.url_plus_as_space=true

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java

@@ -55,12 +55,14 @@ public static class Entry {
 
         /** A parser capability of parser the entry's class. */
         private final ContextParser<Object, ?> parser;
+        private final boolean requiresCompatible;
 
         /** Creates a new entry which can be stored by the registry. */
         public <T> Entry(Class<T> categoryClass, ParseField name, CheckedFunction<XContentParser, ? extends T, IOException> parser) {
             this.categoryClass = Objects.requireNonNull(categoryClass);
             this.name = Objects.requireNonNull(name);
             this.parser = Objects.requireNonNull((p, c) -> parser.apply(p));
+            this.requiresCompatible = false;
         }
         /**
          * Creates a new entry which can be stored by the registry.
@@ -70,6 +72,18 @@ public <T> Entry(Class<T> categoryClass, ParseField name, ContextParser<Object,
             this.categoryClass = Objects.requireNonNull(categoryClass);
             this.name = Objects.requireNonNull(name);
             this.parser = Objects.requireNonNull(parser);
+            this.requiresCompatible = false;
+        }
+
+        public <T> Entry(Class<T> categoryClass, ParseField name, ContextParser<Object, ? extends T> parser, boolean requiresCompatible) {
+            this.categoryClass = Objects.requireNonNull(categoryClass);
+            this.name = Objects.requireNonNull(name);
+            this.parser = Objects.requireNonNull(parser);
+            this.requiresCompatible = requiresCompatible;
+        }
+
+        public boolean isCompatibleWith(XContentParser parser) {
+            return requiresCompatible == false || parser.isCompatible();
         }
     }
 
@@ -128,7 +142,7 @@ public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentPa
             throw new XContentParseException("unknown named object category [" + categoryClass.getName() + "]");
         }
         Entry entry = parsers.get(name);
-        if (entry == null) {
+        if (entry == null || entry.isCompatibleWith(parser) == false) {
             throw new NamedObjectNotFoundException(parser.getTokenLocation(), "unknown field [" + name + "]", parsers.keySet());
         }
         if (false == entry.name.match(name, parser.getDeprecationHandler())) {
@@ -139,5 +153,4 @@ public <T, C> T parseNamedObject(Class<T> categoryClass, String name, XContentPa
         }
         return categoryClass.cast(entry.parser.parse(parser, context));
     }
-
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -19,28 +19,12 @@
 
 package org.elasticsearch.common.xcontent;
 
-import java.io.ByteArrayOutputStream;
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.file.Path;
 import java.time.ZonedDateTime;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.ServiceLoader;
-import java.util.Set;
+import java.util.*;
 import java.util.function.Function;
 
 /**
@@ -137,6 +121,7 @@ public static XContentBuilder builder(XContent xContent, Set<String> includes, S
         DATE_TRANSFORMERS = Collections.unmodifiableMap(dateTransformers);
     }
 
+
     @FunctionalInterface
     public interface Writer {
         void write(XContentBuilder builder, Object value) throws IOException;
@@ -165,6 +150,8 @@ public interface HumanReadableTransformer {
      */
     private boolean humanReadable = false;
 
+    private byte compatibleVersion;
+
     /**
      * Constructs a new builder using the provided XContent and an OutputStream. Make sure
      * to call {@link #close()} when the builder is done with.
@@ -998,6 +985,16 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce
         return this;
     }
 
+    public XContentBuilder compatibleVersion(byte compatibleVersion){
+        this.compatibleVersion = compatibleVersion;
+        return this;
+    }
+
+    public byte getCompatibleMajorVersion() {
+        return compatibleVersion;
+    }
+
+
     @Override
     public void flush() throws IOException {
         generator.flush();

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java

@@ -266,4 +266,13 @@ <T> Map<String, T> map(
      * The callback to notify when parsing encounters a deprecated field.
      */
     DeprecationHandler getDeprecationHandler();
+
+
+    default XContentParser withIsCompatible(boolean isCompatible) {
+         return this;
+    }
+
+    default boolean isCompatible() {
+        return false;
+    }
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentType.java

@@ -26,6 +26,8 @@
 
 import java.util.Locale;
 import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * The content type of {@link org.elasticsearch.common.xcontent.XContent}.
@@ -142,7 +144,9 @@ public static XContentType fromMediaTypeOrFormat(String mediaType) {
      * The provided media type should not include any parameters. This method is suitable for parsing part of the {@code Content-Type}
      * HTTP header. This method will return {@code null} if no match is found
      */
-    public static XContentType fromMediaType(String mediaType) {
+    public static XContentType fromMediaType(String mediaTypeHeaderValue) {
+        String mediaType = parseMediaType(mediaTypeHeaderValue);
+
         final String lowercaseMediaType = Objects.requireNonNull(mediaType, "mediaType cannot be null").toLowerCase(Locale.ROOT);
         for (XContentType type : values()) {
             if (type.mediaTypeWithoutParameters().equals(lowercaseMediaType)) {
@@ -157,6 +161,29 @@ public static XContentType fromMediaType(String mediaType) {
         return null;
     }
 
+    static Pattern pattern = Pattern.compile("application/(vnd.elasticsearch\\+)?([^;]+)(\\s*;\\s*compatible-with=(\\d+))?");
+
+    public static String parseMediaType(String mediaType) {
+        if (mediaType != null) {
+            Matcher matcher = pattern.matcher(mediaType);
+            if (matcher.find()) {
+                return "application/"+matcher.group(2);
+            }
+        }
+
+        return mediaType;
+    }
+
+    public static String parseVersion(String mediaType){
+        if(mediaType != null){
+            Matcher matcher = pattern.matcher(mediaType);
+            if (matcher.find() && "vnd.elasticsearch+".equals(matcher.group(1))) {
+
+                return matcher.group(4);
+            }
+        }
+        return null;
+    }
     private static boolean isSameMediaTypeOrFormatAs(String stringType, XContentType type) {
         return type.mediaTypeWithoutParameters().equalsIgnoreCase(stringType) ||
                 stringType.toLowerCase(Locale.ROOT).startsWith(type.mediaTypeWithoutParameters().toLowerCase(Locale.ROOT) + ";") ||

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java

@@ -57,12 +57,24 @@ private static void checkCoerceString(boolean coerce, Class<? extends Number> cl
 
     private final NamedXContentRegistry xContentRegistry;
     private final DeprecationHandler deprecationHandler;
+    private boolean isCompatible = false;
 
     public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler) {
         this.xContentRegistry = xContentRegistry;
         this.deprecationHandler = deprecationHandler;
     }
 
+    @Override
+    public XContentParser withIsCompatible(boolean isCompatible) {
+        this.isCompatible = isCompatible;
+        return this;
+    }
+
+    @Override
+    public boolean isCompatible() {
+        return isCompatible;
+    }
+
     // The 3rd party parsers we rely on are known to silently truncate fractions: see
     //   http://fasterxml.github.io/jackson-core/javadoc/2.3.0/com/fasterxml/jackson/core/JsonParser.html#getShortValue()
     // If this behaviour is flagged as undesirable and any truncation occurs

qa/os/bats/plugins/25_tar_plugins.bats

@@ -0,0 +1 @@
+module_and_plugin_test_cases.bash

qa/os/bats/plugins/50_modules_and_plugins.bats

@@ -0,0 +1 @@
+module_and_plugin_test_cases.bash

qa/rest-compat-tests/build.gradle

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.testclusters'
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.rest-test'
+
+def restCompatAPIRoot =  new File(new File(new File(project.buildDir, 'resources'), 'test'), 'restCompatAPIRoot')
+def restCompatTestRoot = new File(new File(new File(project.buildDir, 'resources'), 'test'), 'restCompatTestRoot');
+
+task copyAPIFromBwc(type: Copy) {
+  dependsOn ':distribution:bwc:minor:checkoutBwcBranch'
+  from (project(':distribution:bwc:minor').checkoutDir){
+    include 'rest-api-spec/**/rest-api-spec/api/**/*.json'
+    include 'modules/**/rest-api-spec/api/**/*.json'
+    include 'plugins/**/rest-api-spec/api/**/*.json'
+  }
+  into restCompatAPIRoot
+  includeEmptyDirs = false
+}
+
+task copyTestsFromBwc(type: Copy) {
+  dependsOn ':distribution:bwc:minor:checkoutBwcBranch'
+  from (project(':distribution:bwc:minor').checkoutDir){
+    include 'rest-api-spec/**/rest-api-spec/test/**/*.yml'
+    include 'modules/**/rest-api-spec/test/**/*.yml'
+    include 'plugins/**/rest-api-spec/test/**/*.yml'
+  }
+  into restCompatTestRoot
+  includeEmptyDirs = false
+}
+
+task copyFromBwc {
+  dependsOn copyAPIFromBwc
+  dependsOn copyTestsFromBwc
+}
+
+integTest.runner {
+  dependsOn copyFromBwc
+  systemProperty 'restCompatAPIRoot', restCompatAPIRoot
+  systemProperty 'restCompatTestRoot', restCompatTestRoot
+}
+
+dependencies {
+  testCompile project(':test:framework')
+}
+
+testClusters.integTest {
+  //reindex module
+  setting 'reindex.remote.whitelist', '127.0.0.1:*'
+
+}
+
+