From d8fff72c4e75c808053615f5c878b65e9ebefd21 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 29 Nov 2019 15:00:30 +0100 Subject: [PATCH] draft not working --- .../test/indices.create/20_basic.yml | 17 + .../indices/create/CreateIndexRequest.java | 35 ++ .../indices/create/CreateIndexRequestV7.java | 488 ++++++++++++++++++ .../create/CreateIndexRequestTests.java | 37 ++ .../mapper/ExternalFieldMapperTests.java | 1 + 5 files changed, 578 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_basic.yml create mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestV7.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_basic.yml new file mode 100644 index 0000000000000..2b2f89f078408 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_basic.yml @@ -0,0 +1,17 @@ +--- +"Create index with mappings": + + - do: + indices.create: + index: test_index + body: + mappings: + properties: + field: + type: text + + - do: + indices.get_mapping: + index: test_index + + - is_true: test_index.mappings diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 39014d173c468..b49f72f698178 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Objects; @@ -290,6 +291,7 @@ else if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) { } try { XContentBuilder builder = XContentFactory.jsonBuilder(); + source = fix((Map) source); builder.map(source); return mapping(Strings.toString(builder)); } catch (IOException e) { @@ -297,6 +299,39 @@ else if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) { } } + public Map fix(Map source) { + Map destination = new HashMap<>(); + traverse(source,destination,null,false); + System.out.println(destination); + return destination; + } + + private void traverse(Map current, Map destination, Map parent, boolean isWithinMultifield) { + for (Map.Entry entry : current.entrySet()) { + if (!(entry.getValue() instanceof Map)) { + destination.put(entry.getKey(), entry.getValue()); + } else { + if(entry.getKey().equals("fields") ){ + if(isWithinMultifield){ + HashMap destination1 = new HashMap<>(); + traverse((Map) entry.getValue(), destination1, parent, true); + parent.put(entry.getKey(),destination1); + }else{ + HashMap destination1 = new HashMap<>(); + traverse((Map) entry.getValue(), destination1, destination, true); + destination.put(entry.getKey(), destination1); + + } + }else { + HashMap destination1 = new HashMap<>(); + traverse((Map) entry.getValue(), destination1, destination, isWithinMultifield); + destination.put(entry.getKey(), destination1); + } + } + } + + } + /** * A specialized simplified mapping source method, takes the form of simple properties definition: * ("field1", "type=string,store=true"). diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestV7.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestV7.java new file mode 100644 index 0000000000000..c02a4fe7e17f4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestV7.java @@ -0,0 +1,488 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.ElasticsearchGenerationException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; +import static org.elasticsearch.common.settings.Settings.readSettingsFromStream; +import static org.elasticsearch.common.settings.Settings.writeSettingsToStream; + +/** + * A request to create an index. Best created with {@link org.elasticsearch.client.Requests#createIndexRequest(String)}. + *

+ * The index created can optionally be created with {@link #settings(Settings)}. + * + * @see org.elasticsearch.client.Requests#createIndexRequest(String) + * @see CreateIndexResponse + */ +public class CreateIndexRequestV7 extends AcknowledgedRequest implements IndicesRequest { + + public static final ParseField MAPPINGS = new ParseField("mappings"); + public static final ParseField SETTINGS = new ParseField("settings"); + public static final ParseField ALIASES = new ParseField("aliases"); + + private String cause = ""; + + private String index; + + private Settings settings = EMPTY_SETTINGS; + + private String mappings = "{}"; + + private final Set aliases = new HashSet<>(); + + private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; + + public CreateIndexRequestV7(StreamInput in) throws IOException { + super(in); + cause = in.readString(); + index = in.readString(); + settings = readSettingsFromStream(in); + if (in.getVersion().before(Version.V_8_0_0)) { + int size = in.readVInt(); + assert size <= 1 : "Expected to read 0 or 1 mappings, but received " + size; + if (size == 1) { + String type = in.readString(); + if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) { + throw new IllegalArgumentException("Expected to receive mapping type of [_doc] but got [" + type + "]"); + } + mappings = in.readString(); + } + } else { + mappings = in.readString(); + } + int aliasesSize = in.readVInt(); + for (int i = 0; i < aliasesSize; i++) { + aliases.add(new Alias(in)); + } + waitForActiveShards = ActiveShardCount.readFrom(in); + } + + public CreateIndexRequestV7() { + } + + /** + * Constructs a new request to create an index with the specified name. + */ + public CreateIndexRequestV7(String index) { + this(index, EMPTY_SETTINGS); + } + + /** + * Constructs a new request to create an index with the specified name and settings. + */ + public CreateIndexRequestV7(String index, Settings settings) { + this.index = index; + this.settings = settings; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (index == null) { + validationException = addValidationError("index is missing", validationException); + } + return validationException; + } + + @Override + public String[] indices() { + return new String[]{index}; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + + /** + * The index name to create. + */ + public String index() { + return index; + } + + public CreateIndexRequestV7 index(String index) { + this.index = index; + return this; + } + + /** + * The settings to create the index with. + */ + public Settings settings() { + return settings; + } + + /** + * The cause for this index creation. + */ + public String cause() { + return cause; + } + + /** + * The settings to create the index with. + */ + public CreateIndexRequestV7 settings(Settings.Builder settings) { + this.settings = settings.build(); + return this; + } + + /** + * The settings to create the index with. + */ + public CreateIndexRequestV7 settings(Settings settings) { + this.settings = settings; + return this; + } + + /** + * The settings to create the index with (either json or yaml format) + */ + public CreateIndexRequestV7 settings(String source, XContentType xContentType) { + this.settings = Settings.builder().loadFromSource(source, xContentType).build(); + return this; + } + + /** + * Allows to set the settings using a json builder. + */ + public CreateIndexRequestV7 settings(XContentBuilder builder) { + settings(Strings.toString(builder), builder.contentType()); + return this; + } + + /** + * The settings to create the index with (either json/yaml/properties format) + */ + public CreateIndexRequestV7 settings(Map source) { + try { + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + builder.map(source); + settings(Strings.toString(builder), XContentType.JSON); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + return this; + } + + /** + * Set the mapping for this index + * + * The mapping should be in the form of a JSON string, with an outer _doc key + * 

+     *     .mapping("{\"_doc\":{\"properties\": ... }}")
+     *
+ */ + public CreateIndexRequestV7 mapping(String mapping) { + this.mappings = mapping; + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType The content type of the source + */ + public CreateIndexRequestV7 mapping(String type, String source, XContentType xContentType) { + return mapping(type, new BytesArray(source), xContentType); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + * @param xContentType the content type of the mapping source + */ + private CreateIndexRequestV7 mapping(String type, BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); + Map mappingAsMap = XContentHelper.convertToMap(source, false, xContentType).v2(); + return mapping(type, mappingAsMap); + } + + /** + * The cause for this index creation. + */ + public CreateIndexRequestV7 cause(String cause) { + this.cause = cause; + return this; + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + */ + public CreateIndexRequestV7 mapping(String type, XContentBuilder source) { + return mapping(type, BytesReference.bytes(source), source.contentType()); + } + + /** + * Adds mapping that will be added when the index gets created. + * + * @param type The mapping type + * @param source The mapping source + */ + public CreateIndexRequestV7 mapping(String type, Map source) { + // wrap it in a type map if its not + if (source.size() != 1 || !source.containsKey(type)) { + source = Map.of(MapperService.SINGLE_MAPPING_NAME, source); + } + else if (MapperService.SINGLE_MAPPING_NAME.equals(type) == false) { + // if it has a different type name, then unwrap and rewrap with _doc + source = Map.of(MapperService.SINGLE_MAPPING_NAME, source.get(type)); + } + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.map(source); + return mapping(Strings.toString(builder)); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + /** + * A specialized simplified mapping source method, takes the form of simple properties definition: + * ("field1", "type=string,store=true"). + */ + public CreateIndexRequestV7 mapping(String type, Object... source) { + mapping(type, PutMappingRequest.buildFromSimplifiedDef(type, source)); + return this; + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public CreateIndexRequestV7 aliases(Map source) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.map(source); + return aliases(BytesReference.bytes(builder)); + } catch (IOException e) { + throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); + } + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public CreateIndexRequestV7 aliases(XContentBuilder source) { + return aliases(BytesReference.bytes(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public CreateIndexRequestV7 aliases(String source) { + return aliases(new BytesArray(source)); + } + + /** + * Sets the aliases that will be associated with the index when it gets created + */ + public CreateIndexRequestV7 aliases(BytesReference source) { + // EMPTY is safe here because we never call namedObject + try (XContentParser parser = XContentHelper + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, source)) { + //move to the first alias + parser.nextToken(); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + alias(Alias.fromXContent(parser)); + } + return this; + } catch(IOException e) { + throw new ElasticsearchParseException("Failed to parse aliases", e); + } + } + + /** + * Adds an alias that will be associated with the index when it gets created + */ + public CreateIndexRequestV7 alias(Alias alias) { + this.aliases.add(alias); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestV7 source(String source, XContentType xContentType) { + return source(new BytesArray(source), xContentType); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestV7 source(XContentBuilder source) { + return source(BytesReference.bytes(source), source.contentType()); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestV7 source(byte[] source, XContentType xContentType) { + return source(source, 0, source.length, xContentType); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestV7 source(byte[] source, int offset, int length, XContentType xContentType) { + return source(new BytesArray(source, offset, length), xContentType); + } + + /** + * Sets the settings and mappings as a single source. + */ + public CreateIndexRequestV7 source(BytesReference source, XContentType xContentType) { + Objects.requireNonNull(xContentType); + source(XContentHelper.convertToMap(source, false, xContentType).v2(), LoggingDeprecationHandler.INSTANCE); + return this; + } + + /** + * Sets the settings and mappings as a single source. + */ + @SuppressWarnings("unchecked") + public CreateIndexRequestV7 source(Map source, DeprecationHandler deprecationHandler) { + for (Map.Entry entry : source.entrySet()) { + String name = entry.getKey(); + if (SETTINGS.match(name, deprecationHandler)) { + if (entry.getValue() instanceof Map == false) { + throw new ElasticsearchParseException("key [settings] must be an object"); + } + settings((Map) entry.getValue()); + } else if (MAPPINGS.match(name, deprecationHandler)) { + Map mappings = (Map) entry.getValue(); + for (Map.Entry entry1 : mappings.entrySet()) { + mapping(entry1.getKey(), (Map) entry1.getValue()); + } + } else if (ALIASES.match(name, deprecationHandler)) { + aliases((Map) entry.getValue()); + } else { + throw new ElasticsearchParseException("unknown key [{}] for create index", name); + } + } + return this; + } + + public String mappings() { + return this.mappings; + } + + public Set aliases() { + return this.aliases; + } + + public ActiveShardCount waitForActiveShards() { + return waitForActiveShards; + } + + /** + * Sets the number of shard copies that should be active for index creation to return. + * Defaults to {@link ActiveShardCount#DEFAULT}, which will wait for one shard copy + * (the primary) to become active. Set this value to {@link ActiveShardCount#ALL} to + * wait for all shards (primary and all replicas) to be active before returning. + * Otherwise, use {@link ActiveShardCount#from(int)} to set this value to any + * non-negative integer, up to the number of copies per shard (number of replicas + 1), + * to wait for the desired amount of shard copies to become active before returning. + * Index creation will only wait up until the timeout value for the number of shard copies + * to be active before returning. Check {@link CreateIndexResponse#isShardsAcknowledged()} to + * determine if the requisite shard copies were all started before returning or timing out. + * + * @param waitForActiveShards number of active shard copies to wait on + */ + public CreateIndexRequestV7 waitForActiveShards(ActiveShardCount waitForActiveShards) { + this.waitForActiveShards = waitForActiveShards; + return this; + } + + /** + * A shortcut for {@link #waitForActiveShards(ActiveShardCount)} where the numerical + * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} + * to get the ActiveShardCount. + */ + public CreateIndexRequestV7 waitForActiveShards(final int waitForActiveShards) { + return waitForActiveShards(ActiveShardCount.from(waitForActiveShards)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(cause); + out.writeString(index); + writeSettingsToStream(settings, out); + if (out.getVersion().before(Version.V_8_0_0)) { + if ("{}".equals(mappings)) { + out.writeVInt(0); + } else { + out.writeVInt(1); + out.writeString(MapperService.SINGLE_MAPPING_NAME); + out.writeString(mappings); + } + } else { + out.writeString(mappings); + } + out.writeVInt(aliases.size()); + for (Alias alias : aliases) { + alias.writeTo(out); + } + waitForActiveShards.writeTo(out); + } + +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java index c3d8a1853c1f4..e92a269738be0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestTests.java @@ -80,6 +80,43 @@ public void testTopLevelKeys() { () -> {request.source(createIndex, XContentType.JSON);}); assertEquals("unknown key [FOO_SHOULD_BE_ILLEGAL_HERE] for create index", e.getMessage()); } + public void testx() throws IOException { + CreateIndexRequest request2 = new CreateIndexRequest("bar"); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + builder.startObject(); + { + builder.startObject("properties"); + { + builder.startObject("field1"); + { + builder.field("type", "text"); + builder.startObject("fields"); + { + builder.startObject("raw"); + { + builder.field("type", "keyword"); + builder.startObject("fields"); + { + builder.startObject("raw2"); + { + builder.field("type", "keyword"); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + } + builder.endObject(); + request2.mapping("type1", builder); + + } public void testMappingKeyedByType() throws IOException { CreateIndexRequest request1 = new CreateIndexRequest("foo"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index 0c56090648ecb..01d12b3cbc807 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings;