diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java index 15c2595f53..42801797b3 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/PluginModel.java @@ -12,11 +12,11 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.core.JacksonException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.function.BiFunction; @@ -62,7 +61,6 @@ public class PluginModel { */ static class InternalJsonModel { @JsonAnySetter - @JsonAnyGetter private final Map pluginSettings; @JsonCreator @@ -73,6 +71,11 @@ static class InternalJsonModel { InternalJsonModel(final Map pluginSettings) { this.pluginSettings = pluginSettings; } + + @JsonAnyGetter + Map getPluginSettingsForSerialization() { + return pluginSettings != null ? pluginSettings : new HashMap<>(); + } } public PluginModel(final String pluginName, final Map pluginSettings) { @@ -117,10 +120,28 @@ public PluginModelSerializer(final Class valueClass) { public void serialize( final PluginModel value, final JsonGenerator gen, final SerializerProvider provider) throws IOException { gen.writeStartObject(); - Map serializedInner = SERIALIZER_OBJECT_MAPPER.convertValue(value.innerModel, Map.class); - if(serializedInner != null && serializedInner.isEmpty()) - serializedInner = null; - gen.writeObjectField(value.getPluginName(), serializedInner); + + // Serialize innerModel to a Map via a JSON round-trip so that all Jackson-annotated + // fields on potential subclasses of InternalJsonModel (e.g. SinkInternalJsonModel with + // routes, tagsTargetKey, etc.) are included. Directly reading pluginSettings would miss + // those extra fields. The resulting Map is then inspected to distinguish between a + // truly empty/null inner model and one that has actual content to write. + final String jsonString = SERIALIZER_OBJECT_MAPPER.writeValueAsString(value.innerModel); + final Map serializedInner = SERIALIZER_OBJECT_MAPPER.readValue(jsonString, Map.class); + + if (serializedInner.isEmpty()) { + // Empty inner model: output null if pluginSettings was null, {} if it was an empty map + if (value.innerModel.pluginSettings == null) { + gen.writeObjectField(value.getPluginName(), null); + } else { + gen.writeFieldName(value.getPluginName()); + gen.writeStartObject(); + gen.writeEndObject(); + } + } else { + gen.writeObjectField(value.getPluginName(), serializedInner); + } + gen.writeEndObject(); } } @@ -136,7 +157,7 @@ public void serialize( static final class PluginModelDeserializer extends AbstractPluginModelDeserializer { public PluginModelDeserializer() { - super(PluginModel.class, InternalJsonModel.class, PluginModel::new, InternalJsonModel::new); + super(PluginModel.class, InternalJsonModel.class, PluginModel::new, () -> new InternalJsonModel(null)); } } @@ -156,33 +177,57 @@ abstract static class AbstractPluginModelDeserializer innerModelClass; private final BiFunction constructorFunction; - private final Supplier emptyInnerModelConstructor; + private final Supplier nullSettingsModelSupplier; protected AbstractPluginModelDeserializer( final Class valueClass, final Class innerModelClass, final BiFunction constructorFunction, - final Supplier emptyInnerModelConstructor) { + final Supplier nullSettingsModelSupplier) { super(valueClass); this.innerModelClass = innerModelClass; this.constructorFunction = constructorFunction; - this.emptyInnerModelConstructor = emptyInnerModelConstructor; + this.nullSettingsModelSupplier = nullSettingsModelSupplier; } @Override - public PluginModel deserialize(final JsonParser jsonParser, final DeserializationContext context) throws IOException, JacksonException { - final JsonNode node = jsonParser.getCodec().readTree(jsonParser); - - final Iterator> fields = node.fields(); - final Map.Entry onlyField = fields.next(); - - final String pluginName = onlyField.getKey(); - final JsonNode value = onlyField.getValue(); - - M innerModel = SERIALIZER_OBJECT_MAPPER.convertValue(value, innerModelClass); - if(innerModel == null) - innerModel = emptyInnerModelConstructor.get(); - + public PluginModel deserialize(final JsonParser jsonParser, final DeserializationContext context) throws IOException { + final ObjectMapper mapper = (ObjectMapper) jsonParser.getCodec(); + + jsonParser.nextToken(); + + final String pluginName = jsonParser.currentName(); + jsonParser.nextToken(); + + boolean isNull = false; + Map data = null; + if (jsonParser.currentToken() == JsonToken.START_OBJECT) { + data = mapper.readValue(jsonParser, Map.class); + // readValue consumed up to the inner END_OBJECT; advance to the outer END_OBJECT + jsonParser.nextToken(); + } else if (jsonParser.currentToken() == JsonToken.VALUE_NULL) { + // null or no-value (stdout: null / stdout:) -> preserve as null settings + isNull = true; + // advance to the outer END_OBJECT + jsonParser.nextToken(); + } else if (jsonParser.currentToken() == JsonToken.VALUE_STRING) { + final String value = jsonParser.getValueAsString(); + if (value.isEmpty()) { + throw context.weirdStringException(value, Map.class, + "Empty string is not allowed for plugin '" + pluginName + "'. Use null, empty (no value), or {} instead."); + } else { + throw context.weirdStringException(value, Map.class, + "String values not allowed for plugin '" + pluginName + "'"); + } + } else { + throw JsonMappingException.from(jsonParser, + "Unexpected value for plugin '" + pluginName + "': expected an object, null, or no value, but got " + + jsonParser.currentToken()); + } + + final M innerModel = isNull + ? nullSettingsModelSupplier.get() + : SERIALIZER_OBJECT_MAPPER.convertValue(data, innerModelClass); return constructorFunction.apply(pluginName, innerModel); } } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkModel.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkModel.java index 716255492f..0bb1b29a71 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkModel.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/configuration/SinkModel.java @@ -164,7 +164,8 @@ private static List validateKeys(List input, String tag) { static class SinkModelDeserializer extends AbstractPluginModelDeserializer { SinkModelDeserializer() { - super(SinkModel.class, SinkInternalJsonModel.class, SinkModel::new, () -> new SinkInternalJsonModel(null, null, null, null, null)); + super(SinkModel.class, SinkInternalJsonModel.class, SinkModel::new, + () -> new SinkInternalJsonModel(null, null, null, null, null, null)); } } } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java index b2a98308c1..086f73c1a5 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/PluginModelTests.java @@ -26,6 +26,7 @@ import java.io.Reader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -33,7 +34,9 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasKey; +import static org.junit.jupiter.api.Assertions.assertThrows; class PluginModelTests { @@ -75,14 +78,14 @@ final void testSerialization_empty_plugin_to_YAML() throws JsonGenerationExcepti final String serialized = mapper.writeValueAsString(pluginModel); - InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_null.yaml"); + InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_with_empty_object.yaml"); assertThat(serialized, notNullValue()); - assertThat(serialized, equalTo(convertInputStreamToString(inputStream))); + assertThat(serialized, equalTo(stripComments(convertInputStreamToString(inputStream)))); } @ParameterizedTest - @ValueSource(strings = {"plugin_model_empty.yaml", "plugin_model_not_present.yaml", "plugin_model_null.yaml"}) + @ValueSource(strings = {"plugin_model_with_empty_object.yaml"}) final void deserialize_with_empty_inner(final String resourceName) throws IOException { final InputStream inputStream = PluginModelTests.class.getResourceAsStream(resourceName); @@ -94,6 +97,17 @@ final void deserialize_with_empty_inner(final String resourceName) throws IOExce assertThat(pluginModel.getPluginSettings().size(), equalTo(0)); } + @Test + final void deserialize_with_no_value_returns_null_settings() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_not_present.yaml"); + + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final PluginModel pluginModel = mapper.readValue(inputStream, PluginModel.class); + assertThat(pluginModel.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel.getPluginSettings(), equalTo(null)); + } + @Test final void testUsingCustomSerializerWithPluginSettings_noExceptions() throws JsonGenerationException, JsonMappingException, IOException { final PluginModel pluginModel = new PluginModel("customPlugin", validPluginSettings()); @@ -141,6 +155,116 @@ final void testUsingCustomDeserializer_with_array() throws JsonParseException, J assertThat(readValue.listOfPlugins.get(1).getPluginSettings().get("key2"), equalTo("value2")); } + @Test + final void testUsingCustomDeserializer_with_array_of_three_preserves_all_entries() throws JsonParseException, JsonMappingException, IOException { + InputStream inputStream = PluginModelTests.class.getResourceAsStream("/list_of_plugins_multiple_fields.yaml"); + + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final PluginHolder readValue = mapper.readValue(inputStream, PluginHolder.class); + assertThat(readValue, notNullValue()); + assertThat(readValue.listOfPlugins, notNullValue()); + assertThat(readValue.listOfPlugins.size(), equalTo(3)); + assertThat(readValue.listOfPlugins.get(0).getPluginName(), equalTo("customPluginA")); + assertThat(readValue.listOfPlugins.get(0).getPluginSettings().get("key1"), equalTo("value1")); + assertThat(readValue.listOfPlugins.get(1).getPluginName(), equalTo("customPluginB")); + assertThat(readValue.listOfPlugins.get(1).getPluginSettings().get("key2"), equalTo("value2")); + assertThat(readValue.listOfPlugins.get(2).getPluginName(), equalTo("customPluginC")); + assertThat(readValue.listOfPlugins.get(2).getPluginSettings().get("key3"), equalTo("value3")); + } + + @Test + final void testRoundTrip_withEmptyObject() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_with_empty_object.yaml"); + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final PluginModel pluginModel1 = mapper.readValue(inputStream, PluginModel.class); + assertThat(pluginModel1.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel1.getPluginSettings().size(), equalTo(0)); + + final String serialized = mapper.writeValueAsString(pluginModel1); + assertThat(serialized.contains("{}"), equalTo(true)); + + final PluginModel pluginModel2 = mapper.readValue(serialized, PluginModel.class); + assertThat(pluginModel2.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel2.getPluginSettings().size(), equalTo(0)); + } + + @Test + final void testRoundTrip_withNullValue() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_with_null.yaml"); + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + // explicit null input -> deserializes with null settings (preserves null) + final PluginModel pluginModel1 = mapper.readValue(inputStream, PluginModel.class); + assertThat(pluginModel1.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel1.getPluginSettings(), equalTo(null)); + + // null settings -> serializes back as null (round-trip preserved) + final String serialized = mapper.writeValueAsString(pluginModel1); + assertThat(serialized.contains("null"), equalTo(true)); + + final PluginModel pluginModel2 = mapper.readValue(serialized, PluginModel.class); + assertThat(pluginModel2.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel2.getPluginSettings(), equalTo(null)); + } + + @Test + final void testRoundTrip_withEmptyValue() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_with_empty_value.yaml"); + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + // no-value (customPlugin:) -> same as null -> deserializes with null settings + final PluginModel pluginModel1 = mapper.readValue(inputStream, PluginModel.class); + assertThat(pluginModel1.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel1.getPluginSettings(), equalTo(null)); + + // null settings -> serializes back as null (round-trip preserved) + final String serialized = mapper.writeValueAsString(pluginModel1); + assertThat(serialized.contains("null"), equalTo(true)); + + final PluginModel pluginModel2 = mapper.readValue(serialized, PluginModel.class); + assertThat(pluginModel2.getPluginName(), equalTo("customPlugin")); + assertThat(pluginModel2.getPluginSettings(), equalTo(null)); + } + + @Test + final void testDeserialize_emptyString_throwsException() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_empty_string.yaml"); + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final JsonMappingException exception = assertThrows( + JsonMappingException.class, + () -> mapper.readValue(inputStream, PluginModel.class) + ); + assertThat(exception.getMessage(), containsString("Empty string is not allowed")); + } + + @Test + final void testDeserialize_nonEmptyString_throwsException() throws IOException { + final String yaml = "customPlugin: someStringValue"; + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final JsonMappingException exception = assertThrows( + JsonMappingException.class, + () -> mapper.readValue(yaml, PluginModel.class) + ); + assertThat(exception.getMessage(), containsString("String values not allowed")); + } + + @ParameterizedTest + @ValueSource(strings = {"plugin_model_number_value.yaml", "plugin_model_boolean_value.yaml", "plugin_model_array_value.yaml"}) + final void testDeserialize_invalidTokenType_throwsException(final String resourceName) throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream(resourceName); + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final JsonMappingException exception = assertThrows( + JsonMappingException.class, + () -> mapper.readValue(inputStream, PluginModel.class) + ); + assertThat(exception.getMessage(), containsString("Unexpected value for plugin")); + } + static Map validPluginSettings() { final Map settings = new HashMap<>(); settings.put("key1", "value1"); @@ -160,4 +284,12 @@ static String convertInputStreamToString(InputStream inputStream) throws IOExcep return stringBuilder.toString(); } + static String stripComments(final String content) { + final String stripped = Arrays.stream(content.split("\n")) + .filter(line -> !line.startsWith("#")) + .collect(java.util.stream.Collectors.joining("\n")) + .replaceAll("^\n+", ""); + return content.endsWith("\n") ? stripped + "\n" : stripped; + } + } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SamplePipelineConfigurationTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SamplePipelineConfigurationTest.java new file mode 100644 index 0000000000..ccf53753c9 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SamplePipelineConfigurationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.model.configuration; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * End-to-end tests for sample pipeline YAML configurations, verifying that + * valid plugin config formats are accepted and invalid ones are rejected. + * + * Valid formats: null, empty value (no value after colon), {} + * Invalid format: "" (empty string) + */ +class SamplePipelineConfigurationTest { + + private static final String PIPELINE_NAME = "test-pipeline"; + + private ObjectMapper objectMapper; + + @BeforeEach + void setup() { + objectMapper = new ObjectMapper(new YAMLFactory()); + } + + @ParameterizedTest(name = "pipeline with plugin config [{0}] should deserialize successfully") + @ValueSource(strings = { + "sample_pipelines/sample_pipeline_plugin_null.yaml", + "sample_pipelines/sample_pipeline_plugin_empty_value.yaml", + "sample_pipelines/sample_pipeline_plugin_empty_object.yaml", + "sample_pipelines/sample_pipeline_plugin_with_settings.yaml" + }) + void deserialize_validPipeline_succeeds(final String resourcePath) throws IOException { + final InputStream inputStream = getClass().getResourceAsStream(resourcePath); + + final PipelinesDataFlowModel model = objectMapper.readValue(inputStream, PipelinesDataFlowModel.class); + + assertThat(model, notNullValue()); + assertThat(model.getPipelines(), notNullValue()); + assertThat(model.getPipelines().containsKey(PIPELINE_NAME), equalTo(true)); + + final PipelineModel pipeline = model.getPipelines().get(PIPELINE_NAME); + assertThat(pipeline.getSource(), notNullValue()); + assertThat(pipeline.getSource().getPluginName(), notNullValue()); + assertThat(pipeline.getSinks(), notNullValue()); + assertThat(pipeline.getSinks().size(), equalTo(1)); + } + + @Test + void deserialize_pipeline_withPluginSettings_hasCorrectValues() throws IOException { + final InputStream inputStream = getClass().getResourceAsStream("sample_pipelines/sample_pipeline_plugin_with_settings.yaml"); + + final PipelinesDataFlowModel model = objectMapper.readValue(inputStream, PipelinesDataFlowModel.class); + final PipelineModel pipeline = model.getPipelines().get(PIPELINE_NAME); + + // source: http with host/port settings + assertThat(pipeline.getSource().getPluginName(), equalTo("http")); + assertThat(pipeline.getSource().getPluginSettings().get("host"), equalTo("0.0.0.0")); + assertThat(pipeline.getSource().getPluginSettings().get("port"), equalTo(2021)); + + // processor: grok + assertThat(pipeline.getProcessors().get(0).getPluginName(), equalTo("grok")); + + // sink: opensearch with hosts/credentials + assertThat(pipeline.getSinks().get(0).getPluginName(), equalTo("opensearch")); + assertThat(pipeline.getSinks().get(0).getPluginSettings().containsKey("hosts"), equalTo(true)); + } + + @ParameterizedTest(name = "pipeline with null/empty plugin [{0}] should have empty plugin settings") + @ValueSource(strings = { + "sample_pipelines/sample_pipeline_plugin_null.yaml", + "sample_pipelines/sample_pipeline_plugin_empty_value.yaml", + "sample_pipelines/sample_pipeline_plugin_empty_object.yaml" + }) + void deserialize_pipeline_withEmptyPluginConfig_hasEmptySettings(final String resourcePath) throws IOException { + final InputStream inputStream = getClass().getResourceAsStream(resourcePath); + + final PipelinesDataFlowModel model = objectMapper.readValue(inputStream, PipelinesDataFlowModel.class); + final PipelineModel pipeline = model.getPipelines().get(PIPELINE_NAME); + + final Map sourceSettings = pipeline.getSource().getPluginSettings(); + assertThat(sourceSettings == null || sourceSettings.isEmpty(), equalTo(true)); + final Map processorSettings = pipeline.getProcessors().get(0).getPluginSettings(); + assertThat(processorSettings == null || processorSettings.isEmpty(), equalTo(true)); + final Map sinkSettings = pipeline.getSinks().get(0).getPluginSettings(); + assertThat(sinkSettings == null || sinkSettings.isEmpty(), equalTo(true)); + } + + @Test + void deserialize_pipeline_withEmptyStringPluginConfig_throwsException() { + final InputStream inputStream = getClass().getResourceAsStream("sample_pipelines/sample_pipeline_plugin_empty_string.yaml"); + + final JsonMappingException exception = assertThrows( + JsonMappingException.class, + () -> objectMapper.readValue(inputStream, PipelinesDataFlowModel.class) + ); + assertThat(exception.getMessage(), containsString("Empty string is not allowed")); + } +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkModelTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkModelTest.java index b89bdc7826..831e9c0fa5 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkModelTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/configuration/SinkModelTest.java @@ -121,7 +121,7 @@ void deserialize_with_any_pluginModel() throws IOException { } @ParameterizedTest - @ValueSource(strings = {"plugin_model_empty.yaml", "plugin_model_not_present.yaml", "plugin_model_null.yaml"}) + @ValueSource(strings = {"plugin_model_with_empty_object.yaml"}) final void deserialize_with_empty_inner(final String resourceName) throws IOException { final InputStream inputStream = PluginModelTests.class.getResourceAsStream(resourceName); @@ -133,6 +133,17 @@ final void deserialize_with_empty_inner(final String resourceName) throws IOExce assertThat(sinkModel.getPluginSettings().size(), equalTo(0)); } + @Test + final void deserialize_with_no_value_returns_null_settings() throws IOException { + final InputStream inputStream = PluginModelTests.class.getResourceAsStream("plugin_model_not_present.yaml"); + + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + final SinkModel sinkModel = mapper.readValue(inputStream, SinkModel.class); + assertThat(sinkModel.getPluginName(), equalTo("customPlugin")); + assertThat(sinkModel.getPluginSettings(), equalTo(null)); + } + @Test void serialize_with_just_pluginModel() throws IOException { final Map pluginSettings = new LinkedHashMap<>(); diff --git a/data-prepper-api/src/test/resources/list_of_plugins_multiple_fields.yaml b/data-prepper-api/src/test/resources/list_of_plugins_multiple_fields.yaml new file mode 100644 index 0000000000..43730ddaf3 --- /dev/null +++ b/data-prepper-api/src/test/resources/list_of_plugins_multiple_fields.yaml @@ -0,0 +1,16 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +listOfPlugins: + - customPluginA: + key1: value1 + - customPluginB: + key2: value2 + - customPluginC: + key3: value3 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_array_value.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_array_value.yaml new file mode 100644 index 0000000000..e01e89eb0d --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_array_value.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: [1, 2, 3] diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_boolean_value.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_boolean_value.yaml new file mode 100644 index 0000000000..32077427be --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_boolean_value.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: true diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty.yaml index c11e1f7c90..6bea6ea607 100644 --- a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty.yaml +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty.yaml @@ -1,2 +1,2 @@ --- -customPlugin: {} +customPlugin: diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty_string.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty_string.yaml new file mode 100644 index 0000000000..0b3ae03d6d --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_empty_string.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: "" diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_explicit_empty.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_explicit_empty.yaml new file mode 100644 index 0000000000..7712058004 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_explicit_empty.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: {} diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_number_value.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_number_value.yaml new file mode 100644 index 0000000000..c80c3cb3e8 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_number_value.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: 123 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_object.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_object.yaml new file mode 100644 index 0000000000..7712058004 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_object.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: {} diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_value.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_value.yaml new file mode 100644 index 0000000000..da92e68240 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_empty_value.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_null.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_null.yaml new file mode 100644 index 0000000000..d49e6c2164 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/plugin_model_with_null.yaml @@ -0,0 +1,10 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +--- +customPlugin: null diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_object.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_object.yaml new file mode 100644 index 0000000000..a8204d430e --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_object.yaml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +test-pipeline: + source: + stdin: {} + processor: + - uppercase_string: {} + sink: + - stdout: {} + workers: 1 + delay: 100 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_string.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_string.yaml new file mode 100644 index 0000000000..113d1f7f0f --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_string.yaml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +test-pipeline: + source: + stdin: "" + processor: + - uppercase_string: null + sink: + - stdout: null + workers: 1 + delay: 100 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_value.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_value.yaml new file mode 100644 index 0000000000..8f77612f98 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_empty_value.yaml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +test-pipeline: + source: + stdin: + processor: + - uppercase_string: + sink: + - stdout: + workers: 1 + delay: 100 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_null.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_null.yaml new file mode 100644 index 0000000000..ce15aa17e4 --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_null.yaml @@ -0,0 +1,18 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +test-pipeline: + source: + stdin: null + processor: + - uppercase_string: null + sink: + - stdout: null + workers: 1 + delay: 100 diff --git a/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_with_settings.yaml b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_with_settings.yaml new file mode 100644 index 0000000000..1ba0db072b --- /dev/null +++ b/data-prepper-api/src/test/resources/org/opensearch/dataprepper/model/configuration/sample_pipelines/sample_pipeline_plugin_with_settings.yaml @@ -0,0 +1,27 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +test-pipeline: + source: + http: + host: "0.0.0.0" + port: 2021 + processor: + - grok: + match: + message: + - "%{COMMONAPACHELOG}" + sink: + - opensearch: + hosts: + - "https://localhost:9200" + username: "admin" + password: "admin" + workers: 4 + delay: 100 diff --git a/data-prepper-api/src/test/resources/pipeline_with_extension.yaml b/data-prepper-api/src/test/resources/pipeline_with_extension.yaml index 5a6ec55450..360864996d 100644 --- a/data-prepper-api/src/test/resources/pipeline_with_extension.yaml +++ b/data-prepper-api/src/test/resources/pipeline_with_extension.yaml @@ -8,4 +8,4 @@ test-pipeline: sink: - testSink: null workers: 8 - delay: 50 \ No newline at end of file + delay: 50 diff --git a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-opensearch.expected.yaml b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-opensearch.expected.yaml index af770116f8..e06d50447e 100644 --- a/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-opensearch.expected.yaml +++ b/data-prepper-logstash-configuration/src/test/resources/org/opensearch/dataprepper/logstash/log-ingest-to-opensearch.expected.yaml @@ -12,7 +12,7 @@ logstash-converted-pipeline: match: log: - "%{COMBINEDAPACHELOG}" - - drop_events: null + - drop_events: {} - key_value: source: "message" destination: "test"