From 1f2d9d7a62f07f57243ea1092ed006ff06ce7e4b Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Mon, 10 Nov 2025 19:23:25 +0530 Subject: [PATCH 01/14] PLUGIN-1936: Initial Commit --- .../ServiceNowTableAPIClientImpl.java | 48 ++++++++++++++++++- .../servicenow/restapi/RestAPIClient.java | 3 +- .../servicenow/restapi/RestAPIResponse.java | 43 +++++++++++++++-- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 04df0178..2a69421b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -25,8 +25,11 @@ import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -55,8 +58,12 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -154,7 +161,9 @@ public List> fetchTableRecords( String accessToken = getAccessToken(); requestBuilder.setAuthHeader(accessToken); RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build()); - return parseResponseToResultListOfMap(apiResponse.getResponseBody()); + //return parseResponseToResultListOfMap(apiResponse.getResponseBody()); + return parseResponseStreamToResultListOfMap(apiResponse.getInputStream()); + } private void applyDateRangeToRequest(ServiceNowTableAPIRequestBuilder requestBuilder, String startDate, @@ -197,6 +206,43 @@ public List> parseResponseToResultListOfMap(String responseB return GSON.fromJson(ja, type); } + public List> parseResponseStreamToResultListOfMap(InputStream in) throws ServiceNowAPIException { + List> records = new ArrayList<>(); + // InputStream in = httpResponse.getEntity().getContent(); + try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8); + JsonReader jsonReader = new JsonReader(reader)) { + jsonReader.setLenient(true); + jsonReader.beginObject(); + while (jsonReader.hasNext()) { + String name = jsonReader.nextName(); + if (ServiceNowConstants.RESULT.equals(name) && jsonReader.peek() == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + while (jsonReader.hasNext()) { + jsonReader.beginObject(); + Map record = new HashMap<>(); + while (jsonReader.hasNext()) { + String field = jsonReader.nextName(); + JsonToken token = jsonReader.peek(); + // JsonElement resultElement = GSON.fromJson(jsonReader, JsonElement.class); + // responseBody = resultElement.toString(); + record.put(field, token == JsonToken.NULL ? null : jsonReader.nextString()); + } + jsonReader.endObject(); + records.add(record); + } + jsonReader.endArray(); + } else { + // skip other fields (e.g., metadata like result_count) + jsonReader.skipValue(); + } + } + jsonReader.endObject(); + return records; + } catch (IOException e) { + throw new ServiceNowAPIException(e, null); + } + } + private String getErrorMessage(String responseBody) { try { JsonObject jo = GSON.fromJson(responseBody, JsonObject.class); diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index 7329aaaf..5be6fffc 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; import java.net.SocketException; import java.util.Collections; import java.util.concurrent.Callable; @@ -82,7 +83,7 @@ public RestAPIResponse executeGet(RestAPIRequest request) throws IOException { } } catch (ConnectTimeoutException | SocketException e) { ServiceNowAPIException exception = new ServiceNowAPIException(e, null); - return new RestAPIResponse(Collections.emptyMap(), null, exception); + return new RestAPIResponse(Collections.emptyMap(), (InputStream) null, exception); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index ceeab972..15a88902 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -17,21 +17,28 @@ package io.cdap.plugin.servicenow.restapi; import com.google.gson.Gson; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.stream.JsonReader; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.Header; +import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -46,9 +53,14 @@ public class RestAPIResponse { private static final Set SUCCESS_CODES = new HashSet<>(Arrays.asList(HttpStatus.SC_CREATED, HttpStatus.SC_OK)); private final Map headers; - private final String responseBody; + // Deprecated: storing full body as String can cause OOM + @Deprecated + private String responseBody; @Nullable private final ServiceNowAPIException exception; + // New: store InputStream for streaming consumption + private InputStream inputStream; + public RestAPIResponse( Map headers, @Nullable String responseBody, @@ -58,6 +70,15 @@ public RestAPIResponse( this.exception = exception; } + public RestAPIResponse( + Map headers, + InputStream inputStream, + @Nullable ServiceNowAPIException exception) { + this.headers = headers; + this.inputStream = inputStream; + this.exception = exception; + } + /** * Parses HttpResponse into RestAPIResponse object when no errors occur. * Throws a {@link ServiceNowAPIException}. @@ -80,17 +101,27 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN ServiceNowAPIException serviceNowAPIException = validateHttpResponse(httpResponse); if (serviceNowAPIException != null) { - return new RestAPIResponse(headers, null, serviceNowAPIException); + return new RestAPIResponse(headers, (InputStream) null, serviceNowAPIException); } String responseBody = null; try { responseBody = EntityUtils.toString(httpResponse.getEntity()); } catch (IOException e) { - return new RestAPIResponse(headers, null, new ServiceNowAPIException(e, httpResponse)); + return new RestAPIResponse(headers, (String) null, new ServiceNowAPIException(e, httpResponse)); + } + // Instead of reading the entire entity, store the stream + HttpEntity httpEntity = httpResponse.getEntity(); + InputStream responseStream; + try { + responseStream = (httpEntity != null) ? httpEntity.getContent() : null; + } catch (IOException e) { + return new RestAPIResponse(headers, (InputStream) null, new ServiceNowAPIException(e, httpResponse)); } serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); - return new RestAPIResponse(headers, responseBody, serviceNowAPIException); + // return new RestAPIResponse(headers, responseBody, serviceNowAPIException); + return new RestAPIResponse(headers, responseStream, serviceNowAPIException); + } public static RestAPIResponse parse(HttpResponse httpResponse) throws IOException { @@ -131,6 +162,10 @@ public String getResponseBody() { return responseBody; } + public InputStream getInputStream() { + return inputStream; + } + @Nullable public ServiceNowAPIException getException() { return exception; From 42351538f8638041a6b16a43bb5f85aa479254b7 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Tue, 18 Nov 2025 09:17:12 +0530 Subject: [PATCH 02/14] PLUGIN-1936: Refactor code to implement streaming logic --- .../ServiceNowTableAPIClientImpl.java | 25 ++- .../servicenow/restapi/RestAPIResponse.java | 10 +- .../source/ServiceNowMultiRecordReader.java | 13 +- .../source/ServiceNowRecordReader.java | 191 ++++++++++++++++-- .../ServiceNowTableAPIClientImplTest.java | 15 +- .../connector/ServiceNowConnectorTest.java | 2 +- .../sink/ServiceNowRecordWriterTest.java | 6 +- .../sink/ServiceNowSinkConfigTest.java | 4 +- .../servicenow/sink/ServiceNowSinkTest.java | 4 +- .../source/ServiceNowInputFormatTest.java | 6 +- .../ServiceNowMultiSourceConfigTest.java | 8 +- .../source/ServiceNowMultiSourceTest.java | 6 +- .../source/ServiceNowRecordReaderTest.java | 10 +- .../source/ServiceNowSourceConfigTest.java | 2 +- .../source/ServiceNowSourceTest.java | 6 +- 15 files changed, 241 insertions(+), 67 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 2a69421b..9d30d3fe 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -138,7 +138,7 @@ public String getAccessTokenRetryableMode() throws ExecutionException, RetryExce * @param limit The number of records to be fetched * @return The list of Map; each Map representing a table row */ - public List> fetchTableRecords( + public RestAPIResponse fetchTableRecords( String tableName, SourceValueType valueType, String startDate, @@ -161,8 +161,9 @@ public List> fetchTableRecords( String accessToken = getAccessToken(); requestBuilder.setAuthHeader(accessToken); RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build()); + return apiResponse; //return parseResponseToResultListOfMap(apiResponse.getResponseBody()); - return parseResponseStreamToResultListOfMap(apiResponse.getInputStream()); + // return parseResponseStreamToRecord(apiResponse.getInputStream()); } @@ -206,20 +207,20 @@ public List> parseResponseToResultListOfMap(String responseB return GSON.fromJson(ja, type); } - public List> parseResponseStreamToResultListOfMap(InputStream in) throws ServiceNowAPIException { - List> records = new ArrayList<>(); + public Map parseResponseStreamToRecord(InputStream in) throws ServiceNowAPIException { + // List> records = new ArrayList<>(); // InputStream in = httpResponse.getEntity().getContent(); try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8); JsonReader jsonReader = new JsonReader(reader)) { jsonReader.setLenient(true); jsonReader.beginObject(); + Map record = new HashMap<>(); while (jsonReader.hasNext()) { String name = jsonReader.nextName(); if (ServiceNowConstants.RESULT.equals(name) && jsonReader.peek() == JsonToken.BEGIN_ARRAY) { jsonReader.beginArray(); while (jsonReader.hasNext()) { jsonReader.beginObject(); - Map record = new HashMap<>(); while (jsonReader.hasNext()) { String field = jsonReader.nextName(); JsonToken token = jsonReader.peek(); @@ -228,7 +229,7 @@ public List> parseResponseStreamToResultListOfMap(InputStrea record.put(field, token == JsonToken.NULL ? null : jsonReader.nextString()); } jsonReader.endObject(); - records.add(record); + // records.add(record); } jsonReader.endArray(); } else { @@ -237,7 +238,7 @@ public List> parseResponseStreamToResultListOfMap(InputStrea } } jsonReader.endObject(); - return records; + return record; } catch (IOException e) { throw new ServiceNowAPIException(e, null); } @@ -277,12 +278,14 @@ private String getErrorMessage(String responseBody) { * @param limit The number of records to be fetched * @return The list of Map; each Map representing a table row */ - public List> fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, + public RestAPIResponse fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, String startDate, String endDate, int offset, int limit) throws ServiceNowAPIException { - final List> results = new ArrayList<>(); + //final List> results = new ArrayList<>(); + final RestAPIResponse[] restAPIResponse = new RestAPIResponse[1]; Callable fetchRecords = () -> { - results.addAll(fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit)); + // results.addAll(fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit)); + restAPIResponse[0] = fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit); return true; }; @@ -300,7 +303,7 @@ public List> fetchTableRecordsRetryableMode(String tableName e, null, false); } - return results; + return restAPIResponse[0]; } /** diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index 15a88902..c53c6ac2 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -64,9 +64,11 @@ public class RestAPIResponse { public RestAPIResponse( Map headers, @Nullable String responseBody, + InputStream inputStream, @Nullable ServiceNowAPIException exception) { this.headers = headers; this.responseBody = responseBody; + this.inputStream = inputStream; this.exception = exception; } @@ -101,14 +103,14 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN ServiceNowAPIException serviceNowAPIException = validateHttpResponse(httpResponse); if (serviceNowAPIException != null) { - return new RestAPIResponse(headers, (InputStream) null, serviceNowAPIException); + return new RestAPIResponse(headers, null, null, serviceNowAPIException); } String responseBody = null; try { responseBody = EntityUtils.toString(httpResponse.getEntity()); } catch (IOException e) { - return new RestAPIResponse(headers, (String) null, new ServiceNowAPIException(e, httpResponse)); + return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); } // Instead of reading the entire entity, store the stream HttpEntity httpEntity = httpResponse.getEntity(); @@ -116,11 +118,11 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN try { responseStream = (httpEntity != null) ? httpEntity.getContent() : null; } catch (IOException e) { - return new RestAPIResponse(headers, (InputStream) null, new ServiceNowAPIException(e, httpResponse)); + return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); } serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); // return new RestAPIResponse(headers, responseBody, serviceNowAPIException); - return new RestAPIResponse(headers, responseStream, serviceNowAPIException); + return new RestAPIResponse(headers, responseBody, responseStream, serviceNowAPIException); } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index d10d1bc7..ddfc85ef 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -22,6 +22,7 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -91,14 +92,14 @@ public StructuredRecord getCurrentValue() throws IOException { } @VisibleForTesting - void fetchData() throws ServiceNowAPIException { + RestAPIResponse fetchData() throws ServiceNowAPIException { // Get the table data - results = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(), - multiSourcePluginConf.getStartDate(), - multiSourcePluginConf.getEndDate(), split.getOffset(), - multiSourcePluginConf.getPageSize()); + RestAPIResponse restAPIResponse = restApi.fetchTableRecordsRetryableMode(tableName, + multiSourcePluginConf.getValueType(), multiSourcePluginConf.getStartDate(), multiSourcePluginConf.getEndDate(), + split.getOffset(), multiSourcePluginConf.getPageSize()); - iterator = results.iterator(); + // iterator = results.iterator(); + return restAPIResponse; } private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 41d58c7c..b9651348 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -16,11 +16,17 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import io.cdap.plugin.servicenow.util.SourceQueryMode; import org.apache.hadoop.mapreduce.InputSplit; @@ -28,8 +34,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Record reader that reads the entire contents of a ServiceNow table. @@ -38,6 +50,9 @@ public class ServiceNowRecordReader extends ServiceNowBaseRecordReader { private static final Logger LOG = LoggerFactory.getLogger(ServiceNowRecordReader.class); private final ServiceNowSourceConfig pluginConf; private ServiceNowTableAPIClientImpl restApi; + private final Gson gson = new Gson(); + private final Type mapType = new TypeToken>() { }.getType(); + private JsonReader jsonReader = null; public ServiceNowRecordReader(ServiceNowSourceConfig pluginConf) { super(); @@ -62,25 +77,111 @@ public void initialize(InputSplit split, Schema schema) { initializeSchema(tableName, schema); } + /** + * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. + * Returns true when it assigned `row` to the next Map record. + * Returns false only when there are no more pages/records (i.e., openNextPage() fails). + */ @Override public boolean nextKeyValue() throws IOException { - try { - if (results == null) { - fetchData(); + while (true) { + // Ensure we have an active page/jsonReader + if (jsonReader == null) { + // Need to open the next page + boolean pageOpened; + try { + pageOpened = openNextPage(); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue" + tableName, e); + } + if (!pageOpened) { + // No more pages + return false; + } } - if (!iterator.hasNext()) { - return false; + // At this point jsonReader is positioned inside the "result" array. + JsonToken token = jsonReader.peek(); + + if (token == JsonToken.END_ARRAY) { + // Current page exhausted. Close current page and try to open the next page. + closeCurrentPage(); + + // Attempt to open next page; if none, we must return false (end of data) + boolean openedNext; + try { + openedNext = openNextPage(); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue " + tableName, e); + } + if (!openedNext) { + // No more pages + return false; + } + // continue loop to attempt to read from newly opened page } - row = iterator.next(); + if (token == JsonToken.BEGIN_OBJECT) { + // Read exactly one object from the stream into a Map + Map recordMap = gson.fromJson(jsonReader, mapType); + this.row = recordMap; // assign row — getCurrentValue() will expose it + pos++; + return true; + } - pos++; + if (token == JsonToken.NULL) { + // skip nulls if any and continue + jsonReader.nextNull(); + continue; + } + + // Skip any unexpected or non-object token and loop + jsonReader.skipValue(); + + /*// read the next record from the current page + try { + if (jsonreader.hasnext()) { + // there is another record in the current page + map recordmap = new hashmap<>(); + jsonreader.beginobject(); + while (jsonreader.hasnext()) { + string name = jsonreader.nextname(); + jsontoken token = jsonreader.peek(); + string value = null; + if (token == jsontoken.null) { + jsonreader.nextnull(); + } else { + value = jsonreader.nextstring(); + } + recordmap.put(name, value); + } + jsonreader.endobject(); + row = recordmap; + pos++; + return true; + } else { + // end of current page + closecurrentpage(); + } + } catch (ioexception e) { + // cleanup on parse error + closecurrentpage(); + log.error("error parsing json response from table " + tablename, e); + throw e; + }*/ + } + + /*try { + InputStream inputStream = fetchData(); + do { + row = restApi.parseResponseStreamToRecord(inputStream); + pos++; + return true; + } while (inputStream.available() > 0); } catch (Exception e) { LOG.error("Error in nextKeyValue", e); throw new IOException("Exception in nextKeyValue", e); - } - return true; + }*/ } @Override @@ -103,14 +204,74 @@ public StructuredRecord getCurrentValue() throws IOException { return recordBuilder.build(); } - private void fetchData() throws ServiceNowAPIException { + private RestAPIResponse fetchData() throws ServiceNowAPIException { // Get the table data - results = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), - pluginConf.getEndDate(), split.getOffset(), - pluginConf.getPageSize()); - LOG.debug("Results size={}", results.size()); + RestAPIResponse restAPIResponse = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), + pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), pluginConf.getPageSize()); + + + // LOG.debug("Results size={}", results.size()); + return restAPIResponse; - iterator = results.iterator(); + // iterator = results.iterator(); + // iterator = record; + } + + private boolean openNextPage() throws IOException, ServiceNowAPIException { + closeCurrentPage(); + RestAPIResponse resp = fetchData(); + InputStream in = resp.getInputStream(); + if (in == null) { + return false; + } + this.jsonReader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + this.jsonReader.setLenient(true); + + // Position the reader to the "result" array: { "result": [ ... ], ... } + try { + JsonToken top = jsonReader.peek(); + if (top == JsonToken.BEGIN_OBJECT) { + jsonReader.beginObject(); + while (jsonReader.hasNext()) { + String name = jsonReader.nextName(); + if (name.equals(ServiceNowConstants.RESULT)) { + if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + return true; + } else { + jsonReader.skipValue(); + break; + } + } else { + jsonReader.skipValue(); + } + } // if we fall through, "result" not found as array + } else if (top == JsonToken.BEGIN_ARRAY) { + // Just in case response is an array (very unlikely for ServiceNow), handle it. + jsonReader.beginArray(); + return true; + } + } catch (IOException e) { + // cleanup on parse error + closeCurrentPage(); + throw e; + } + + // No "result" array not found, close the current page and return false + closeCurrentPage(); + return false; + } + + public void closeCurrentPage() { + if (this.jsonReader != null) { + try { + this.jsonReader.close(); + } catch (IOException e) { + LOG.warn("Error closing JSON reader", e); + } finally { + this.jsonReader = null; + } + } } protected void initialize(InputSplit split) { diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java index 65902ad9..98199840 100644 --- a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -29,6 +29,8 @@ public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws Servi ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); + RestAPIResponse mockApiResponse = new RestAPIResponse( + Collections.emptyMap(), "", null, null); List> mockResults = new ArrayList<>(); mockResults.add(new HashMap() {{ put("keyTest", "valueTest"); @@ -45,7 +47,7 @@ public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws Servi Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()); - List> receivedResults = + RestAPIResponse restAPIResponse = implSpy.fetchTableRecordsRetryableMode( "test", SourceValueType.SHOW_DISPLAY_VALUE, "", "", 0, 0); Mockito.verify(implSpy, Mockito.times(2)).fetchTableRecords( @@ -55,7 +57,7 @@ public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws Servi Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()); - Assert.assertEquals(receivedResults, mockResults); + Assert.assertEquals(restAPIResponse, mockApiResponse); } @Test @@ -104,7 +106,8 @@ public void testFetchTableSchema_ActualValueType() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, + null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.SCHEMA_API_BASED); @@ -136,7 +139,7 @@ public void testFetchTableSchema_GlideTimeFieldWithActualValueType() throws Exce " }\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("u_custom_table", @@ -175,7 +178,7 @@ public void testFetchTableSchema_DisplayValueType() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.SCHEMA_API_BASED); @@ -212,7 +215,7 @@ public void testFetchTableSchema_StcFieldsWithDisplayValueType_ParseAsString() t " }\n" + " }\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("incident", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.METADATA_API_BASED); diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index 2240e8b5..c16a4a67 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -131,7 +131,7 @@ public void testGenerateSpec() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index 259e4582..3a35e825 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -99,7 +99,7 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { result.add(map); Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse( - headers, responseBody, null); + headers, responseBody, null, null); Mockito.when(restApi.executePost(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -143,7 +143,7 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { map.put("key", "value"); result.add(map); Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -191,7 +191,7 @@ public void testWriteWithUnservicedRequests() throws Exception { map.put("key", "value"); result.add(map); Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index 04ba38e3..df767840 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -306,7 +306,7 @@ public void testValidateSchema() throws Exception { Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); RestAPIResponse restAPIResponse = new RestAPIResponse( - headers, responseBody, new ServiceNowAPIException("", mockResponse)); + headers, responseBody, null, new ServiceNowAPIException("", mockResponse)); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -362,7 +362,7 @@ public void testValidateSchemaWithOperation() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index 3f748d61..cde2a205 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -93,7 +93,7 @@ public void testConfigurePipeline() throws Exception { " \"result\": []\n" + "}"; MockFailureCollector collector = new MockFailureCollector(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowSink.configurePipeline(mockPipelineConfigurer); @@ -130,7 +130,7 @@ public void testPrepareRun() throws Exception { Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE))); Emitter> emitter = Mockito.mock(Emitter.class); Mockito.when(context.getInputSchema()).thenReturn(schema); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index d206458c..771e4b62 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -142,7 +142,7 @@ public void testFetchTableInfo() throws Exception { "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("table", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); @@ -246,7 +246,7 @@ public void testFetchTableInfoReportingMode() throws Exception { "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("proc_po", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); @@ -291,7 +291,7 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index c0a0302a..7ad89592 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -174,7 +174,7 @@ public void testValidate() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); @@ -207,7 +207,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); @@ -301,7 +301,7 @@ public void testValidateReferenceName() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { @@ -402,7 +402,7 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index 033f05ae..4e6e8afc 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -156,7 +156,7 @@ public void testConfigurePipeline() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); @@ -176,7 +176,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { @@ -280,7 +280,7 @@ public void testPrepareRun() throws Exception { "}"; PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); Mockito.when(ServiceNowMultiInputFormat.setInput(Mockito.any(), Mockito.any())).thenReturn((tableInfo)); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index 3366f527..9ee945de 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -25,6 +25,7 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceQueryMode; @@ -277,11 +278,12 @@ public void testFetchData() throws Exception { response.setColumns(columns); response.setResult(results); response.setTotalRecordCount(1); + RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); Mockito.when(restApi.fetchTableSchema(tableName, valueType)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); @@ -330,11 +332,12 @@ public void testFetchDataReportingMode() throws Exception { response.setColumns(columns); response.setResult(results); response.setTotalRecordCount(1); + RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); @@ -362,11 +365,12 @@ public void testFetchDataOnInvalidTable() throws Exception { ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); List> results = new ArrayList<>(); + RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(results); + serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index d9bc48c6..af2716a6 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -624,7 +624,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); config.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 29871ad1..d3445a84 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -171,7 +171,7 @@ public void testConfigurePipeline() throws Exception { PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(tableInfo); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -209,7 +209,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { @@ -297,7 +297,7 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); PowerMockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); From 80033a388575d5dd6bc162b5abac80a318f3c340 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Mon, 24 Nov 2025 21:11:49 +0530 Subject: [PATCH 03/14] PLUGIN-1936: Fix unexpected closure of stream issue --- .../servicenow/restapi/RestAPIResponse.java | 46 ++++--- .../source/ServiceNowRecordReader.java | 120 ++++++++++-------- 2 files changed, 100 insertions(+), 66 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index c53c6ac2..ff21f2d4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -17,20 +17,18 @@ package io.cdap.plugin.servicenow.restapi; import com.google.gson.Gson; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.stream.JsonReader; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.util.EntityUtils; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; @@ -38,7 +36,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -105,25 +102,42 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN if (serviceNowAPIException != null) { return new RestAPIResponse(headers, null, null, serviceNowAPIException); } - - String responseBody = null; - try { + /*try { responseBody = EntityUtils.toString(httpResponse.getEntity()); } catch (IOException e) { return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); - } - // Instead of reading the entire entity, store the stream - HttpEntity httpEntity = httpResponse.getEntity(); - InputStream responseStream; + }*/ try { - responseStream = (httpEntity != null) ? httpEntity.getContent() : null; + return prepareResponseWithBodyAndStream(httpResponse, headers, serviceNowAPIException); } catch (IOException e) { return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); } - serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); - // return new RestAPIResponse(headers, responseBody, serviceNowAPIException); - return new RestAPIResponse(headers, responseBody, responseStream, serviceNowAPIException); + } + public static RestAPIResponse prepareResponseWithBodyAndStream(HttpResponse httpResponse, Map headers, + ServiceNowAPIException serviceNowAPIException) throws IOException { + HttpEntity httpEntity = httpResponse.getEntity(); + if (httpEntity != null) { + try (InputStream inputStream = httpEntity.getContent(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { + + // Copy the InputStream into the ByteArrayOutputStream + byte[] data = new byte[8192]; + int bytesRead; + while ((bytesRead = inputStream.read(data)) != -1) { + buffer.write(data, 0, bytesRead); + } + // Convert the buffer to a String for the responseBody + String responseBody = buffer.toString(String.valueOf(StandardCharsets.UTF_8)); + serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); + // Create a new InputStream from the buffer for further processing + InputStream reusableStream = new ByteArrayInputStream(buffer.toByteArray()); + // return new RestAPIResponse(headers, responseBody, serviceNowAPIException); + return new RestAPIResponse(headers, responseBody, reusableStream, serviceNowAPIException); + } + } else { + return new RestAPIResponse(headers, null, null, serviceNowAPIException); + } } public static RestAPIResponse parse(HttpResponse httpResponse) throws IOException { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index b9651348..8d0ec282 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -80,64 +80,67 @@ public void initialize(InputSplit split, Schema schema) { /** * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. * Returns true when it assigned `row` to the next Map record. - * Returns false only when there are no more pages/records (i.e., openNextPage() fails). + * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). */ @Override public boolean nextKeyValue() throws IOException { - while (true) { - // Ensure we have an active page/jsonReader - if (jsonReader == null) { - // Need to open the next page - boolean pageOpened; - try { - pageOpened = openNextPage(); - } catch (ServiceNowAPIException e) { - throw new IOException("Exception in nextKeyValue" + tableName, e); - } - if (!pageOpened) { - // No more pages - return false; - } + // Ensure we have an active page/jsonReader + if (jsonReader == null) { + // Need to open the next page + boolean pageOpened; + try { + pageOpened = openNextPage(); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue" + tableName, e); + } + if (!pageOpened) { + // No more pages + return false; } + } - // At this point jsonReader is positioned inside the "result" array. - JsonToken token = jsonReader.peek(); + // At this point jsonReader is positioned inside the "result" array. + JsonToken token = jsonReader.peek(); - if (token == JsonToken.END_ARRAY) { - // Current page exhausted. Close current page and try to open the next page. - closeCurrentPage(); + if (token == JsonToken.END_ARRAY) { + // Current page exhausted. Close current page and try to open the next page. + closeCurrentPage(); - // Attempt to open next page; if none, we must return false (end of data) - boolean openedNext; - try { - openedNext = openNextPage(); - } catch (ServiceNowAPIException e) { - throw new IOException("Exception in nextKeyValue " + tableName, e); - } - if (!openedNext) { - // No more pages - return false; - } - // continue loop to attempt to read from newly opened page + // Attempt to open next page; if none, we must return false (end of data) + boolean openedNext; + try { + openedNext = openNextPage(); + LOG.info("Opened next page for table {} at offset {}: {}", tableName, split.getOffset(), openedNext); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue " + tableName, e); } - - if (token == JsonToken.BEGIN_OBJECT) { - // Read exactly one object from the stream into a Map - Map recordMap = gson.fromJson(jsonReader, mapType); - this.row = recordMap; // assign row — getCurrentValue() will expose it - pos++; - return true; + if (!openedNext) { + // No more pages + return false; } + // continue loop to attempt to read from newly opened page + } - if (token == JsonToken.NULL) { - // skip nulls if any and continue - jsonReader.nextNull(); - continue; - } + if (token == JsonToken.BEGIN_OBJECT) { + LOG.info("Reading record object for table {} at position {}", tableName, pos); + // Read exactly one object from the stream into a Map + Map recordMap = gson.fromJson(jsonReader, mapType); + this.row = recordMap; // assign row + pos++; + return true; + } + return false; + + /*if (token == JsonToken.NULL) { + // skip nulls if any and continue + jsonReader.nextNull(); + continue; + }*/ // Skip any unexpected or non-object token and loop - jsonReader.skipValue(); + //jsonReader.skipValue(); + /*********************----------------**********************/ /*// read the next record from the current page try { if (jsonreader.hasnext()) { @@ -169,7 +172,6 @@ public boolean nextKeyValue() throws IOException { log.error("error parsing json response from table " + tablename, e); throw e; }*/ - } /*try { InputStream inputStream = fetchData(); @@ -218,8 +220,11 @@ private RestAPIResponse fetchData() throws ServiceNowAPIException { } private boolean openNextPage() throws IOException, ServiceNowAPIException { + LOG.info("Opening next page for table {} at offset {}", tableName, split.getOffset()); closeCurrentPage(); + LOG.info("Fetching data for table {} at offset {}", tableName, split.getOffset()); RestAPIResponse resp = fetchData(); + LOG.info("Fetched data for table {} at offset {}", tableName, split.getOffset()); InputStream in = resp.getInputStream(); if (in == null) { return false; @@ -229,7 +234,15 @@ private boolean openNextPage() throws IOException, ServiceNowAPIException { // Position the reader to the "result" array: { "result": [ ... ], ... } try { - JsonToken top = jsonReader.peek(); + JsonToken top; + try { + top = jsonReader.peek(); + LOG.info("Peeking JSON token for table {}: {}", tableName, top); + } catch (IOException e) { + LOG.warn("Unexpected closure of stream while peeking JSON token for table {}", tableName, e); + closeCurrentPage(); + return false; + } if (top == JsonToken.BEGIN_OBJECT) { jsonReader.beginObject(); while (jsonReader.hasNext()) { @@ -245,14 +258,20 @@ private boolean openNextPage() throws IOException, ServiceNowAPIException { } else { jsonReader.skipValue(); } - } // if we fall through, "result" not found as array + } } else if (top == JsonToken.BEGIN_ARRAY) { - // Just in case response is an array (very unlikely for ServiceNow), handle it. jsonReader.beginArray(); return true; + } else if (jsonReader.peek() == JsonToken.END_ARRAY) { + // empty result array — treat as no-more-data for this split/page + LOG.info("openNextPage: found empty result array (no records). Closing and returning false."); + // consume the END_ARRAY token to leave stream consistent (optional) + jsonReader.endArray(); + // cleanup + closeCurrentPage(); + return false; } } catch (IOException e) { - // cleanup on parse error closeCurrentPage(); throw e; } @@ -263,6 +282,7 @@ private boolean openNextPage() throws IOException, ServiceNowAPIException { } public void closeCurrentPage() { + LOG.info("Closing current page for table {}", tableName); if (this.jsonReader != null) { try { this.jsonReader.close(); From 5e99944fcee16094ec75d78958bf09e4af255cfd Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Mon, 24 Nov 2025 22:19:15 +0530 Subject: [PATCH 04/14] PLUGIN-1936 : Fix duplicate record issue --- .../source/ServiceNowRecordReader.java | 59 +------------------ 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 8d0ec282..31d2551b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -102,7 +102,7 @@ public boolean nextKeyValue() throws IOException { // At this point jsonReader is positioned inside the "result" array. JsonToken token = jsonReader.peek(); - if (token == JsonToken.END_ARRAY) { + /*if (token == JsonToken.END_ARRAY) { // Current page exhausted. Close current page and try to open the next page. closeCurrentPage(); @@ -119,7 +119,7 @@ public boolean nextKeyValue() throws IOException { return false; } // continue loop to attempt to read from newly opened page - } + }*/ if (token == JsonToken.BEGIN_OBJECT) { LOG.info("Reading record object for table {} at position {}", tableName, pos); @@ -129,61 +129,8 @@ public boolean nextKeyValue() throws IOException { pos++; return true; } + closeCurrentPage(); return false; - - /*if (token == JsonToken.NULL) { - // skip nulls if any and continue - jsonReader.nextNull(); - continue; - }*/ - - // Skip any unexpected or non-object token and loop - //jsonReader.skipValue(); - - /*********************----------------**********************/ - /*// read the next record from the current page - try { - if (jsonreader.hasnext()) { - // there is another record in the current page - map recordmap = new hashmap<>(); - jsonreader.beginobject(); - while (jsonreader.hasnext()) { - string name = jsonreader.nextname(); - jsontoken token = jsonreader.peek(); - string value = null; - if (token == jsontoken.null) { - jsonreader.nextnull(); - } else { - value = jsonreader.nextstring(); - } - recordmap.put(name, value); - } - jsonreader.endobject(); - row = recordmap; - pos++; - return true; - } else { - // end of current page - closecurrentpage(); - } - } catch (ioexception e) { - // cleanup on parse error - closecurrentpage(); - log.error("error parsing json response from table " + tablename, e); - throw e; - }*/ - - /*try { - InputStream inputStream = fetchData(); - do { - row = restApi.parseResponseStreamToRecord(inputStream); - pos++; - return true; - } while (inputStream.available() > 0); - } catch (Exception e) { - LOG.error("Error in nextKeyValue", e); - throw new IOException("Exception in nextKeyValue", e); - }*/ } @Override From cf899073c5d48e498e2421d9b5944025a318aa08 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Tue, 25 Nov 2025 10:32:52 +0530 Subject: [PATCH 05/14] PLUGIN-1936 : Remove commented code and unused imports --- .../ServiceNowTableAPIClientImpl.java | 1 - .../source/ServiceNowRecordReader.java | 20 ------------------- 2 files changed, 21 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 9d30d3fe..38659f46 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -25,7 +25,6 @@ import com.google.common.base.Strings; import com.google.gson.Gson; import com.google.gson.JsonArray; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 31d2551b..aaf0e7d6 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -39,7 +39,6 @@ import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -102,25 +101,6 @@ public boolean nextKeyValue() throws IOException { // At this point jsonReader is positioned inside the "result" array. JsonToken token = jsonReader.peek(); - /*if (token == JsonToken.END_ARRAY) { - // Current page exhausted. Close current page and try to open the next page. - closeCurrentPage(); - - // Attempt to open next page; if none, we must return false (end of data) - boolean openedNext; - try { - openedNext = openNextPage(); - LOG.info("Opened next page for table {} at offset {}: {}", tableName, split.getOffset(), openedNext); - } catch (ServiceNowAPIException e) { - throw new IOException("Exception in nextKeyValue " + tableName, e); - } - if (!openedNext) { - // No more pages - return false; - } - // continue loop to attempt to read from newly opened page - }*/ - if (token == JsonToken.BEGIN_OBJECT) { LOG.info("Reading record object for table {} at position {}", tableName, pos); // Read exactly one object from the stream into a Map From 2274e97c04d935d9c2cb3826203a332d162eb6d6 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Sat, 20 Dec 2025 16:54:38 +0530 Subject: [PATCH 06/14] PLUGIN-1936: Rework based on new changes suggested in the design --- pom.xml | 5 + .../servicenow/ServiceNowBaseConfig.java | 33 +++- .../ServiceNowTableAPIClientImpl.java | 44 ++--- .../connector/ServiceNowConnector.java | 4 +- .../servicenow/restapi/RestAPIClient.java | 2 +- .../servicenow/restapi/RestAPIResponse.java | 84 ++++----- .../service/ServiceNowSinkAPIRequestImpl.java | 4 +- .../source/ServiceNowMultiRecordReader.java | 126 ++++++++++++-- .../source/ServiceNowRecordReader.java | 24 +-- .../ServiceNowTableAPIClientImplTest.java | 28 ++- .../connector/ServiceNowConnectorTest.java | 9 +- .../sink/ServiceNowRecordWriterTest.java | 12 +- .../sink/ServiceNowSinkConfigTest.java | 13 +- .../servicenow/sink/ServiceNowSinkTest.java | 18 +- .../source/ServiceNowInputFormatTest.java | 21 ++- .../ServiceNowMultiRecordReaderTest.java | 81 ++++++--- .../ServiceNowMultiSourceConfigTest.java | 25 ++- .../source/ServiceNowMultiSourceTest.java | 23 ++- .../source/ServiceNowRecordReaderTest.java | 161 ++++++++++++------ .../source/ServiceNowSourceConfigTest.java | 9 +- .../source/ServiceNowSourceTest.java | 23 ++- 21 files changed, 515 insertions(+), 234 deletions(-) diff --git a/pom.xml b/pom.xml index e37a6978..f9fff8e3 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,11 @@ ${cdap.version} test + + commons-io + commons-io + 2.5 + org.apache.hadoop hadoop-common diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 9bdcbdd7..681c39cd 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -17,12 +17,14 @@ package io.cdap.plugin.servicenow; import com.google.common.annotations.VisibleForTesting; +import com.google.gson.stream.JsonReader; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.ConfigUtil; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -31,7 +33,13 @@ import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceValueType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import javax.annotation.Nullable; /** @@ -39,6 +47,7 @@ */ public class ServiceNowBaseConfig extends PluginConfig { + private static final Logger log = LoggerFactory.getLogger(ServiceNowBaseConfig.class); @Name(ConfigUtil.NAME_USE_CONNECTION) @Nullable @Description("Whether to use an existing connection.") @@ -140,7 +149,7 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); - if (serviceNowTableAPIClient.parseResponseToResultListOfMap(apiResponse.getResponseBody()).isEmpty()) { + if (isResultEmpty(apiResponse)) { // Removed config property as in case of MultiSource, only first table error was populating. collector.addFailure("Table: " + tableName + " is empty.", ""); } @@ -152,4 +161,26 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo } } + /** + * Check whether the result is empty or not. + * @param restAPIResponse + * @return true if result is empty + * @throws IOException + */ + public boolean isResultEmpty(RestAPIResponse restAPIResponse) throws IOException { + JsonReader reader = new JsonReader(new InputStreamReader(restAPIResponse.getBodyAsStream(), + StandardCharsets.UTF_8)); + reader.beginObject(); + while (reader.hasNext()) { + String name = reader.nextName(); + if (ServiceNowConstants.RESULT.equals(name)) { + reader.beginArray(); + return !reader.hasNext(); + } else { + reader.skipValue(); + } + } + return true; + } + } diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 38659f46..3c5b4703 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -161,9 +161,6 @@ public RestAPIResponse fetchTableRecords( requestBuilder.setAuthHeader(accessToken); RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build()); return apiResponse; - //return parseResponseToResultListOfMap(apiResponse.getResponseBody()); - // return parseResponseStreamToRecord(apiResponse.getInputStream()); - } private void applyDateRangeToRequest(ServiceNowTableAPIRequestBuilder requestBuilder, String startDate, @@ -206,14 +203,13 @@ public List> parseResponseToResultListOfMap(String responseB return GSON.fromJson(ja, type); } - public Map parseResponseStreamToRecord(InputStream in) throws ServiceNowAPIException { - // List> records = new ArrayList<>(); - // InputStream in = httpResponse.getEntity().getContent(); + public List> parseResponseToResultListOfMap(InputStream in) throws ServiceNowAPIException { try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8); JsonReader jsonReader = new JsonReader(reader)) { jsonReader.setLenient(true); jsonReader.beginObject(); - Map record = new HashMap<>(); + + List> records = new ArrayList<>(); while (jsonReader.hasNext()) { String name = jsonReader.nextName(); if (ServiceNowConstants.RESULT.equals(name) && jsonReader.peek() == JsonToken.BEGIN_ARRAY) { @@ -221,23 +217,21 @@ public Map parseResponseStreamToRecord(InputStream in) throws Se while (jsonReader.hasNext()) { jsonReader.beginObject(); while (jsonReader.hasNext()) { + Map record = new HashMap<>(); String field = jsonReader.nextName(); JsonToken token = jsonReader.peek(); - // JsonElement resultElement = GSON.fromJson(jsonReader, JsonElement.class); - // responseBody = resultElement.toString(); record.put(field, token == JsonToken.NULL ? null : jsonReader.nextString()); + records.add(record); } jsonReader.endObject(); - // records.add(record); } jsonReader.endArray(); } else { - // skip other fields (e.g., metadata like result_count) jsonReader.skipValue(); } } jsonReader.endObject(); - return record; + return records; } catch (IOException e) { throw new ServiceNowAPIException(e, null); } @@ -325,8 +319,8 @@ public Schema fetchTableSchema(String tableName, FailureCollector collector) { } @VisibleForTesting - public MetadataAPISchemaResponse parseSchemaResponse(String responseBody) { - return GSON.fromJson(responseBody, MetadataAPISchemaResponse.class); + public MetadataAPISchemaResponse parseSchemaResponse(InputStream responseStream) { + return GSON.fromJson(createJsonReader(responseStream), MetadataAPISchemaResponse.class); } /** @@ -400,7 +394,7 @@ public Schema fetchTableSchema(String tableName, String accessToken, SourceValue private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List columns, String tableName) throws ServiceNowAPIException { SchemaAPISchemaResponse schemaAPISchemaResponse = - GSON.fromJson(restAPIResponse.getResponseBody(), SchemaAPISchemaResponse.class); + GSON.fromJson(createJsonReader(restAPIResponse.getBodyAsStream()), SchemaAPISchemaResponse.class); if (schemaAPISchemaResponse.getResult() == null || schemaAPISchemaResponse.getResult().isEmpty()) { throw new ServiceNowAPIException( @@ -434,8 +428,7 @@ private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List< private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List columns, String tableName, SourceValueType valueType) throws ServiceNowAPIException { - MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody()); - + MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getBodyAsStream()); if (metadataAPISchemaResponse.getResult() == null || metadataAPISchemaResponse.getResult().getColumns() == null || metadataAPISchemaResponse.getResult().getColumns().isEmpty()) { throw new ServiceNowAPIException( @@ -461,6 +454,11 @@ private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, Lis return SchemaBuilder.constructSchema(tableName, columns); } + public JsonReader createJsonReader(InputStream inputStream) { + Objects.requireNonNull(inputStream, "InputStream must not be null"); + return new JsonReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + /** * Get the total number of records in the table * @@ -551,8 +549,8 @@ public String createRecordInDisplayMode(String tableName, HttpEntity entity) thr } private String getSystemId(RestAPIResponse restAPIResponse) { - CreateRecordAPIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), - CreateRecordAPIResponse.class); + CreateRecordAPIResponse apiResponse = GSON.fromJson( + new InputStreamReader(restAPIResponse.getBodyAsStream(), StandardCharsets.UTF_8), CreateRecordAPIResponse.class); return apiResponse.getResult().get(ServiceNowConstants.SYSTEM_ID).toString(); } @@ -575,7 +573,8 @@ public Map getRecordFromServiceNowTable(String tableName, String requestBuilder.setAuthHeader(accessToken); restAPIResponse = executeGetWithRetries(requestBuilder.build()); - APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class); + APIResponse apiResponse = GSON.fromJson( + new InputStreamReader(restAPIResponse.getBodyAsStream(), StandardCharsets.UTF_8), APIResponse.class); return apiResponse.getResult().get(0); } @@ -593,8 +592,9 @@ public Map getRecordFromServiceNowTable(String tableName, String * @throws RuntimeException if the schema response is null or contains no result. */ private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List columns, - String tableName) { - List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody()); + String tableName) throws ServiceNowAPIException { + List> result = parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream()); + // List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody()); if (result != null && !result.isEmpty()) { Map firstRecord = result.get(0); for (String key : firstRecord.keySet()) { diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index 81aa364c..9ab05be4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -135,7 +135,7 @@ private TableList listTables(String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); - return GSON.fromJson(apiResponse.getResponseBody(), TableList.class); + return GSON.fromJson(serviceNowTableAPIClient.createJsonReader(apiResponse.getBodyAsStream()), TableList.class); } public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSpecRequest connectorSpecRequest) { @@ -184,7 +184,7 @@ private List getTableData(String tableName, int limit) requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap - (apiResponse.getResponseBody()); + (apiResponse.getBodyAsStream()); List recordList = new ArrayList<>(); Schema schema = getSchema(tableName); if (schema != null) { diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index 5be6fffc..a99ac19a 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -83,7 +83,7 @@ public RestAPIResponse executeGet(RestAPIRequest request) throws IOException { } } catch (ConnectTimeoutException | SocketException e) { ServiceNowAPIException exception = new ServiceNowAPIException(e, null); - return new RestAPIResponse(Collections.emptyMap(), (InputStream) null, exception); + return new RestAPIResponse(Collections.emptyMap(), null, exception); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index ff21f2d4..3c154ce1 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -20,13 +20,16 @@ import com.google.gson.JsonObject; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.util.ServiceNowConstants; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BoundedInputStream; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -44,42 +47,33 @@ * Pojo class to capture the API response. */ public class RestAPIResponse { + private static final Logger LOG = LoggerFactory.getLogger(RestAPIResponse.class); private static final Gson GSON = new Gson(); private static final String HTTP_ERROR_MESSAGE = "Http call to ServiceNow instance returned status code %d."; private static final String REST_ERROR_MESSAGE = "Rest Api response has errors. Error message: %s."; private static final Set SUCCESS_CODES = new HashSet<>(Arrays.asList(HttpStatus.SC_CREATED, HttpStatus.SC_OK)); + private static final long MAX_PAGE_BYTES = 50L * 1024 * 1024; // 50 MB (Upper Bound) private final Map headers; - // Deprecated: storing full body as String can cause OOM - @Deprecated - private String responseBody; @Nullable private final ServiceNowAPIException exception; - // New: store InputStream for streaming consumption - private InputStream inputStream; - - public RestAPIResponse( - Map headers, - @Nullable String responseBody, - InputStream inputStream, - @Nullable ServiceNowAPIException exception) { - this.headers = headers; - this.responseBody = responseBody; - this.inputStream = inputStream; - this.exception = exception; - } + // New: store byte array + private byte[] responseBody; public RestAPIResponse( Map headers, - InputStream inputStream, + byte[] responseBody, @Nullable ServiceNowAPIException exception) { this.headers = headers; - this.inputStream = inputStream; + this.responseBody = responseBody; this.exception = exception; } /** * Parses HttpResponse into RestAPIResponse object when no errors occur. + * The RESTAPIResponse contains the HTTP response body as a stream. This stream is: + * single-use, forward-only and owned by the caller. Caller is responsible for consuming and closing it. + * * Throws a {@link ServiceNowAPIException}. * * @param httpResponse The HttpResponse object to parse @@ -100,43 +94,33 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN ServiceNowAPIException serviceNowAPIException = validateHttpResponse(httpResponse); if (serviceNowAPIException != null) { - return new RestAPIResponse(headers, null, null, serviceNowAPIException); + return new RestAPIResponse(headers, null, serviceNowAPIException); } - /*try { - responseBody = EntityUtils.toString(httpResponse.getEntity()); - } catch (IOException e) { - return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); - }*/ try { - return prepareResponseWithBodyAndStream(httpResponse, headers, serviceNowAPIException); + return prepareResponseStream(httpResponse, headers, serviceNowAPIException); } catch (IOException e) { - return new RestAPIResponse(headers, null, null, new ServiceNowAPIException(e, httpResponse)); + return new RestAPIResponse(headers, null, new ServiceNowAPIException(e, httpResponse)); } } - public static RestAPIResponse prepareResponseWithBodyAndStream(HttpResponse httpResponse, Map headers, + public static RestAPIResponse prepareResponseStream(HttpResponse httpResponse, Map headers, ServiceNowAPIException serviceNowAPIException) throws IOException { HttpEntity httpEntity = httpResponse.getEntity(); + byte[] responseBody = new byte[0]; + InputStream inputStream; if (httpEntity != null) { - try (InputStream inputStream = httpEntity.getContent(); - ByteArrayOutputStream buffer = new ByteArrayOutputStream()) { - - // Copy the InputStream into the ByteArrayOutputStream - byte[] data = new byte[8192]; - int bytesRead; - while ((bytesRead = inputStream.read(data)) != -1) { - buffer.write(data, 0, bytesRead); - } - // Convert the buffer to a String for the responseBody - String responseBody = buffer.toString(String.valueOf(StandardCharsets.UTF_8)); - serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); - // Create a new InputStream from the buffer for further processing - InputStream reusableStream = new ByteArrayInputStream(buffer.toByteArray()); - // return new RestAPIResponse(headers, responseBody, serviceNowAPIException); - return new RestAPIResponse(headers, responseBody, reusableStream, serviceNowAPIException); + inputStream = httpEntity.getContent(); + BoundedInputStream boundedInputStream = new BoundedInputStream( + inputStream, MAX_PAGE_BYTES + 1); // +1 to detect overflow + responseBody = IOUtils.toByteArray(boundedInputStream); + LOG.info("RAW JSON: {}", new String(responseBody, StandardCharsets.UTF_8)); + if (responseBody.length > MAX_PAGE_BYTES) { + throw new IOException( + "ServiceNow page exceeded max allowed size: " + MAX_PAGE_BYTES); } + return new RestAPIResponse(headers, responseBody, serviceNowAPIException); } else { - return new RestAPIResponse(headers, null, null, serviceNowAPIException); + return new RestAPIResponse(headers, responseBody, serviceNowAPIException); } } @@ -174,12 +158,16 @@ public Map getHeaders() { } @Nullable - public String getResponseBody() { + public byte[] getResponseBody() { return responseBody; } - public InputStream getInputStream() { - return inputStream; + /** + * Returns a fresh InputStream for the response body. Caller must close the stream. + * @return InputStream + */ + public InputStream getBodyAsStream() { + return responseBody == null ? null : new ByteArrayInputStream(responseBody); } @Nullable diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java index 6d1f8d1f..af64cd60 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; @@ -121,7 +122,8 @@ public void createPostRequest(Map restRequestsMap, String a requestBuilder.setEntity(stringEntity); apiResponse = restApi.executePost(requestBuilder.build()); - JsonObject responseJSON = jsonParser.parse(apiResponse.getResponseBody()).getAsJsonObject(); + JsonObject responseJSON = jsonParser.parse( + new InputStreamReader(apiResponse.getBodyAsStream(), StandardCharsets.UTF_8)).getAsJsonObject(); JsonArray servicedRequestsArray = responseJSON.get(ServiceNowConstants.SERVICED_REQUESTS).getAsJsonArray(); JsonElement failedRequestId = null; for (int i = 0; i < servicedRequestsArray.size(); i++) { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index ddfc85ef..ba90b143 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -17,26 +17,42 @@ package io.cdap.plugin.servicenow.source; import com.google.common.annotations.VisibleForTesting; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Record reader that reads the entire contents of a ServiceNow table. */ public class ServiceNowMultiRecordReader extends ServiceNowBaseRecordReader { + private static final Logger LOG = LoggerFactory.getLogger(ServiceNowMultiRecordReader.class); private final ServiceNowMultiSourceConfig multiSourcePluginConf; private ServiceNowTableAPIClientImpl restApi; + private final Gson gson = new Gson(); + private final Type mapType = new TypeToken>() { }.getType(); + private JsonReader jsonReader = null; ServiceNowMultiRecordReader(ServiceNowMultiSourceConfig multiSourcePluginConf) { super(); @@ -55,23 +71,38 @@ public void initialize(InputSplit split, TaskAttemptContext context) { } @Override + /** + * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. + * Returns true when it assigned `row` to the next record. + * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). + */ public boolean nextKeyValue() throws IOException { - try { - if (results == null) { - fetchData(); + // Ensure we have an active page/jsonReader + if (jsonReader == null) { + // Need to open the next page + boolean pageOpened; + try { + pageOpened = openNextPage(); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue" + tableName, e); } - - if (!iterator.hasNext()) { + if (!pageOpened) { + // No more pages return false; } + } - row = iterator.next(); + // At this point jsonReader is positioned inside the "result" array. + JsonToken token = jsonReader.peek(); + if (token == JsonToken.BEGIN_OBJECT) { + LOG.debug("Reading record object for table {} at position {}", tableName, pos); + this.row = gson.fromJson(jsonReader, mapType); // assign row pos++; - } catch (Exception e) { - throw new IOException("Exception in nextKeyValue", e); + return true; } - return true; + closeCurrentPage(); + return false; } @Override @@ -97,11 +128,84 @@ RestAPIResponse fetchData() throws ServiceNowAPIException { RestAPIResponse restAPIResponse = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(), multiSourcePluginConf.getStartDate(), multiSourcePluginConf.getEndDate(), split.getOffset(), multiSourcePluginConf.getPageSize()); - - // iterator = results.iterator(); + return restAPIResponse; } + private boolean openNextPage() throws IOException, ServiceNowAPIException { + LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); + closeCurrentPage(); + LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); + RestAPIResponse resp = fetchData(); + LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); + InputStream in = resp.getBodyAsStream(); + if (in == null) { + return false; + } + this.jsonReader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + this.jsonReader.setLenient(true); + + // Position the reader to the "result" array: { "result": [ ... ], ... } + try { + JsonToken top; + try { + top = jsonReader.peek(); + LOG.info("Peeking JSON token for table {}: {}", tableName, top); + } catch (IOException e) { + LOG.warn("Unexpected closure of stream while peeking JSON token for table {}", tableName, e); + closeCurrentPage(); + return false; + } + if (top == JsonToken.BEGIN_OBJECT) { + jsonReader.beginObject(); + while (jsonReader.hasNext()) { + String name = jsonReader.nextName(); + if (name.equals(ServiceNowConstants.RESULT)) { + if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + return true; + } else { + jsonReader.skipValue(); + break; + } + } else { + jsonReader.skipValue(); + } + } + } else if (top == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + return true; + } else if (jsonReader.peek() == JsonToken.END_ARRAY) { + // empty result array — treat as no-more-data for this split/page + LOG.debug("openNextPage: found empty result array (no records). Closing and returning false."); + jsonReader.endArray(); + // cleanup + closeCurrentPage(); + return false; + } + } catch (IOException e) { + closeCurrentPage(); + throw e; + } + + // No "result" array not found, close the current page and return false + closeCurrentPage(); + return false; + } + + public void closeCurrentPage() { + LOG.info("Closing current page for table {}", tableName); + if (this.jsonReader != null) { + try { + this.jsonReader.close(); + } catch (IOException e) { + LOG.warn("Error closing JSON reader", e); + } finally { + this.jsonReader = null; + } + } + } + private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { // Fetch the schema try { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index aaf0e7d6..da8b47e1 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -78,7 +78,7 @@ public void initialize(InputSplit split, Schema schema) { /** * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. - * Returns true when it assigned `row` to the next Map record. + * Returns true when it assigned `row` to the next record. * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). */ @Override @@ -102,10 +102,8 @@ public boolean nextKeyValue() throws IOException { JsonToken token = jsonReader.peek(); if (token == JsonToken.BEGIN_OBJECT) { - LOG.info("Reading record object for table {} at position {}", tableName, pos); - // Read exactly one object from the stream into a Map - Map recordMap = gson.fromJson(jsonReader, mapType); - this.row = recordMap; // assign row + LOG.debug("Reading record object for table {} at position {}", tableName, pos); + this.row = gson.fromJson(jsonReader, mapType); // assign row pos++; return true; } @@ -137,22 +135,17 @@ private RestAPIResponse fetchData() throws ServiceNowAPIException { // Get the table data RestAPIResponse restAPIResponse = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), pluginConf.getPageSize()); - - // LOG.debug("Results size={}", results.size()); return restAPIResponse; - - // iterator = results.iterator(); - // iterator = record; } private boolean openNextPage() throws IOException, ServiceNowAPIException { - LOG.info("Opening next page for table {} at offset {}", tableName, split.getOffset()); + LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); closeCurrentPage(); - LOG.info("Fetching data for table {} at offset {}", tableName, split.getOffset()); + LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); RestAPIResponse resp = fetchData(); - LOG.info("Fetched data for table {} at offset {}", tableName, split.getOffset()); - InputStream in = resp.getInputStream(); + LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); + InputStream in = resp.getBodyAsStream(); if (in == null) { return false; } @@ -191,8 +184,7 @@ private boolean openNextPage() throws IOException, ServiceNowAPIException { return true; } else if (jsonReader.peek() == JsonToken.END_ARRAY) { // empty result array — treat as no-more-data for this split/page - LOG.info("openNextPage: found empty result array (no records). Closing and returning false."); - // consume the END_ARRAY token to leave stream consistent (optional) + LOG.debug("openNextPage: found empty result array (no records). Closing and returning false."); jsonReader.endArray(); // cleanup closeCurrentPage(); diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java index 98199840..e1f2d18d 100644 --- a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -15,6 +15,9 @@ import org.junit.rules.ExpectedException; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -29,17 +32,16 @@ public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws Servi ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); - RestAPIResponse mockApiResponse = new RestAPIResponse( - Collections.emptyMap(), "", null, null); List> mockResults = new ArrayList<>(); mockResults.add(new HashMap() {{ put("keyTest", "valueTest"); }}); HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + RestAPIResponse mockApiResponse = new RestAPIResponse(Collections.emptyMap(), null, null); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_REQUEST_TIMEOUT); Mockito.doThrow(new ServiceNowAPIException("Retryable Error", mockResponse)) - .doReturn(mockResults) + .doReturn(mockApiResponse) .when(implSpy).fetchTableRecords( Mockito.anyString(), Mockito.any(), @@ -67,11 +69,12 @@ public void testFetchTableRecordsRetryableMode_nonRetryable() ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + RestAPIResponse mockAPIResponse = new RestAPIResponse(Collections.emptyMap(), null, null); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_INTERNAL_SERVER_ERROR); Mockito.doThrow( new ServiceNowAPIException("Non-retryable Error", mockResponse)) - .doReturn(new ArrayList<>()) + .doReturn(mockAPIResponse) .when(implSpy).fetchTableRecords( Mockito.anyString(), Mockito.any(), @@ -106,8 +109,9 @@ public void testFetchTableSchema_ActualValueType() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, - null); + byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.SCHEMA_API_BASED); @@ -139,7 +143,9 @@ public void testFetchTableSchema_GlideTimeFieldWithActualValueType() throws Exce " }\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); + byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("u_custom_table", @@ -178,7 +184,9 @@ public void testFetchTableSchema_DisplayValueType() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); + byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.SCHEMA_API_BASED); @@ -215,7 +223,9 @@ public void testFetchTableSchema_StcFieldsWithDisplayValueType_ParseAsString() t " }\n" + " }\n" + "}"; - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null, null); + byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("incident", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.METADATA_API_BASED); diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index c16a4a67..ab9df750 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -53,7 +53,10 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -131,9 +134,11 @@ public void testGenerateSpec() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index 3a35e825..a44112d5 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -99,9 +99,9 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { result.add(map); Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse( - headers, responseBody, null, null); + headers, null, null); Mockito.when(restApi.executePost(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); @@ -143,9 +143,9 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { map.put("key", "value"); result.add(map); Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -191,9 +191,9 @@ public void testWriteWithUnservicedRequests() throws Exception { map.put("key", "value"); result.add(map); Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index df767840..e75782b0 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -50,6 +50,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -296,6 +299,8 @@ public void testValidateSchema() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); MetadataAPISchemaField schemaField = new MetadataAPISchemaField("Class", "sys_class_name", "sys_class_name", "sys_class_name"); Map columns = new HashMap<>(); @@ -306,7 +311,7 @@ public void testValidateSchema() throws Exception { Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); RestAPIResponse restAPIResponse = new RestAPIResponse( - headers, responseBody, null, new ServiceNowAPIException("", mockResponse)); + headers, body, new ServiceNowAPIException("", mockResponse)); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -326,7 +331,7 @@ public void testValidateSchema() throws Exception { PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); Mockito.when(restApi.executeGetWithRetries(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(Mockito.anyString(), Mockito.any(FailureCollector.class))).thenReturn(schema); - Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getResponseBody())) + Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getBodyAsStream())) .thenReturn(metadataAPISchemaResponse); try { config.validateSchema(schema, collector); @@ -362,7 +367,9 @@ public void testValidateSchemaWithOperation() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index cde2a205..e4d0c318 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -25,14 +25,11 @@ import io.cdap.cdap.etl.mock.common.MockArguments; import io.cdap.cdap.etl.mock.common.MockPipelineConfigurer; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; -import io.cdap.plugin.servicenow.ServiceNowBaseConfig; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.sink.transform.ServiceNowTransformer; import io.cdap.plugin.servicenow.source.ServiceNowBaseSourceConfig; import org.apache.hadoop.io.NullWritable; -import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -48,6 +45,9 @@ import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -92,10 +92,12 @@ public void testConfigurePipeline() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); MockFailureCollector collector = new MockFailureCollector(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); serviceNowSink.configurePipeline(mockPipelineConfigurer); Assert.assertNull(restAPIResponse.getException()); Assert.assertEquals(0, collector.getValidationFailures().size()); @@ -125,14 +127,16 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); Schema schema = Schema.recordOf("record", Schema.Field.of("id", Schema.of(Schema.Type.LONG)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE))); Emitter> emitter = Mockito.mock(Emitter.class); Mockito.when(context.getInputSchema()).thenReturn(schema); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index 771e4b62..10269796 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -42,6 +42,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -138,13 +141,15 @@ public void testFetchTableInfo() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); String schemaString = "{\"type\":\"record\",\"name\":\"ServiceNowColumnMetaData\",\"fields\":[{\"name\":" + "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("table", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -242,13 +247,15 @@ public void testFetchTableInfoReportingMode() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); String schemaString = "{\"type\":\"record\",\"name\":\"ServiceNowColumnMetaData\",\"fields\":[{\"name\":" + "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("proc_po", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); Mockito.when(restApi.fetchTableSchema("proc_po_item", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); @@ -291,9 +298,11 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.mockStatic(ServiceNowInputFormat.class); PowerMockito.whenNew(OAuthClient.class). diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 0169d88b..0d6f7c50 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -24,7 +24,7 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; -import io.cdap.plugin.servicenow.util.ServiceNowConstants; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import org.apache.oltu.oauth2.common.exception.OAuthProblemException; import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.junit.Assert; @@ -32,9 +32,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; @@ -42,6 +47,9 @@ import java.util.List; import java.util.Map; +@RunWith(PowerMockRunner.class) +@PrepareForTest({ServiceNowTableAPIClientImpl.class, ServiceNowMultiSourceConfig.class, + ServiceNowMultiRecordReader.class}) public class ServiceNowMultiRecordReaderTest { private static final String CLIENT_ID = "clientId"; @@ -121,25 +129,60 @@ public void testConvertToBooleanValueForInvalidFieldValue() { } @Test - public void testFetchData() throws ServiceNowAPIException, IOException { + public void testFetchData() throws Exception { String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); - - List> results = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("calendar_integration", "1"); - map.put("country", "India"); - map.put("sys_updated_on", "2019-04-05 21:54:45"); - map.put("web_service_access_only", "false"); - map.put("notification", "2"); - map.put("enable_multifactor_authn", "false"); - map.put("sys_updated_by", "system"); - map.put("sys_created_on", "2019-04-05 21:09:12"); - results.add(map); - - ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); - response.setResult(results); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); + ServiceNowMultiRecordReader serviceNowMultiRecordReader = + new ServiceNowMultiRecordReader(serviceNowMultiSourceConfig); + String responseBody = "{\n" + + " \"result\": [\n" + + " {\n" + + " \"bill_to\": \"\",\n" + + " \"init_request\": \"\",\n" + + " \"short_description\": \"\",\n" + + " \"total_cost\": \"0\",\n" + + " \"due_by\": \"\",\n" + + " \"description\": \"\",\n" + + " \"requested_for\": \"\",\n" + + " \"sys_updated_on\": \"2022-06-16 18:56:23\",\n" + + " \"budget_number\": \"\",\n" + + " \"number\": \"RCS397871\",\n" + + " \"sys_id\": \"00000b7287405910827733373cbb35d5\",\n" + + " \"sys_updated_by\": \"pipeline.user.1\",\n" + + " \"shipping\": \"\",\n" + + " \"terms\": \"\",\n" + + " \"sys_created_on\": \"2022-06-16 18:56:23\",\n" + + " \"vendor\": \"\",\n" + + " \"sys_domain\": \"global\",\n" + + " \"department\": \"\",\n" + + " \"sys_created_by\": \"pipeline.user.1\",\n" + + " \"assigned_to\": \"\",\n" + + " \"ordered\": \"\",\n" + + " \"po_date\": \"2022-06-16 18:56:23\",\n" + + " \"vendor_contract\": \"\",\n" + + " \"contract\": \"\",\n" + + " \"expected_delivery\": \"\",\n" + + " \"sys_mod_count\": \"0\",\n" + + " \"received\": \"2158-05-10 17:14:20\",\n" + + " \"asset_operation\": \"\",\n" + + " \"sys_tags\": \"\",\n" + + " \"requested\": \"2022-06-16 18:56:23\",\n" + + " \"requested_by\": \"\",\n" + + " \"ship_rate\": \"0\",\n" + + " \"location\": \"\",\n" + + " \"vendor_account\": \"\",\n" + + " \"ship_to\": \"\",\n" + + " \"status\": \"requested\"\n" + + " }\n" + + " ]\n" + + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); + Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowMultiSourceConfig.getValueType(), + serviceNowMultiSourceConfig.getStartDate(), serviceNowMultiSourceConfig.getEndDate(), split.getOffset(), + serviceNowMultiSourceConfig.getPageSize())).thenReturn(restAPIResponse); try { Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); @@ -148,10 +191,6 @@ public void testFetchData() throws ServiceNowAPIException, IOException { | ServiceNowAPIException e) { Assert.assertTrue(e instanceof RuntimeException); } - Mockito.doNothing().when(serviceNowMultiRecordReader).fetchData(); - Collections.singletonList(new Object()); - serviceNowMultiRecordReader.iterator = Collections.singletonList(Collections.singletonMap("key", new String())). - iterator(); Assert.assertTrue(serviceNowMultiRecordReader.nextKeyValue()); } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index 7ad89592..2b1407b3 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -31,6 +31,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -174,9 +177,11 @@ public void testValidate() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); @@ -207,7 +212,9 @@ public void testValidateWhenTableIsEmpty() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); @@ -301,9 +308,11 @@ public void testValidateReferenceName() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); try { serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.fail("Exception is not thrown with valid reference name"); @@ -402,9 +411,11 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); Assert.assertEquals("Table name field must be specified.", diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index 4e6e8afc..41d06ec2 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -23,10 +23,8 @@ import io.cdap.cdap.etl.mock.common.MockPipelineConfigurer; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; -import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -43,6 +41,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -156,9 +157,11 @@ public void testConfigurePipeline() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); Assert.assertNull(mockPipelineConfigurer.getOutputSchema()); Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); @@ -176,9 +179,11 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); try { serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); Assert.fail("Exception is not thrown for Non-Empty Tables"); @@ -278,11 +283,13 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); Mockito.when(ServiceNowMultiInputFormat.setInput(Mockito.any(), Mockito.any())).thenReturn((tableInfo)); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index 9ee945de..d4ea6841 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -40,11 +40,15 @@ import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,7 +56,6 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({ServiceNowTableAPIClientImpl.class, ServiceNowSourceConfig.class, ServiceNowRecordReader.class}) public class ServiceNowRecordReaderTest { - private static final String CLIENT_ID = "client_id"; private static final String CLIENT_SECRET = "client_secret"; private static final String REST_API_ENDPOINT = "https://ven05127.service-now.com"; @@ -258,32 +261,55 @@ public void testFetchData() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("calendar_integration", "1"); - map.put("country", "India"); - map.put("sys_updated_on", "2019-04-05 21:54:45"); - map.put("web_service_access_only", "false"); - map.put("notification", "2"); - map.put("enable_multifactor_authn", "false"); - map.put("sys_updated_by", "system"); - map.put("sys_created_on", "2019-04-05 21:09:12"); - results.add(map); - ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); - ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); - ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); - List columns = new ArrayList<>(); - columns.add(column1); - columns.add(column2); - response.setColumns(columns); - response.setResult(results); - response.setTotalRecordCount(1); RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); + String responseBody = "{\n" + + " \"result\": [\n" + + " {\n" + + " \"bill_to\": \"\",\n" + + " \"init_request\": \"\",\n" + + " \"short_description\": \"\",\n" + + " \"total_cost\": \"0\",\n" + + " \"due_by\": \"\",\n" + + " \"description\": \"\",\n" + + " \"requested_for\": \"\",\n" + + " \"sys_updated_on\": \"2022-06-16 18:56:23\",\n" + + " \"budget_number\": \"\",\n" + + " \"number\": \"RCS397871\",\n" + + " \"sys_id\": \"00000b7287405910827733373cbb35d5\",\n" + + " \"sys_updated_by\": \"pipeline.user.1\",\n" + + " \"shipping\": \"\",\n" + + " \"terms\": \"\",\n" + + " \"sys_created_on\": \"2022-06-16 18:56:23\",\n" + + " \"vendor\": \"\",\n" + + " \"sys_domain\": \"global\",\n" + + " \"department\": \"\",\n" + + " \"sys_created_by\": \"pipeline.user.1\",\n" + + " \"assigned_to\": \"\",\n" + + " \"ordered\": \"\",\n" + + " \"po_date\": \"2022-06-16 18:56:23\",\n" + + " \"vendor_contract\": \"\",\n" + + " \"contract\": \"\",\n" + + " \"expected_delivery\": \"\",\n" + + " \"sys_mod_count\": \"0\",\n" + + " \"received\": \"2158-05-10 17:14:20\",\n" + + " \"asset_operation\": \"\",\n" + + " \"sys_tags\": \"\",\n" + + " \"requested\": \"2022-06-16 18:56:23\",\n" + + " \"requested_by\": \"\",\n" + + " \"ship_rate\": \"0\",\n" + + " \"location\": \"\",\n" + + " \"vendor_account\": \"\",\n" + + " \"ship_to\": \"\",\n" + + " \"status\": \"requested\"\n" + + " }\n" + + " ]\n" + + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), - serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. - getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); + serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), + serviceNowSourceConfig.getPageSize())).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(tableName, valueType)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); @@ -312,32 +338,54 @@ public void testFetchDataReportingMode() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); - List> results = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("calendar_integration", "1"); - map.put("country", "India"); - map.put("sys_updated_on", "2019-04-05 21:54:45"); - map.put("web_service_access_only", "false"); - map.put("notification", "2"); - map.put("enable_multifactor_authn", "false"); - map.put("sys_updated_by", "system"); - map.put("sys_created_on", "2019-04-05 21:09:12"); - results.add(map); - ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); - ServiceNowColumn column1 = new ServiceNowColumn("calendar_integration", "integer"); - ServiceNowColumn column2 = new ServiceNowColumn("vip", "boolean"); - List columns = new ArrayList<>(); - columns.add(column1); - columns.add(column2); - response.setColumns(columns); - response.setResult(results); - response.setTotalRecordCount(1); - RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); + String responseBody = "{\n" + + " \"result\": [\n" + + " {\n" + + " \"bill_to\": \"\",\n" + + " \"init_request\": \"\",\n" + + " \"short_description\": \"\",\n" + + " \"total_cost\": \"0\",\n" + + " \"due_by\": \"\",\n" + + " \"description\": \"\",\n" + + " \"requested_for\": \"\",\n" + + " \"sys_updated_on\": \"2022-06-16 18:56:23\",\n" + + " \"budget_number\": \"\",\n" + + " \"number\": \"RCS397871\",\n" + + " \"sys_id\": \"00000b7287405910827733373cbb35d5\",\n" + + " \"sys_updated_by\": \"pipeline.user.1\",\n" + + " \"shipping\": \"\",\n" + + " \"terms\": \"\",\n" + + " \"sys_created_on\": \"2022-06-16 18:56:23\",\n" + + " \"vendor\": \"\",\n" + + " \"sys_domain\": \"global\",\n" + + " \"department\": \"\",\n" + + " \"sys_created_by\": \"pipeline.user.1\",\n" + + " \"assigned_to\": \"\",\n" + + " \"ordered\": \"\",\n" + + " \"po_date\": \"2022-06-16 18:56:23\",\n" + + " \"vendor_contract\": \"\",\n" + + " \"contract\": \"\",\n" + + " \"expected_delivery\": \"\",\n" + + " \"sys_mod_count\": \"0\",\n" + + " \"received\": \"2158-05-10 17:14:20\",\n" + + " \"asset_operation\": \"\",\n" + + " \"sys_tags\": \"\",\n" + + " \"requested\": \"2022-06-16 18:56:23\",\n" + + " \"requested_by\": \"\",\n" + + " \"ship_rate\": \"0\",\n" + + " \"location\": \"\",\n" + + " \"vendor_account\": \"\",\n" + + " \"ship_to\": \"\",\n" + + " \"status\": \"requested\"\n" + + " }\n" + + " ]\n" + + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), - serviceNowSourceConfig.getStartDate(), - serviceNowSourceConfig.getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); + serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), + serviceNowSourceConfig.getPageSize())).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); @@ -353,7 +401,7 @@ public void testFetchDataOnInvalidTable() throws Exception { .setPassword(PASSWORD) .setClientId(CLIENT_ID) .setClientSecret(CLIENT_SECRET) - .setTableName("") + .setTableName("abc") .setValueType("Actual") .setStartDate("2021-01-01") .setEndDate("2022-02-18") @@ -365,12 +413,21 @@ public void testFetchDataOnInvalidTable() throws Exception { ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); List> results = new ArrayList<>(); - RestAPIResponse mockResponse = Mockito.mock(RestAPIResponse.class); + String responseBody = "{\n " + + "\"error\": " + + "{\n " + + "\"message\": \"Invalid table abc\",\n" + + " \"detail\": null\n " + + "},\n " + + "\"status\": \"failure\"\n" + + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(), + Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), - serviceNowSourceConfig.getPageSize())).thenReturn(mockResponse); + serviceNowSourceConfig.getPageSize())).thenReturn(restAPIResponse); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index af2716a6..5663cbb5 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -23,7 +23,6 @@ import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceApplication; @@ -42,7 +41,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.io.IOException; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -624,7 +625,9 @@ public void testValidateWhenTableIsEmpty() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); config.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index d3445a84..b0d3613d 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -23,11 +23,9 @@ import io.cdap.cdap.etl.mock.common.MockPipelineConfigurer; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; -import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -44,6 +42,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -168,12 +169,14 @@ public void testConfigurePipeline() throws Exception { " }\n" + " ]\n" + "}"; + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(tableInfo); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -209,9 +212,11 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); try { serviceNowSource.configurePipeline(mockPipelineConfigurer); Assert.fail("Exception is not thrown for Non-Empty Tables"); @@ -297,9 +302,11 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null, null); + byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); PowerMockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); From ba5eb536e47cd4c00b11fee8766b468e5f0e6dde Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Tue, 13 Jan 2026 15:19:26 +0530 Subject: [PATCH 07/14] PLUGIN-1936: Rework after review comments --- .../servicenow/ServiceNowBaseConfig.java | 10 +- .../ServiceNowTableAPIClientImpl.java | 51 ++----- .../servicenow/restapi/RestAPIResponse.java | 6 +- .../source/ServiceNowBaseRecordReader.java | 126 +++++++++++++++++- .../source/ServiceNowMultiRecordReader.java | 115 +--------------- .../source/ServiceNowRecordReader.java | 114 +--------------- 6 files changed, 146 insertions(+), 276 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 681c39cd..7daa7de3 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -162,10 +162,12 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo } /** - * Check whether the result is empty or not. - * @param restAPIResponse - * @return true if result is empty - * @throws IOException + * Determines if the "result" array in a ServiceNow REST API response is empty. It specifically looks for a top-level + * key named "result". Once found, it opens the associated array and checks for the presence of a first element. + * @param restAPIResponse The response object containing the JSON input stream + * @return true, if the "result" array exists and is empty, or if the "result" key is never found; + * false, if the array contains at least one element. + * @throws IOException If there is an error reading the input stream or parsing the JSON. */ public boolean isResultEmpty(RestAPIResponse restAPIResponse) throws IOException { JsonReader reader = new JsonReader(new InputStreamReader(restAPIResponse.getBodyAsStream(), diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 3c5b4703..8a3687b4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -28,7 +28,6 @@ import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonToken; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -62,13 +61,13 @@ import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; import static io.cdap.plugin.servicenow.util.ServiceNowConstants.STC_FIELD_SUFFIX; @@ -203,38 +202,10 @@ public List> parseResponseToResultListOfMap(String responseB return GSON.fromJson(ja, type); } - public List> parseResponseToResultListOfMap(InputStream in) throws ServiceNowAPIException { - try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8); - JsonReader jsonReader = new JsonReader(reader)) { - jsonReader.setLenient(true); - jsonReader.beginObject(); - - List> records = new ArrayList<>(); - while (jsonReader.hasNext()) { - String name = jsonReader.nextName(); - if (ServiceNowConstants.RESULT.equals(name) && jsonReader.peek() == JsonToken.BEGIN_ARRAY) { - jsonReader.beginArray(); - while (jsonReader.hasNext()) { - jsonReader.beginObject(); - while (jsonReader.hasNext()) { - Map record = new HashMap<>(); - String field = jsonReader.nextName(); - JsonToken token = jsonReader.peek(); - record.put(field, token == JsonToken.NULL ? null : jsonReader.nextString()); - records.add(record); - } - jsonReader.endObject(); - } - jsonReader.endArray(); - } else { - jsonReader.skipValue(); - } - } - jsonReader.endObject(); - return records; - } catch (IOException e) { - throw new ServiceNowAPIException(e, null); - } + public List> parseResponseToResultListOfMap(InputStream in) { + APIResponse apiResponse = GSON.fromJson(new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)), + APIResponse.class); + return apiResponse.getResult(); } private String getErrorMessage(String responseBody) { @@ -274,11 +245,11 @@ private String getErrorMessage(String responseBody) { public RestAPIResponse fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, String startDate, String endDate, int offset, int limit) throws ServiceNowAPIException { - //final List> results = new ArrayList<>(); - final RestAPIResponse[] restAPIResponse = new RestAPIResponse[1]; + // Using AtomicReference to capture a value inside a lambda that needs to be accessed outside. + AtomicReference responseRef = new AtomicReference<>(); Callable fetchRecords = () -> { - // results.addAll(fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit)); - restAPIResponse[0] = fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit); + RestAPIResponse restAPIResponse = fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit); + responseRef.set(restAPIResponse); return true; }; @@ -295,8 +266,8 @@ public RestAPIResponse fetchTableRecordsRetryableMode(String tableName, SourceVa String.format("Data Recovery failed for batch %s to %s.", offset, (offset + limit)), e, null, false); } - - return restAPIResponse[0]; + // Return the value captured inside the lambda + return responseRef.get(); } /** diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index 3c154ce1..57663fcb 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -32,7 +32,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -97,13 +96,13 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN return new RestAPIResponse(headers, null, serviceNowAPIException); } try { - return prepareResponseStream(httpResponse, headers, serviceNowAPIException); + return prepareResponse(httpResponse, headers, serviceNowAPIException); } catch (IOException e) { return new RestAPIResponse(headers, null, new ServiceNowAPIException(e, httpResponse)); } } - public static RestAPIResponse prepareResponseStream(HttpResponse httpResponse, Map headers, + public static RestAPIResponse prepareResponse(HttpResponse httpResponse, Map headers, ServiceNowAPIException serviceNowAPIException) throws IOException { HttpEntity httpEntity = httpResponse.getEntity(); byte[] responseBody = new byte[0]; @@ -113,7 +112,6 @@ public static RestAPIResponse prepareResponseStream(HttpResponse httpResponse, M BoundedInputStream boundedInputStream = new BoundedInputStream( inputStream, MAX_PAGE_BYTES + 1); // +1 to detect overflow responseBody = IOUtils.toByteArray(boundedInputStream); - LOG.info("RAW JSON: {}", new String(responseBody, StandardCharsets.UTF_8)); if (responseBody.length > MAX_PAGE_BYTES) { throw new IOException( "ServiceNow page exceeded max allowed size: " + MAX_PAGE_BYTES); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 4cd903c4..7bd80ed4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -16,12 +16,25 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -30,6 +43,7 @@ * Base Record reader class that provides a basic structure for Derived Record Reader classes. */ public abstract class ServiceNowBaseRecordReader extends RecordReader { + private static final Logger LOG = LoggerFactory.getLogger(ServiceNowRecordReader.class); protected ServiceNowInputSplit split; protected int pos; protected List tableFields; @@ -40,11 +54,121 @@ public abstract class ServiceNowBaseRecordReader extends RecordReader> results; protected Iterator> iterator; protected Map row; + protected final Gson gson = new Gson(); + protected final Type mapType = new TypeToken>() { }.getType(); + protected JsonReader jsonReader = null; public ServiceNowBaseRecordReader() { } + + /** + * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. + * Returns true when it assigned `row` to the next record. + * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). + */ + public boolean nextKeyValue() throws IOException { + // Ensure we have an active page/jsonReader + if (jsonReader == null) { + // Need to open the next page + boolean pageOpened; + try { + pageOpened = openNextPage(); + } catch (ServiceNowAPIException e) { + throw new IOException("Exception in nextKeyValue" + tableName, e); + } + if (!pageOpened) { + // No more pages + return false; + } + } - public abstract boolean nextKeyValue() throws IOException; + // At this point jsonReader is positioned inside the "result" array. + JsonToken token = jsonReader.peek(); + + if (token == JsonToken.BEGIN_OBJECT) { + LOG.debug("Reading record object for table {} at position {}", tableName, pos); + this.row = gson.fromJson(jsonReader, mapType); // assign row + pos++; + return true; + } + closeCurrentPage(); + return false; + } + + public boolean openNextPage() throws IOException, ServiceNowAPIException { + LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); + closeCurrentPage(); + LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); + RestAPIResponse resp = fetchData(); + LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); + InputStream in = resp.getBodyAsStream(); + if (in == null) { + return false; + } + this.jsonReader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + this.jsonReader.setLenient(true); + + // Position the reader to the "result" array: { "result": [ ... ], ... } + try { + JsonToken top; + try { + top = jsonReader.peek(); + LOG.debug("Peeking JSON token for table {}: {}", tableName, top); + } catch (IOException e) { + LOG.warn("Unexpected closure of stream while peeking JSON token for table {}", tableName, e); + closeCurrentPage(); + return false; + } + if (top == JsonToken.BEGIN_OBJECT) { + jsonReader.beginObject(); + while (jsonReader.hasNext()) { + String name = jsonReader.nextName(); + if (name.equals(ServiceNowConstants.RESULT)) { + if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + return true; + } else { + jsonReader.skipValue(); + break; + } + } else { + jsonReader.skipValue(); + } + } + } else if (top == JsonToken.BEGIN_ARRAY) { + jsonReader.beginArray(); + return true; + } else if (jsonReader.peek() == JsonToken.END_ARRAY) { + // empty result array — treat as no-more-data for this split/page + LOG.debug("openNextPage: found empty result array (no records). Closing and returning false."); + jsonReader.endArray(); + // cleanup + closeCurrentPage(); + return false; + } + } catch (IOException e) { + closeCurrentPage(); + throw e; + } + + // No "result" array not found, close the current page and return false + closeCurrentPage(); + return false; + } + + public void closeCurrentPage() { + if (this.jsonReader != null) { + try { + this.jsonReader.close(); + } catch (IOException e) { + LOG.warn("Error closing JSON reader", e); + } finally { + this.jsonReader = null; + } + } + } + + abstract RestAPIResponse fetchData() throws ServiceNowAPIException; public NullWritable getCurrentKey() { return NullWritable.get(); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index ba90b143..f48806a2 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -46,13 +46,9 @@ * Record reader that reads the entire contents of a ServiceNow table. */ public class ServiceNowMultiRecordReader extends ServiceNowBaseRecordReader { - - private static final Logger LOG = LoggerFactory.getLogger(ServiceNowMultiRecordReader.class); + private final ServiceNowMultiSourceConfig multiSourcePluginConf; private ServiceNowTableAPIClientImpl restApi; - private final Gson gson = new Gson(); - private final Type mapType = new TypeToken>() { }.getType(); - private JsonReader jsonReader = null; ServiceNowMultiRecordReader(ServiceNowMultiSourceConfig multiSourcePluginConf) { super(); @@ -70,41 +66,6 @@ public void initialize(InputSplit split, TaskAttemptContext context) { fetchSchema(restApi); } - @Override - /** - * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. - * Returns true when it assigned `row` to the next record. - * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). - */ - public boolean nextKeyValue() throws IOException { - // Ensure we have an active page/jsonReader - if (jsonReader == null) { - // Need to open the next page - boolean pageOpened; - try { - pageOpened = openNextPage(); - } catch (ServiceNowAPIException e) { - throw new IOException("Exception in nextKeyValue" + tableName, e); - } - if (!pageOpened) { - // No more pages - return false; - } - } - - // At this point jsonReader is positioned inside the "result" array. - JsonToken token = jsonReader.peek(); - - if (token == JsonToken.BEGIN_OBJECT) { - LOG.debug("Reading record object for table {} at position {}", tableName, pos); - this.row = gson.fromJson(jsonReader, mapType); // assign row - pos++; - return true; - } - closeCurrentPage(); - return false; - } - @Override public StructuredRecord getCurrentValue() throws IOException { StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); @@ -132,80 +93,6 @@ RestAPIResponse fetchData() throws ServiceNowAPIException { return restAPIResponse; } - private boolean openNextPage() throws IOException, ServiceNowAPIException { - LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); - closeCurrentPage(); - LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); - RestAPIResponse resp = fetchData(); - LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); - InputStream in = resp.getBodyAsStream(); - if (in == null) { - return false; - } - this.jsonReader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - this.jsonReader.setLenient(true); - - // Position the reader to the "result" array: { "result": [ ... ], ... } - try { - JsonToken top; - try { - top = jsonReader.peek(); - LOG.info("Peeking JSON token for table {}: {}", tableName, top); - } catch (IOException e) { - LOG.warn("Unexpected closure of stream while peeking JSON token for table {}", tableName, e); - closeCurrentPage(); - return false; - } - if (top == JsonToken.BEGIN_OBJECT) { - jsonReader.beginObject(); - while (jsonReader.hasNext()) { - String name = jsonReader.nextName(); - if (name.equals(ServiceNowConstants.RESULT)) { - if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { - jsonReader.beginArray(); - return true; - } else { - jsonReader.skipValue(); - break; - } - } else { - jsonReader.skipValue(); - } - } - } else if (top == JsonToken.BEGIN_ARRAY) { - jsonReader.beginArray(); - return true; - } else if (jsonReader.peek() == JsonToken.END_ARRAY) { - // empty result array — treat as no-more-data for this split/page - LOG.debug("openNextPage: found empty result array (no records). Closing and returning false."); - jsonReader.endArray(); - // cleanup - closeCurrentPage(); - return false; - } - } catch (IOException e) { - closeCurrentPage(); - throw e; - } - - // No "result" array not found, close the current page and return false - closeCurrentPage(); - return false; - } - - public void closeCurrentPage() { - LOG.info("Closing current page for table {}", tableName); - if (this.jsonReader != null) { - try { - this.jsonReader.close(); - } catch (IOException e) { - LOG.warn("Error closing JSON reader", e); - } finally { - this.jsonReader = null; - } - } - } - private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { // Fetch the schema try { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index da8b47e1..ab1b6174 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -49,9 +49,6 @@ public class ServiceNowRecordReader extends ServiceNowBaseRecordReader { private static final Logger LOG = LoggerFactory.getLogger(ServiceNowRecordReader.class); private final ServiceNowSourceConfig pluginConf; private ServiceNowTableAPIClientImpl restApi; - private final Gson gson = new Gson(); - private final Type mapType = new TypeToken>() { }.getType(); - private JsonReader jsonReader = null; public ServiceNowRecordReader(ServiceNowSourceConfig pluginConf) { super(); @@ -76,41 +73,6 @@ public void initialize(InputSplit split, Schema schema) { initializeSchema(tableName, schema); } - /** - * The refactored nextKeyValue() — uses Gson JsonReader to stream one record at a time. - * Returns true when it assigned `row` to the next record. - * Returns false only when there are no more pages/records (i.e., openNextPage() returns false). - */ - @Override - public boolean nextKeyValue() throws IOException { - // Ensure we have an active page/jsonReader - if (jsonReader == null) { - // Need to open the next page - boolean pageOpened; - try { - pageOpened = openNextPage(); - } catch (ServiceNowAPIException e) { - throw new IOException("Exception in nextKeyValue" + tableName, e); - } - if (!pageOpened) { - // No more pages - return false; - } - } - - // At this point jsonReader is positioned inside the "result" array. - JsonToken token = jsonReader.peek(); - - if (token == JsonToken.BEGIN_OBJECT) { - LOG.debug("Reading record object for table {} at position {}", tableName, pos); - this.row = gson.fromJson(jsonReader, mapType); // assign row - pos++; - return true; - } - closeCurrentPage(); - return false; - } - @Override public StructuredRecord getCurrentValue() throws IOException { StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); @@ -131,7 +93,7 @@ public StructuredRecord getCurrentValue() throws IOException { return recordBuilder.build(); } - private RestAPIResponse fetchData() throws ServiceNowAPIException { + RestAPIResponse fetchData() throws ServiceNowAPIException { // Get the table data RestAPIResponse restAPIResponse = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), pluginConf.getPageSize()); @@ -139,80 +101,6 @@ private RestAPIResponse fetchData() throws ServiceNowAPIException { return restAPIResponse; } - private boolean openNextPage() throws IOException, ServiceNowAPIException { - LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); - closeCurrentPage(); - LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); - RestAPIResponse resp = fetchData(); - LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); - InputStream in = resp.getBodyAsStream(); - if (in == null) { - return false; - } - this.jsonReader = new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - this.jsonReader.setLenient(true); - - // Position the reader to the "result" array: { "result": [ ... ], ... } - try { - JsonToken top; - try { - top = jsonReader.peek(); - LOG.info("Peeking JSON token for table {}: {}", tableName, top); - } catch (IOException e) { - LOG.warn("Unexpected closure of stream while peeking JSON token for table {}", tableName, e); - closeCurrentPage(); - return false; - } - if (top == JsonToken.BEGIN_OBJECT) { - jsonReader.beginObject(); - while (jsonReader.hasNext()) { - String name = jsonReader.nextName(); - if (name.equals(ServiceNowConstants.RESULT)) { - if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { - jsonReader.beginArray(); - return true; - } else { - jsonReader.skipValue(); - break; - } - } else { - jsonReader.skipValue(); - } - } - } else if (top == JsonToken.BEGIN_ARRAY) { - jsonReader.beginArray(); - return true; - } else if (jsonReader.peek() == JsonToken.END_ARRAY) { - // empty result array — treat as no-more-data for this split/page - LOG.debug("openNextPage: found empty result array (no records). Closing and returning false."); - jsonReader.endArray(); - // cleanup - closeCurrentPage(); - return false; - } - } catch (IOException e) { - closeCurrentPage(); - throw e; - } - - // No "result" array not found, close the current page and return false - closeCurrentPage(); - return false; - } - - public void closeCurrentPage() { - LOG.info("Closing current page for table {}", tableName); - if (this.jsonReader != null) { - try { - this.jsonReader.close(); - } catch (IOException e) { - LOG.warn("Error closing JSON reader", e); - } finally { - this.jsonReader = null; - } - } - } - protected void initialize(InputSplit split) { this.split = (ServiceNowInputSplit) split; this.pos = 0; From 611517352dbbb90cbfab170e0f75b530eba32a8a Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Thu, 15 Jan 2026 23:31:06 +0530 Subject: [PATCH 08/14] PLUGIN-1936: Rework --- .../plugin/servicenow/ServiceNowBaseConfig.java | 6 +++++- .../apiclient/ServiceNowTableAPIClientImpl.java | 15 +++++++++------ .../service/ServiceNowSinkAPIRequestImpl.java | 4 ++-- .../source/ServiceNowBaseRecordReader.java | 10 ++++++---- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 7daa7de3..3a4aa661 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -162,8 +162,12 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo } /** - * Determines if the "result" array in a ServiceNow REST API response is empty. It specifically looks for a top-level + * Checks if the "result" array in the ServiceNow REST API response is empty. + *

+ * Determines if the "result" array in a ServiceNow REST API response is empty by specifically looking for a top-level * key named "result". Once found, it opens the associated array and checks for the presence of a first element. + *

+ * * @param restAPIResponse The response object containing the JSON input stream * @return true, if the "result" array exists and is empty, or if the "result" key is never found; * false, if the array contains at least one element. diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 8a3687b4..638fcdbf 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -247,11 +247,8 @@ public RestAPIResponse fetchTableRecordsRetryableMode(String tableName, SourceVa int limit) throws ServiceNowAPIException { // Using AtomicReference to capture a value inside a lambda that needs to be accessed outside. AtomicReference responseRef = new AtomicReference<>(); - Callable fetchRecords = () -> { - RestAPIResponse restAPIResponse = fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit); - responseRef.set(restAPIResponse); - return true; - }; + Callable fetchRecords = () -> executeFetch(tableName, valueType, startDate, endDate, offset, limit, + responseRef); Retryer retryer = RetryerBuilder.newBuilder() .retryIfException(this::isExceptionRetryable) @@ -270,6 +267,13 @@ public RestAPIResponse fetchTableRecordsRetryableMode(String tableName, SourceVa return responseRef.get(); } + private boolean executeFetch(String tableName, SourceValueType type, String startDate, String endDate, int offset, + int limit, AtomicReference ref) throws ServiceNowAPIException { + RestAPIResponse response = fetchTableRecords(tableName, type, startDate, endDate, offset, limit); + ref.set(response); + return true; + } + /** * Fetch schema for actual value type * @param tableName ServiceNow table name for which schema is getting fetched @@ -565,7 +569,6 @@ public Map getRecordFromServiceNowTable(String tableName, String private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List columns, String tableName) throws ServiceNowAPIException { List> result = parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream()); - // List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody()); if (result != null && !result.isEmpty()) { Map firstRecord = result.get(0); for (String key : firstRecord.keySet()) { diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java index af64cd60..61f47342 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java @@ -122,8 +122,8 @@ public void createPostRequest(Map restRequestsMap, String a requestBuilder.setEntity(stringEntity); apiResponse = restApi.executePost(requestBuilder.build()); - JsonObject responseJSON = jsonParser.parse( - new InputStreamReader(apiResponse.getBodyAsStream(), StandardCharsets.UTF_8)).getAsJsonObject(); + JsonObject responseJSON = jsonParser.parse(new String(apiResponse.getResponseBody(), StandardCharsets.UTF_8)) + .getAsJsonObject().getAsJsonObject(); JsonArray servicedRequestsArray = responseJSON.get(ServiceNowConstants.SERVICED_REQUESTS).getAsJsonArray(); JsonElement failedRequestId = null; for (int i = 0; i < servicedRequestsArray.size(); i++) { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 7bd80ed4..7c063edf 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -60,11 +60,13 @@ public abstract class ServiceNowBaseRecordReader extends RecordReader + * The nextKeyValue() uses the jsonReader to read the next record from the current page. + *

+ * Returns true, when it assigned `row` to the next record. + * Returns false, only when there are no more pages/records (i.e., openNextPage() returns false). */ public boolean nextKeyValue() throws IOException { // Ensure we have an active page/jsonReader From 4b89ca50212af5ef06ddc9bc29f4835ffe43b755 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Thu, 15 Jan 2026 23:36:31 +0530 Subject: [PATCH 09/14] PLUGIN-1936: Remove extra DEBUG Logs --- .../plugin/servicenow/source/ServiceNowBaseRecordReader.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 7c063edf..489daecb 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -98,11 +98,8 @@ public boolean nextKeyValue() throws IOException { } public boolean openNextPage() throws IOException, ServiceNowAPIException { - LOG.debug("Opening next page for table {} at offset {}", tableName, split.getOffset()); closeCurrentPage(); - LOG.debug("Fetching data for table {} at offset {}", tableName, split.getOffset()); RestAPIResponse resp = fetchData(); - LOG.debug("Fetched data for table {} at offset {}", tableName, split.getOffset()); InputStream in = resp.getBodyAsStream(); if (in == null) { return false; From 6b70276fd5d21c4c3acdc3da3c00eeb8939935c9 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Fri, 16 Jan 2026 15:52:46 +0530 Subject: [PATCH 10/14] Return InputStream as a part of the RestAPIResponse object --- .../servicenow/ServiceNowBaseConfig.java | 5 +-- .../ServiceNowTableAPIClientImpl.java | 11 +++--- .../connector/ServiceNowConnector.java | 4 +- .../connector/ServiceNowRecordConverter.java | 2 + .../servicenow/restapi/RestAPIResponse.java | 39 +++++-------------- .../service/ServiceNowSinkAPIRequestImpl.java | 3 +- .../source/ServiceNowBaseRecordReader.java | 2 +- .../ServiceNowTableAPIClientImplTest.java | 8 ++-- .../connector/ServiceNowConnectorTest.java | 4 +- .../sink/ServiceNowRecordWriterTest.java | 6 +-- .../sink/ServiceNowSinkConfigTest.java | 6 +-- .../servicenow/sink/ServiceNowSinkTest.java | 8 ++-- .../source/ServiceNowInputFormatTest.java | 12 +++--- .../ServiceNowMultiRecordReaderTest.java | 5 ++- .../ServiceNowMultiSourceConfigTest.java | 14 +++---- .../source/ServiceNowMultiSourceTest.java | 12 +++--- .../source/ServiceNowRecordReaderTest.java | 11 ++++-- .../source/ServiceNowSourceConfigTest.java | 2 +- .../source/ServiceNowSourceTest.java | 12 +++--- 19 files changed, 78 insertions(+), 88 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 3a4aa661..da61a843 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -92,7 +92,7 @@ public void validateCredentials(FailureCollector collector) { } @VisibleForTesting - public void validateServiceNowConnection(FailureCollector collector) { + public void validateServiceNowConnection(FailureCollector collector) { try { ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection); restApi.getAccessToken(); @@ -174,8 +174,7 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo * @throws IOException If there is an error reading the input stream or parsing the JSON. */ public boolean isResultEmpty(RestAPIResponse restAPIResponse) throws IOException { - JsonReader reader = new JsonReader(new InputStreamReader(restAPIResponse.getBodyAsStream(), - StandardCharsets.UTF_8)); + JsonReader reader = new JsonReader(new InputStreamReader(restAPIResponse.getResponseStream())); reader.beginObject(); while (reader.hasNext()) { String name = reader.nextName(); diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 638fcdbf..1e3ecafc 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -369,7 +369,7 @@ public Schema fetchTableSchema(String tableName, String accessToken, SourceValue private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List columns, String tableName) throws ServiceNowAPIException { SchemaAPISchemaResponse schemaAPISchemaResponse = - GSON.fromJson(createJsonReader(restAPIResponse.getBodyAsStream()), SchemaAPISchemaResponse.class); + GSON.fromJson(createJsonReader(restAPIResponse.getResponseStream()), SchemaAPISchemaResponse.class); if (schemaAPISchemaResponse.getResult() == null || schemaAPISchemaResponse.getResult().isEmpty()) { throw new ServiceNowAPIException( @@ -403,7 +403,7 @@ private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List< private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List columns, String tableName, SourceValueType valueType) throws ServiceNowAPIException { - MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getBodyAsStream()); + MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseStream()); if (metadataAPISchemaResponse.getResult() == null || metadataAPISchemaResponse.getResult().getColumns() == null || metadataAPISchemaResponse.getResult().getColumns().isEmpty()) { throw new ServiceNowAPIException( @@ -525,7 +525,8 @@ public String createRecordInDisplayMode(String tableName, HttpEntity entity) thr private String getSystemId(RestAPIResponse restAPIResponse) { CreateRecordAPIResponse apiResponse = GSON.fromJson( - new InputStreamReader(restAPIResponse.getBodyAsStream(), StandardCharsets.UTF_8), CreateRecordAPIResponse.class); + new InputStreamReader(restAPIResponse.getResponseStream(), StandardCharsets.UTF_8), + CreateRecordAPIResponse.class); return apiResponse.getResult().get(ServiceNowConstants.SYSTEM_ID).toString(); } @@ -549,7 +550,7 @@ public Map getRecordFromServiceNowTable(String tableName, String restAPIResponse = executeGetWithRetries(requestBuilder.build()); APIResponse apiResponse = GSON.fromJson( - new InputStreamReader(restAPIResponse.getBodyAsStream(), StandardCharsets.UTF_8), APIResponse.class); + new InputStreamReader(restAPIResponse.getResponseStream(), StandardCharsets.UTF_8), APIResponse.class); return apiResponse.getResult().get(0); } @@ -568,7 +569,7 @@ public Map getRecordFromServiceNowTable(String tableName, String */ private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List columns, String tableName) throws ServiceNowAPIException { - List> result = parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream()); + List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseStream()); if (result != null && !result.isEmpty()) { Map firstRecord = result.get(0); for (String key : firstRecord.keySet()) { diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index 9ab05be4..0ee1fefd 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -135,7 +135,7 @@ private TableList listTables(String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); - return GSON.fromJson(serviceNowTableAPIClient.createJsonReader(apiResponse.getBodyAsStream()), TableList.class); + return GSON.fromJson(serviceNowTableAPIClient.createJsonReader(apiResponse.getResponseStream()), TableList.class); } public ConnectorSpec generateSpec(ConnectorContext connectorContext, ConnectorSpecRequest connectorSpecRequest) { @@ -184,7 +184,7 @@ private List getTableData(String tableName, int limit) requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap - (apiResponse.getBodyAsStream()); + (apiResponse.getResponseStream()); List recordList = new ArrayList<>(); Schema schema = getSchema(tableName); if (schema != null) { diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java index c7425a49..9ebccab9 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java @@ -27,11 +27,13 @@ import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; /** * Utility class for converting the record from ServiceNow data type to CDAP schema data types diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index 57663fcb..02e45052 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -20,8 +20,6 @@ import com.google.gson.JsonObject; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.util.ServiceNowConstants; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.BoundedInputStream; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -29,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -52,19 +49,18 @@ public class RestAPIResponse { private static final String REST_ERROR_MESSAGE = "Rest Api response has errors. Error message: %s."; private static final Set SUCCESS_CODES = new HashSet<>(Arrays.asList(HttpStatus.SC_CREATED, HttpStatus.SC_OK)); - private static final long MAX_PAGE_BYTES = 50L * 1024 * 1024; // 50 MB (Upper Bound) private final Map headers; @Nullable private final ServiceNowAPIException exception; // New: store byte array - private byte[] responseBody; + private InputStream responseStream; public RestAPIResponse( Map headers, - byte[] responseBody, + InputStream responseStream, @Nullable ServiceNowAPIException exception) { this.headers = headers; - this.responseBody = responseBody; + this.responseStream = responseStream; this.exception = exception; } @@ -105,20 +101,12 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN public static RestAPIResponse prepareResponse(HttpResponse httpResponse, Map headers, ServiceNowAPIException serviceNowAPIException) throws IOException { HttpEntity httpEntity = httpResponse.getEntity(); - byte[] responseBody = new byte[0]; InputStream inputStream; if (httpEntity != null) { inputStream = httpEntity.getContent(); - BoundedInputStream boundedInputStream = new BoundedInputStream( - inputStream, MAX_PAGE_BYTES + 1); // +1 to detect overflow - responseBody = IOUtils.toByteArray(boundedInputStream); - if (responseBody.length > MAX_PAGE_BYTES) { - throw new IOException( - "ServiceNow page exceeded max allowed size: " + MAX_PAGE_BYTES); - } - return new RestAPIResponse(headers, responseBody, serviceNowAPIException); + return new RestAPIResponse(headers, inputStream, serviceNowAPIException); } else { - return new RestAPIResponse(headers, responseBody, serviceNowAPIException); + return new RestAPIResponse(headers, null, serviceNowAPIException); } } @@ -155,19 +143,6 @@ public Map getHeaders() { return headers; } - @Nullable - public byte[] getResponseBody() { - return responseBody; - } - - /** - * Returns a fresh InputStream for the response body. Caller must close the stream. - * @return InputStream - */ - public InputStream getBodyAsStream() { - return responseBody == null ? null : new ByteArrayInputStream(responseBody); - } - @Nullable public ServiceNowAPIException getException() { return exception; @@ -176,4 +151,8 @@ public ServiceNowAPIException getException() { public boolean hasException() { return exception != null; } + + public InputStream getResponseStream() { + return responseStream; + } } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java index 61f47342..99f9ef31 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java @@ -25,6 +25,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.stream.JsonReader; import io.cdap.cdap.api.retry.RetryableException; import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; @@ -122,7 +123,7 @@ public void createPostRequest(Map restRequestsMap, String a requestBuilder.setEntity(stringEntity); apiResponse = restApi.executePost(requestBuilder.build()); - JsonObject responseJSON = jsonParser.parse(new String(apiResponse.getResponseBody(), StandardCharsets.UTF_8)) + JsonObject responseJSON = jsonParser.parse(new JsonReader(new InputStreamReader(apiResponse.getResponseStream()))) .getAsJsonObject().getAsJsonObject(); JsonArray servicedRequestsArray = responseJSON.get(ServiceNowConstants.SERVICED_REQUESTS).getAsJsonArray(); JsonElement failedRequestId = null; diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 489daecb..88f26084 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -100,7 +100,7 @@ public boolean nextKeyValue() throws IOException { public boolean openNextPage() throws IOException, ServiceNowAPIException { closeCurrentPage(); RestAPIResponse resp = fetchData(); - InputStream in = resp.getBodyAsStream(); + InputStream in = resp.getResponseStream(); if (in == null) { return false; } diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java index e1f2d18d..8c3b48b2 100644 --- a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -111,7 +111,7 @@ public void testFetchTableSchema_ActualValueType() throws Exception { "}"; byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.SCHEMA_API_BASED); @@ -145,7 +145,7 @@ public void testFetchTableSchema_GlideTimeFieldWithActualValueType() throws Exce byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("u_custom_table", @@ -186,7 +186,7 @@ public void testFetchTableSchema_DisplayValueType() throws Exception { "}"; byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.SCHEMA_API_BASED); @@ -225,7 +225,7 @@ public void testFetchTableSchema_StcFieldsWithDisplayValueType_ParseAsString() t "}"; byte[] body = jsonResponse.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); Schema schema = implSpy.fetchTableSchema("incident", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.METADATA_API_BASED); diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index ab9df750..ebb81cc0 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -136,9 +136,9 @@ public void testGenerateSpec() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index a44112d5..face9a78 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -101,7 +101,7 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { RestAPIResponse restAPIResponse = new RestAPIResponse( headers, null, null); Mockito.when(restApi.executePost(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); @@ -145,7 +145,7 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -193,7 +193,7 @@ public void testWriteWithUnservicedRequests() throws Exception { Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index e75782b0..5d08d717 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -311,7 +311,7 @@ public void testValidateSchema() throws Exception { Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); RestAPIResponse restAPIResponse = new RestAPIResponse( - headers, body, new ServiceNowAPIException("", mockResponse)); + headers, inputStream, new ServiceNowAPIException("", mockResponse)); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -331,7 +331,7 @@ public void testValidateSchema() throws Exception { PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); Mockito.when(restApi.executeGetWithRetries(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(Mockito.anyString(), Mockito.any(FailureCollector.class))).thenReturn(schema); - Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getBodyAsStream())) + Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getResponseStream())) .thenReturn(metadataAPISchemaResponse); try { config.validateSchema(schema, collector); @@ -369,7 +369,7 @@ public void testValidateSchemaWithOperation() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index e4d0c318..036860fe 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -95,9 +95,9 @@ public void testConfigurePipeline() throws Exception { byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); MockFailureCollector collector = new MockFailureCollector(); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); serviceNowSink.configurePipeline(mockPipelineConfigurer); Assert.assertNull(restAPIResponse.getException()); Assert.assertEquals(0, collector.getValidationFailures().size()); @@ -134,9 +134,9 @@ public void testPrepareRun() throws Exception { Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE))); Emitter> emitter = Mockito.mock(Emitter.class); Mockito.when(context.getInputSchema()).thenReturn(schema); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index 10269796..555fc405 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -147,9 +147,9 @@ public void testFetchTableInfo() throws Exception { "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("table", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -253,9 +253,9 @@ public void testFetchTableInfoReportingMode() throws Exception { "\"backgroundElementId\",\"type\":\"long\"},{\"name\":\"bgOrderPos\",\"type\":\"long\"},{\"name\":" + "\"description\",\"type\":[\"string\",\"null\"]},{\"name\":\"userId\",\"type\":\"string\"}]}"; Schema schema = Schema.parseJson(schemaString); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); Mockito.when(restApi.fetchTableSchema("proc_po", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); Mockito.when(restApi.fetchTableSchema("proc_po_item", SourceValueType.SHOW_ACTUAL_VALUE)).thenReturn(schema); @@ -300,9 +300,9 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.mockStatic(ServiceNowInputFormat.class); PowerMockito.whenNew(OAuthClient.class). diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 0d6f7c50..7f883de1 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -38,7 +38,9 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; @@ -178,7 +180,8 @@ public void testFetchData() throws Exception { " ]\n" + "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); - RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowMultiSourceConfig.getValueType(), serviceNowMultiSourceConfig.getStartDate(), serviceNowMultiSourceConfig.getEndDate(), split.getOffset(), diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index 2b1407b3..d42039ea 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -179,9 +179,9 @@ public void testValidate() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); @@ -214,7 +214,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); @@ -310,9 +310,9 @@ public void testValidateReferenceName() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); try { serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.fail("Exception is not thrown with valid reference name"); @@ -413,9 +413,9 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); Assert.assertEquals("Table name field must be specified.", diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index 41d06ec2..aef77575 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -159,9 +159,9 @@ public void testConfigurePipeline() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); Assert.assertNull(mockPipelineConfigurer.getOutputSchema()); Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); @@ -181,9 +181,9 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); try { serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); Assert.fail("Exception is not thrown for Non-Empty Tables"); @@ -287,9 +287,9 @@ public void testPrepareRun() throws Exception { InputStream inputStream = new ByteArrayInputStream(body); PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); Mockito.when(ServiceNowMultiInputFormat.setInput(Mockito.any(), Mockito.any())).thenReturn((tableInfo)); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index d4ea6841..fd0eb2cd 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -43,7 +43,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayList; @@ -305,7 +307,8 @@ public void testFetchData() throws Exception { " ]\n" + "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); - RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), @@ -381,7 +384,8 @@ public void testFetchDataReportingMode() throws Exception { " ]\n" + "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); - RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), @@ -422,7 +426,8 @@ public void testFetchDataOnInvalidTable() throws Exception { "\"status\": \"failure\"\n" + "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); - RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), body, null); + InputStream inputStream = new ByteArrayInputStream(body); + RestAPIResponse restAPIResponse = new RestAPIResponse(Collections.emptyMap(), inputStream, null); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index 5663cbb5..2fd96e0a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -627,7 +627,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); config.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index b0d3613d..6e6b853b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -174,9 +174,9 @@ public void testConfigurePipeline() throws Exception { PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(tableInfo); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -214,9 +214,9 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); try { serviceNowSource.configurePipeline(mockPipelineConfigurer); Assert.fail("Exception is not thrown for Non-Empty Tables"); @@ -304,9 +304,9 @@ public void testPrepareRun() throws Exception { "}"; byte[] body = responseBody.getBytes(StandardCharsets.UTF_8); InputStream inputStream = new ByteArrayInputStream(body); - RestAPIResponse restAPIResponse = new RestAPIResponse(headers, body, null); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, inputStream, null); PowerMockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); - Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getBodyAsStream())).thenReturn(result); + Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseStream())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); From 89642d318a03b0da7cc6d7c346632c4f7be950a2 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Tue, 3 Feb 2026 12:43:44 +0530 Subject: [PATCH 11/14] Keep the Http Connection open --- .../servicenow/restapi/RestAPIClient.java | 21 +++++++++++++++---- .../servicenow/restapi/RestAPIResponse.java | 8 ++++++- .../source/ServiceNowBaseRecordReader.java | 1 + 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index a99ac19a..86da1289 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -62,11 +62,25 @@ public abstract class RestAPIClient { /* Read Timeout in ms for waiting for data after the connection is established */ private static final int DEFAULT_READ_TIMEOUT_MS = 300000; + // These settings are the "Ideal Cap" for parallel processing + private static final int MAX_CONNECTIONS = 200; + private static final int MAX_PER_ROUTE = 100; + private static final int TIMEOUT_MILLIS = 120000; // 2 minutes + private static final RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MS) .setSocketTimeout(DEFAULT_READ_TIMEOUT_MS) .build(); + private static final CloseableHttpClient httpClient = HttpClientBuilder.create() + .setDefaultRequestConfig(requestConfig) + .setMaxConnTotal(MAX_CONNECTIONS) + .setMaxConnPerRoute(MAX_PER_ROUTE) + .setConnectionTimeToLive(5, TimeUnit.MINUTES) + .evictIdleConnections(30, TimeUnit.SECONDS) + .evictExpiredConnections() + .build(); + /** * Executes the Rest API request and returns the response. * @@ -77,10 +91,9 @@ public RestAPIResponse executeGet(RestAPIRequest request) throws IOException { HttpGet httpGet = new HttpGet(request.getUrl()); request.getHeaders().entrySet().forEach(e -> httpGet.addHeader(e.getKey(), e.getValue())); - try (CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig).build()) { - try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { - return RestAPIResponse.parse(httpResponse, request.getResponseHeaders()); - } + try { + CloseableHttpResponse httpResponse = httpClient.execute(httpGet); + return RestAPIResponse.parse(httpResponse, request.getResponseHeaders()); } catch (ConnectTimeoutException | SocketException e) { ServiceNowAPIException exception = new ServiceNowAPIException(e, null); return new RestAPIResponse(Collections.emptyMap(), null, exception); diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index 02e45052..df3e2c4a 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -52,7 +52,7 @@ public class RestAPIResponse { private final Map headers; @Nullable private final ServiceNowAPIException exception; - // New: store byte array + // Input stream of the response body. private InputStream responseStream; public RestAPIResponse( @@ -98,6 +98,12 @@ public static RestAPIResponse parse(HttpResponse httpResponse, String... headerN } } + public void close() throws IOException { + if (responseStream != null) { + responseStream.close(); + } + } + public static RestAPIResponse prepareResponse(HttpResponse httpResponse, Map headers, ServiceNowAPIException serviceNowAPIException) throws IOException { HttpEntity httpEntity = httpResponse.getEntity(); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 88f26084..a8181475 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -143,6 +143,7 @@ public boolean openNextPage() throws IOException, ServiceNowAPIException { jsonReader.endArray(); // cleanup closeCurrentPage(); + resp.close(); return false; } } catch (IOException e) { From 1dea8a3eba8e91cda638c8fdd2f5812b916d796e Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Tue, 3 Feb 2026 18:20:04 +0530 Subject: [PATCH 12/14] Add constants and close the response stream properly --- .../servicenow/restapi/RestAPIClient.java | 19 +++++++++++++------ .../source/ServiceNowBaseRecordReader.java | 12 +++++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index 86da1289..9f030c1c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.InputStream; import java.net.SocketException; import java.util.Collections; import java.util.concurrent.Callable; @@ -56,17 +55,25 @@ public abstract class RestAPIClient { private static final Logger LOG = LoggerFactory.getLogger(RestAPIClient.class); - /* Connect Timout in ms for establishing the conenction with the server */ + /* Connect Timeout in ms for establishing the connection with the server */ private static final int DEFAULT_CONNECT_TIMEOUT_MS = 120000; /* Read Timeout in ms for waiting for data after the connection is established */ private static final int DEFAULT_READ_TIMEOUT_MS = 300000; - // These settings are the "Ideal Cap" for parallel processing + /* Maximum total connections. */ private static final int MAX_CONNECTIONS = 200; + + // Maximum connections per route. */ private static final int MAX_PER_ROUTE = 100; - private static final int TIMEOUT_MILLIS = 120000; // 2 minutes + /** The maximum time a connection is allowed to live in the pool before being retired. + * Helps avoid "stale connection" errors during long-running pipelines. */ + private static final long CONNECTION_TTL_MINUTES = 5; + + /** The interval at which idle connections are scanned and closed by the background monitor. */ + private static final long IDLE_EVICTION_SECONDS = 30; + private static final RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MS) .setSocketTimeout(DEFAULT_READ_TIMEOUT_MS) @@ -76,8 +83,8 @@ public abstract class RestAPIClient { .setDefaultRequestConfig(requestConfig) .setMaxConnTotal(MAX_CONNECTIONS) .setMaxConnPerRoute(MAX_PER_ROUTE) - .setConnectionTimeToLive(5, TimeUnit.MINUTES) - .evictIdleConnections(30, TimeUnit.SECONDS) + .setConnectionTimeToLive(CONNECTION_TTL_MINUTES, TimeUnit.MINUTES) + .evictIdleConnections(IDLE_EVICTION_SECONDS, TimeUnit.SECONDS) .evictExpiredConnections() .build(); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index a8181475..9a38b372 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -143,7 +143,7 @@ public boolean openNextPage() throws IOException, ServiceNowAPIException { jsonReader.endArray(); // cleanup closeCurrentPage(); - resp.close(); + closeRestAPIResponse(resp); return false; } } catch (IOException e) { @@ -168,6 +168,16 @@ public void closeCurrentPage() { } } + public void closeRestAPIResponse(RestAPIResponse resp) { + if (resp != null) { + try { + resp.close(); + } catch (IOException e) { + LOG.warn("Error closing RestAPIResponse", e); + } + } + } + abstract RestAPIResponse fetchData() throws ServiceNowAPIException; public NullWritable getCurrentKey() { From e20b79594bb7db7482a9c1a35ab2845ca0f3c025 Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Mon, 16 Feb 2026 00:57:00 +0530 Subject: [PATCH 13/14] Fix failing JUnits --- .../ServiceNowTableAPIClientImpl.java | 1 + .../servicenow/restapi/RestAPIClient.java | 39 +++++++++++++------ .../connector/ServiceNowConnectorTest.java | 7 ++++ .../servicenow/restapi/RestAPIClientTest.java | 24 ++++-------- .../ServiceNowMultiRecordReaderTest.java | 5 +++ .../ServiceNowMultiSourceConfigTest.java | 5 +++ 6 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 1e3ecafc..b0eb9496 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -90,6 +90,7 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient { public static JsonArray serviceNowJsonResultArray; public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf, Boolean useConnection) { + super(); this.conf = conf; this.schemaType = getSchemaTypeBasedOnUseConnection(useConnection); } diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index 9f030c1c..8821a8dd 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -64,9 +64,6 @@ public abstract class RestAPIClient { /* Maximum total connections. */ private static final int MAX_CONNECTIONS = 200; - // Maximum connections per route. */ - private static final int MAX_PER_ROUTE = 100; - /** The maximum time a connection is allowed to live in the pool before being retired. * Helps avoid "stale connection" errors during long-running pipelines. */ private static final long CONNECTION_TTL_MINUTES = 5; @@ -79,14 +76,34 @@ public abstract class RestAPIClient { .setSocketTimeout(DEFAULT_READ_TIMEOUT_MS) .build(); - private static final CloseableHttpClient httpClient = HttpClientBuilder.create() - .setDefaultRequestConfig(requestConfig) - .setMaxConnTotal(MAX_CONNECTIONS) - .setMaxConnPerRoute(MAX_PER_ROUTE) - .setConnectionTimeToLive(CONNECTION_TTL_MINUTES, TimeUnit.MINUTES) - .evictIdleConnections(IDLE_EVICTION_SECONDS, TimeUnit.SECONDS) - .evictExpiredConnections() - .build(); + private final CloseableHttpClient httpClient; + + /* Default constructor to initialize the HttpClient. */ + protected RestAPIClient() { + this.httpClient = getHttpClient(); + } + + /* Lazy Holder to protect unit tests from premature static initialization errors. */ + static class HttpClientHolder { + static final CloseableHttpClient HTTP_CLIENT = createClient(); + + private static CloseableHttpClient createClient() { + return HttpClientBuilder.create() + .setMaxConnTotal(MAX_CONNECTIONS) + .setMaxConnPerRoute(MAX_CONNECTIONS) + .setConnectionTimeToLive(CONNECTION_TTL_MINUTES, TimeUnit.MINUTES) + .evictIdleConnections(IDLE_EVICTION_SECONDS, TimeUnit.SECONDS) + .setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MS) + .setSocketTimeout(DEFAULT_READ_TIMEOUT_MS).build()) + .build(); + } + } + /** + * Protected method to return the HttpClient instance. This is used to mock the HttpClient in unit tests. + */ + protected CloseableHttpClient getHttpClient() { + return HttpClientHolder.HTTP_CLIENT; + } /** * Executes the Rest API request and returns the response. diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index ebb81cc0..d893b6a0 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -26,6 +26,7 @@ import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; +import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.source.ServiceNowBaseSourceConfig; import io.cdap.plugin.servicenow.source.ServiceNowInputFormat; @@ -98,6 +99,9 @@ public void initialize() { public void testTest() throws Exception { MockFailureCollector collector = new MockFailureCollector(); ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + CloseableHttpClient mockHttpClient = Mockito.mock(CloseableHttpClient.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(mockHttpClient); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); @@ -109,6 +113,9 @@ public void testTest() throws Exception { @Test public void testTestWithInvalidToken() throws Exception { ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); + CloseableHttpClient mockHttpClient = Mockito.mock(CloseableHttpClient.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(mockHttpClient); ServiceNowConnector serviceNowConnector = new ServiceNowConnector(serviceNowSourceConfig.getConnection()); serviceNowConnector.test(context); Assert.assertEquals(1, context.getFailureCollector().getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java index 93649af7..b288bd6e 100644 --- a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java @@ -40,16 +40,14 @@ public void testExecuteGet_throwRetryableException() throws IOException { Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine); CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class); - HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); - PowerMockito.mockStatic(HttpClientBuilder.class); - PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); - Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); ServiceNowTableAPIRequestBuilder builder = new ServiceNowTableAPIRequestBuilder("url"); RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(httpClient); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); @@ -64,16 +62,14 @@ public void testExecuteGet_throwNonRetryableException() throws IOException { Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine); CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class); - HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); - PowerMockito.mockStatic(HttpClientBuilder.class); - PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); - Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); ServiceNowTableAPIRequestBuilder builder = new ServiceNowTableAPIRequestBuilder("url"); RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(httpClient); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); @@ -108,10 +104,6 @@ public void testExecuteGet_StatusOk() throws IOException { @Test public void testExecuteGet_throwConnectTimeoutException_markAsRetryable() throws IOException { CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class); - HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); - PowerMockito.mockStatic(HttpClientBuilder.class); - PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); - Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); Mockito.when(httpClient.execute(Mockito.any())) .thenThrow(new ConnectTimeoutException("Connection timed out")); @@ -119,6 +111,8 @@ public void testExecuteGet_throwConnectTimeoutException_markAsRetryable() throws RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(httpClient); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); @@ -131,10 +125,6 @@ public void testExecuteGet_throwConnectTimeoutException_markAsRetryable() throws @Test public void testExecuteGet_throwSocketException_markAsRetryable() throws IOException { CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class); - HttpClientBuilder httpClientBuilder = Mockito.mock(HttpClientBuilder.class); - PowerMockito.mockStatic(HttpClientBuilder.class); - PowerMockito.when(HttpClientBuilder.create()).thenReturn(httpClientBuilder); - Mockito.when(httpClientBuilder.build()).thenReturn(httpClient); Mockito.when(httpClient.execute(Mockito.any())) .thenThrow(new SocketException()); @@ -142,6 +132,8 @@ public void testExecuteGet_throwSocketException_markAsRetryable() throws IOExcep RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(httpClient); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 7f883de1..cc9dfc61 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -24,7 +24,9 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; +import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.oltu.oauth2.common.exception.OAuthProblemException; import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.junit.Assert; @@ -216,6 +218,9 @@ public void testFetchDataOnInvalidTable() .setTableNameField("tablename") .buildMultiSource(); + CloseableHttpClient mockHttpClient = Mockito.mock(CloseableHttpClient.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(mockHttpClient); String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index d42039ea..946e661c 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -20,7 +20,9 @@ import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; +import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -80,6 +82,9 @@ public void testValidateInvalidConnection() { .setEndDate("2021-12-31") .setTableNameField("tablename") .buildMultiSource(); + CloseableHttpClient mockHttpClient = Mockito.mock(CloseableHttpClient.class); + PowerMockito.stub(PowerMockito.method(RestAPIClient.class, "getHttpClient")) + .toReturn(mockHttpClient); try { serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.fail("Exception is not thrown if connection is successful"); From ed1aa53712cec5f44d34fccc9ad4fd17a30a28fb Mon Sep 17 00:00:00 2001 From: sgarg-CS Date: Mon, 16 Feb 2026 00:58:14 +0530 Subject: [PATCH 14/14] Use JsonObject instead of Map --- .../ServiceNowTableAPIClientImpl.java | 13 +++++----- .../connector/ServiceNowConnector.java | 3 ++- .../connector/ServiceNowRecordConverter.java | 6 +++-- .../plugin/servicenow/model/APIResponse.java | 6 +++-- .../source/ServiceNowBaseRecordReader.java | 5 ++-- .../connector/ServiceNowConnectorTest.java | 9 ++++--- .../sink/ServiceNowRecordWriterTest.java | 24 +++++++++--------- .../servicenow/sink/ServiceNowSinkTest.java | 10 ++++---- .../source/ServiceNowInputFormatTest.java | 25 ++++++++++--------- .../ServiceNowMultiRecordReaderTest.java | 7 +++--- .../ServiceNowMultiSourceConfigTest.java | 25 ++++++++++--------- .../source/ServiceNowMultiSourceTest.java | 19 +++++++------- .../source/ServiceNowRecordReaderTest.java | 25 ++++++++++--------- .../source/ServiceNowSourceTest.java | 19 +++++++------- 14 files changed, 104 insertions(+), 92 deletions(-) diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index b0eb9496..c0bb8f41 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -203,7 +203,7 @@ public List> parseResponseToResultListOfMap(String responseB return GSON.fromJson(ja, type); } - public List> parseResponseToResultListOfMap(InputStream in) { + public List parseResponseToResultListOfMap(InputStream in) { APIResponse apiResponse = GSON.fromJson(new JsonReader(new InputStreamReader(in, StandardCharsets.UTF_8)), APIResponse.class); return apiResponse.getResult(); @@ -538,7 +538,7 @@ private String getSystemId(RestAPIResponse restAPIResponse) { * @param tableName The ServiceNow table name * @param query The query */ - public Map getRecordFromServiceNowTable(String tableName, String query) + public JsonObject getRecordFromServiceNowTable(String tableName, String query) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( @@ -570,12 +570,11 @@ public Map getRecordFromServiceNowTable(String tableName, String */ private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List columns, String tableName) throws ServiceNowAPIException { - List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseStream()); + List result = parseResponseToResultListOfMap(restAPIResponse.getResponseStream()); if (result != null && !result.isEmpty()) { - Map firstRecord = result.get(0); - for (String key : firstRecord.keySet()) { - columns.add(new ServiceNowColumn(key, "string")); - } + result.get(0).entrySet().forEach(entry -> + columns.add(new ServiceNowColumn(entry.getKey(), "string")) + ); return SchemaBuilder.constructSchema(tableName, columns); } return null; diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index 0ee1fefd..6d9af4fc 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.connector; import com.google.gson.Gson; +import com.google.gson.JsonObject; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; @@ -183,7 +184,7 @@ private List getTableData(String tableName, int limit) requestBuilder.setAuthHeader(accessToken); requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); - List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap + List result = serviceNowTableAPIClient.parseResponseToResultListOfMap (apiResponse.getResponseStream()); List recordList = new ArrayList<>(); Schema schema = getSchema(tableName); diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java index 9ebccab9..56b32171 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.connector; import com.google.common.annotations.VisibleForTesting; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; @@ -84,9 +85,10 @@ public class ServiceNowRecordConverter { DateTimeFormatter.ofPattern("HH:mm") )); - public static void convertToValue(String fieldName, Schema fieldSchema, Map record, + public static void convertToValue(String fieldName, Schema fieldSchema, JsonObject record, StructuredRecord.Builder recordBuilder) { - String fieldValue = record.get(fieldName); + String fieldValue = record.has(fieldName) && !record.get(fieldName).isJsonNull() ? record.get(fieldName) + .getAsString() : null; if (fieldValue == null || fieldValue.isEmpty()) { // Set 'null' value as it is recordBuilder.set(fieldName, null); diff --git a/src/main/java/io/cdap/plugin/servicenow/model/APIResponse.java b/src/main/java/io/cdap/plugin/servicenow/model/APIResponse.java index c191e9a6..e5e37417 100644 --- a/src/main/java/io/cdap/plugin/servicenow/model/APIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/model/APIResponse.java @@ -16,6 +16,8 @@ package io.cdap.plugin.servicenow.model; +import com.google.gson.JsonObject; + import java.util.List; import java.util.Map; @@ -24,9 +26,9 @@ */ public class APIResponse { - private List> result; + private List result; - public List> getResult() { + public List getResult() { return result; } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java index 9a38b372..945ed095 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseRecordReader.java @@ -17,6 +17,7 @@ package io.cdap.plugin.servicenow.source; import com.google.gson.Gson; +import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; @@ -53,7 +54,7 @@ public abstract class ServiceNowBaseRecordReader extends RecordReader> results; protected Iterator> iterator; - protected Map row; + protected JsonObject row; protected final Gson gson = new Gson(); protected final Type mapType = new TypeToken>() { }.getType(); protected JsonReader jsonReader = null; @@ -89,7 +90,7 @@ public boolean nextKeyValue() throws IOException { if (token == JsonToken.BEGIN_OBJECT) { LOG.debug("Reading record object for table {} at position {}", tableName, pos); - this.row = gson.fromJson(jsonReader, mapType); // assign row + this.row = gson.fromJson(jsonReader, JsonObject.class); // assign row pos++; return true; } diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index d893b6a0..94be7c57 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -15,6 +15,7 @@ */ package io.cdap.plugin.servicenow.connector; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.connector.ConnectorContext; @@ -126,10 +127,10 @@ public void testGenerateSpec() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index face9a78..0ecd893b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -93,10 +93,10 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject1 = new JsonObject(); + List result = new ArrayList<>(); + jsonObject1.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse( headers, null, null); @@ -138,10 +138,10 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject1 = new JsonObject(); + List result = new ArrayList<>(); + jsonObject1.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); @@ -186,10 +186,10 @@ public void testWriteWithUnservicedRequests() throws Exception { ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject1 = new JsonObject(); + List result = new ArrayList<>(); + jsonObject1.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, null, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index 036860fe..05cf3b4a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -87,7 +87,7 @@ public void testConfigurePipeline() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); + List result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + @@ -112,10 +112,10 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index 555fc405..6b3b82d0 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -17,6 +17,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -74,10 +75,10 @@ public void testFetchTableInfo() throws Exception { SourceQueryMode mode = SourceQueryMode.TABLE; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -180,10 +181,10 @@ public void testFetchTableInfoReportingMode() throws Exception { SourceQueryMode mode = SourceQueryMode.REPORTING; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -290,10 +291,10 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { SourceQueryMode mode = SourceQueryMode.TABLE; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index cc9dfc61..e03bf6ef 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; @@ -107,9 +108,9 @@ public void testConvertToValueInvalidFieldType() { Schema fieldSchema = Schema.recordOf("record", Schema.Field.of("TimeField", Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS))); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(fieldSchema); - Map map = new HashMap<>(); - map.put("TimeField", "value"); - ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, map, recordBuilder); + JsonObject record = new JsonObject(); + record.addProperty("TimeField", "value"); + ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, record, recordBuilder); } @Test diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index 946e661c..cdef326b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; @@ -116,10 +117,10 @@ public void testValidate() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); String responseBody = "{\n" + " \"result\": [\n" + " {\n" + @@ -247,10 +248,10 @@ public void testValidateReferenceName() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); String responseBody = "{\n" + " \"result\": [\n" + " {\n" + @@ -350,10 +351,10 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); String responseBody = "{\n" + " \"result\": [\n" + " {\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index aef77575..7de5d35b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.validation.ValidationException; @@ -90,10 +91,10 @@ public void testConfigurePipeline() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -174,7 +175,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); + List result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + @@ -216,10 +217,10 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index fd0eb2cd..2e5ef45b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; @@ -134,10 +135,10 @@ public void testConvertToValue() { Schema fieldSchema = Schema.recordOf("record", Schema.Field.of("TimeField", Schema.of(Schema.LogicalType.TIMESTAMP_MILLIS))); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(fieldSchema); - Map map = new HashMap<>(); - map.put("TimeField", "value"); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("TimeField", "value"); thrown.expect(IllegalStateException.class); - ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, map, recordBuilder); + ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, jsonObject, recordBuilder); } @Test @@ -160,12 +161,12 @@ public void testConvertToDateTimeValue() { ); for (String value : dateTimeValues) { - Map inputMap = new HashMap<>(); - inputMap.put("DateTimeField", value); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("DateTimeField", value); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(recordSchema); try { - ServiceNowRecordConverter.convertToValue("DateTimeField", fieldSchema, inputMap, recordBuilder); + ServiceNowRecordConverter.convertToValue("DateTimeField", fieldSchema, jsonObject, recordBuilder); StructuredRecord record = recordBuilder.build(); Assert.assertNotNull("Parsed datetime should not be null for input: " + value, record.get("DateTimeField")); @@ -191,12 +192,12 @@ public void testConvertToDateValue() { ); for (String value : dateValues) { - Map inputMap = new HashMap<>(); - inputMap.put("DateField", value); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("DateField", value); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(recordSchema); try { - ServiceNowRecordConverter.convertToValue("DateField", fieldSchema, inputMap, recordBuilder); + ServiceNowRecordConverter.convertToValue("DateField", fieldSchema, jsonObject, recordBuilder); StructuredRecord record = recordBuilder.build(); Assert.assertNotNull("Parsed date should not be null for input: " + value, record.get("DateField")); @@ -220,12 +221,12 @@ public void testConvertToTimeValue() { ); for (String value : timeValues) { - Map inputMap = new HashMap<>(); - inputMap.put("TimeField", value); + JsonObject jsonObject = new JsonObject(); + jsonObject.addProperty("TimeField", value); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(recordSchema); try { - ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, inputMap, recordBuilder); + ServiceNowRecordConverter.convertToValue("TimeField", fieldSchema, jsonObject, recordBuilder); StructuredRecord record = recordBuilder.build(); Assert.assertNotNull("Parsed date should not be null for input: " + value, record.get("TimeField")); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 6e6b853b..a62bdee4 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -16,6 +16,7 @@ package io.cdap.plugin.servicenow.source; +import com.google.gson.JsonObject; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.cdap.etl.api.validation.ValidationException; @@ -102,10 +103,10 @@ public void testConfigurePipeline() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token1"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - Map map = new HashMap<>(); - List> result = new ArrayList<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -207,7 +208,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); + List result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + @@ -235,10 +236,10 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); - List> result = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("key", "value"); - result.add(map); + JsonObject jsonObject = new JsonObject(); + List result = new ArrayList<>(); + jsonObject.addProperty("key", "value"); + result.add(jsonObject); Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" +