diff --git a/.github/workflows/internal-release.yml b/.github/workflows/internal-release.yml index e8b8ee61..ce1145b3 100644 --- a/.github/workflows/internal-release.yml +++ b/.github/workflows/internal-release.yml @@ -1,4 +1,4 @@ -name: Publish package to the JFROG Artifactory +name: Publish v3 module to the JFROG Artifactory on: push: tags-ignore: @@ -6,16 +6,17 @@ on: paths-ignore: - "*.md" branches: - - release/* + - v3-release/* jobs: - build-and-deploy: + build-and-deploy-v3: uses: ./.github/workflows/shared-build-and-deploy.yml with: ref: ${{ github.ref_name }} server-id: central profile: jfrog tag: 'internal' + module: 'v3' secrets: server-username: ${{ secrets.ARTIFACTORY_USERNAME }} server-password: ${{ secrets.ARTIFACTORY_PASSWORD }} @@ -23,4 +24,4 @@ jobs: gpg-passphrase: ${{ secrets.JFROG_GPG_PASSPHRASE }} skyflow-credentials: ${{ secrets.SKYFLOW_CREDENTIALS }} >> .env test-expired-token: ${{ secrets.TEST_EXPIRED_TOKEN }} >> .env - test-reusable-token: ${{ secrets.TEST_REUSABLE_TOKEN }} >> .env + test-reusable-token: ${{ secrets.TEST_REUSABLE_TOKEN }} >> .env \ No newline at end of file diff --git a/.github/workflows/shared-build-and-deploy.yml b/.github/workflows/shared-build-and-deploy.yml index c9cadd2c..59d2b27c 100644 --- a/.github/workflows/shared-build-and-deploy.yml +++ b/.github/workflows/shared-build-and-deploy.yml @@ -21,6 +21,13 @@ on: description: 'Release Tag' required: true type: string + + module: + description: 'Module to build and publish' + required: false + type: string + default: '' + secrets: server-username: required: true @@ -99,7 +106,7 @@ jobs: git checkout ${{ env.branch_name }} fi - git add pom.xml + git add v3/pom.xml if [[ "${{ inputs.tag }}" == "internal" ]]; then git commit -m "[AUTOMATED] Private Release ${{ steps.previoustag.outputs.tag }}-dev-$(git rev-parse --short $GITHUB_SHA)" git push origin ${{ github.ref_name }} -f @@ -125,9 +132,14 @@ jobs: json: ${{ secrets.TEST_CREDENTIALS_FILE_STRING }} - name: Publish package - run: mvn --batch-mode deploy -P ${{ inputs.profile }} + run: | + if [ -n "${{ inputs.module }}" ]; then + mvn --batch-mode -pl v3 -am deploy -P jfrog + else + mvn --batch-mode deploy -P ${{ inputs.profile }} + fi env: SERVER_USERNAME: ${{ secrets.server-username }} SERVER_PASSWORD: ${{ secrets.server-password }} - GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml index d971c1b5..a18bd86f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -5,13 +5,15 @@ 4.0.0 com.skyflow - skyflow-java + skyflow 1.0.0 + ../pom.xml common 1.0.0 + 8 8 diff --git a/common/src/main/java/com/skyflow/errors/ErrorMessage.java b/common/src/main/java/com/skyflow/errors/ErrorMessage.java index e3b0bbbe..0084d727 100644 --- a/common/src/main/java/com/skyflow/errors/ErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/ErrorMessage.java @@ -6,6 +6,7 @@ public enum ErrorMessage { // Client initialization VaultIdAlreadyInConfigList("%s0 Validation error. VaultId is present in an existing config. Specify a new vaultId in config."), VaultIdNotInConfigList("%s0 Validation error. VaultId is missing from the config. Specify the vaultIds from configs."), + OnlySingleVaultConfigAllowed("%s0 Validation error. A vault config already exists. Cannot add another vault config."), ConnectionIdAlreadyInConfigList("%s0 Validation error. ConnectionId is present in an existing config. Specify a connectionId in config."), ConnectionIdNotInConfigList("%s0 Validation error. ConnectionId is missing from the config. Specify the connectionIds from configs."), EmptyCredentials("%s0 Validation error. Invalid credentials. Credentials must not be empty."), diff --git a/common/src/main/java/com/skyflow/generated/rest/ApiClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/ApiClient.java similarity index 73% rename from common/src/main/java/com/skyflow/generated/rest/ApiClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/ApiClient.java index f49fc4c3..721942e9 100644 --- a/common/src/main/java/com/skyflow/generated/rest/ApiClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/ApiClient.java @@ -1,11 +1,11 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest; +package com.skyflow.generated.auth.rest; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.Suppliers; -import com.skyflow.generated.rest.resources.authentication.AuthenticationClient; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.Suppliers; +import com.skyflow.generated.auth.rest.resources.authentication.AuthenticationClient; import java.util.function.Supplier; public class ApiClient { diff --git a/common/src/main/java/com/skyflow/generated/rest/ApiClientBuilder.java b/common/src/main/java/com/skyflow/generated/auth/rest/ApiClientBuilder.java similarity index 91% rename from common/src/main/java/com/skyflow/generated/rest/ApiClientBuilder.java rename to common/src/main/java/com/skyflow/generated/auth/rest/ApiClientBuilder.java index b361812a..aed3ed24 100644 --- a/common/src/main/java/com/skyflow/generated/rest/ApiClientBuilder.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/ApiClientBuilder.java @@ -1,10 +1,10 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest; +package com.skyflow.generated.auth.rest; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.Environment; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.Environment; import okhttp3.OkHttpClient; public final class ApiClientBuilder { diff --git a/common/src/main/java/com/skyflow/generated/rest/AsyncApiClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClient.java similarity index 73% rename from common/src/main/java/com/skyflow/generated/rest/AsyncApiClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClient.java index d9163e14..674a9ef9 100644 --- a/common/src/main/java/com/skyflow/generated/rest/AsyncApiClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClient.java @@ -1,11 +1,11 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest; +package com.skyflow.generated.auth.rest; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.Suppliers; -import com.skyflow.generated.rest.resources.authentication.AsyncAuthenticationClient; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.Suppliers; +import com.skyflow.generated.auth.rest.resources.authentication.AsyncAuthenticationClient; import java.util.function.Supplier; public class AsyncApiClient { diff --git a/common/src/main/java/com/skyflow/generated/rest/AsyncApiClientBuilder.java b/common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClientBuilder.java similarity index 91% rename from common/src/main/java/com/skyflow/generated/rest/AsyncApiClientBuilder.java rename to common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClientBuilder.java index 8e4b8474..2e30d45a 100644 --- a/common/src/main/java/com/skyflow/generated/rest/AsyncApiClientBuilder.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/AsyncApiClientBuilder.java @@ -1,10 +1,10 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest; +package com.skyflow.generated.auth.rest; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.Environment; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.Environment; import okhttp3.OkHttpClient; public final class AsyncApiClientBuilder { diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientApiException.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientApiException.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/core/ApiClientApiException.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientApiException.java index a4487b1e..dc4c008c 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientApiException.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientApiException.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.ArrayList; import java.util.HashMap; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientException.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientException.java similarity index 89% rename from common/src/main/java/com/skyflow/generated/rest/core/ApiClientException.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientException.java index 7987eba6..f08afa2e 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientException.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientException.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; /** * This class serves as the base exception for all errors in the SDK. diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientHttpResponse.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientHttpResponse.java similarity index 95% rename from common/src/main/java/com/skyflow/generated/rest/core/ApiClientHttpResponse.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientHttpResponse.java index 9c81f1f5..733c056c 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ApiClientHttpResponse.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ApiClientHttpResponse.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.ArrayList; import java.util.HashMap; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ClientOptions.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ClientOptions.java similarity index 99% rename from common/src/main/java/com/skyflow/generated/rest/core/ClientOptions.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ClientOptions.java index e7388551..4dd049dd 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ClientOptions.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ClientOptions.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.HashMap; import java.util.Map; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/DateTimeDeserializer.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/DateTimeDeserializer.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/core/DateTimeDeserializer.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/DateTimeDeserializer.java index 6be10979..a5a2fbe8 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/DateTimeDeserializer.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/DateTimeDeserializer.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/Environment.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/Environment.java similarity index 92% rename from common/src/main/java/com/skyflow/generated/rest/core/Environment.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/Environment.java index c5a294b7..098667c0 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/Environment.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/Environment.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; public final class Environment { public static final Environment PRODUCTION = new Environment("https://manage.skyflowapis.com"); diff --git a/common/src/main/java/com/skyflow/generated/rest/core/FileStream.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/FileStream.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/core/FileStream.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/FileStream.java index 6b459431..b1aad352 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/FileStream.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/FileStream.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.InputStream; import java.util.Objects; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/InputStreamRequestBody.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/InputStreamRequestBody.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/core/InputStreamRequestBody.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/InputStreamRequestBody.java index 545f6088..22b17b14 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/InputStreamRequestBody.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/InputStreamRequestBody.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.IOException; import java.io.InputStream; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/MediaTypes.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/MediaTypes.java similarity index 84% rename from common/src/main/java/com/skyflow/generated/rest/core/MediaTypes.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/MediaTypes.java index 11714cb8..d4374647 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/MediaTypes.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/MediaTypes.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import okhttp3.MediaType; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/Nullable.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/Nullable.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/core/Nullable.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/Nullable.java index 5929c12d..efabe806 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/Nullable.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/Nullable.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.Optional; import java.util.function.Function; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/NullableNonemptyFilter.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/NullableNonemptyFilter.java similarity index 90% rename from common/src/main/java/com/skyflow/generated/rest/core/NullableNonemptyFilter.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/NullableNonemptyFilter.java index 98c33be4..dd32d66c 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/NullableNonemptyFilter.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/NullableNonemptyFilter.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.Optional; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ObjectMappers.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ObjectMappers.java similarity index 96% rename from common/src/main/java/com/skyflow/generated/rest/core/ObjectMappers.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ObjectMappers.java index 3b7894e0..ee160b93 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ObjectMappers.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ObjectMappers.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/QueryStringMapper.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/QueryStringMapper.java similarity index 99% rename from common/src/main/java/com/skyflow/generated/rest/core/QueryStringMapper.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/QueryStringMapper.java index e9e18fb9..b35c7174 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/QueryStringMapper.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/QueryStringMapper.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/RequestOptions.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/RequestOptions.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/core/RequestOptions.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/RequestOptions.java index edc6d0ae..b70c02df 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/RequestOptions.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/RequestOptions.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.HashMap; import java.util.Map; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyInputStream.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyInputStream.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyInputStream.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyInputStream.java index d8df7715..5bb66b4a 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyInputStream.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyInputStream.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.FilterInputStream; import java.io.IOException; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyReader.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyReader.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyReader.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyReader.java index ed894407..e3052b4c 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/ResponseBodyReader.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/ResponseBodyReader.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.FilterReader; import java.io.IOException; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/RetryInterceptor.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/RetryInterceptor.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/core/RetryInterceptor.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/RetryInterceptor.java index eda7d265..93e86644 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/RetryInterceptor.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/RetryInterceptor.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.IOException; import java.time.Duration; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/Stream.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/Stream.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/core/Stream.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/Stream.java index f037712a..4a984faa 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/Stream.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/Stream.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.io.Reader; import java.util.Iterator; diff --git a/common/src/main/java/com/skyflow/generated/rest/core/Suppliers.java b/common/src/main/java/com/skyflow/generated/auth/rest/core/Suppliers.java similarity index 93% rename from common/src/main/java/com/skyflow/generated/rest/core/Suppliers.java rename to common/src/main/java/com/skyflow/generated/auth/rest/core/Suppliers.java index 307d5852..d3ab5e53 100644 --- a/common/src/main/java/com/skyflow/generated/rest/core/Suppliers.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/core/Suppliers.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.core; +package com.skyflow.generated.auth.rest.core; import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; diff --git a/common/src/main/java/com/skyflow/generated/rest/errors/BadRequestError.java b/common/src/main/java/com/skyflow/generated/auth/rest/errors/BadRequestError.java similarity index 86% rename from common/src/main/java/com/skyflow/generated/rest/errors/BadRequestError.java rename to common/src/main/java/com/skyflow/generated/auth/rest/errors/BadRequestError.java index d4b2d99f..ec08ddf2 100644 --- a/common/src/main/java/com/skyflow/generated/rest/errors/BadRequestError.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/errors/BadRequestError.java @@ -1,9 +1,9 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.errors; +package com.skyflow.generated.auth.rest.errors; -import com.skyflow.generated.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; import java.util.Map; import okhttp3.Response; diff --git a/common/src/main/java/com/skyflow/generated/rest/errors/HttpStatus.java b/common/src/main/java/com/skyflow/generated/auth/rest/errors/HttpStatus.java similarity index 83% rename from common/src/main/java/com/skyflow/generated/rest/errors/HttpStatus.java rename to common/src/main/java/com/skyflow/generated/auth/rest/errors/HttpStatus.java index dc9aecfe..2e1c45d2 100644 --- a/common/src/main/java/com/skyflow/generated/rest/errors/HttpStatus.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/errors/HttpStatus.java @@ -1,4 +1,4 @@ -package com.skyflow.generated.rest.errors; +package com.skyflow.generated.auth.rest.errors; public enum HttpStatus { BAD_REQUEST("Bad Request"); diff --git a/common/src/main/java/com/skyflow/generated/rest/errors/NotFoundError.java b/common/src/main/java/com/skyflow/generated/auth/rest/errors/NotFoundError.java similarity index 86% rename from common/src/main/java/com/skyflow/generated/rest/errors/NotFoundError.java rename to common/src/main/java/com/skyflow/generated/auth/rest/errors/NotFoundError.java index 46291af6..b889e1b0 100644 --- a/common/src/main/java/com/skyflow/generated/rest/errors/NotFoundError.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/errors/NotFoundError.java @@ -1,9 +1,9 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.errors; +package com.skyflow.generated.auth.rest.errors; -import com.skyflow.generated.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; import java.util.Map; import okhttp3.Response; diff --git a/common/src/main/java/com/skyflow/generated/rest/errors/UnauthorizedError.java b/common/src/main/java/com/skyflow/generated/auth/rest/errors/UnauthorizedError.java similarity index 86% rename from common/src/main/java/com/skyflow/generated/rest/errors/UnauthorizedError.java rename to common/src/main/java/com/skyflow/generated/auth/rest/errors/UnauthorizedError.java index 1e90cfc2..40ea9c25 100644 --- a/common/src/main/java/com/skyflow/generated/rest/errors/UnauthorizedError.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/errors/UnauthorizedError.java @@ -1,9 +1,9 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.errors; +package com.skyflow.generated.auth.rest.errors; -import com.skyflow.generated.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; import java.util.Map; import okhttp3.Response; diff --git a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncAuthenticationClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncAuthenticationClient.java similarity index 84% rename from common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncAuthenticationClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncAuthenticationClient.java index 43ffab73..e75fad24 100644 --- a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncAuthenticationClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncAuthenticationClient.java @@ -1,12 +1,13 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.resources.authentication; +package com.skyflow.generated.auth.rest.resources.authentication; + +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.RequestOptions; +import com.skyflow.generated.auth.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.RequestOptions; -import com.skyflow.generated.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.types.V1GetAuthTokenResponse; import java.util.concurrent.CompletableFuture; public class AsyncAuthenticationClient { diff --git a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncRawAuthenticationClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncRawAuthenticationClient.java similarity index 87% rename from common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncRawAuthenticationClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncRawAuthenticationClient.java index 10ad0c01..61ff4335 100644 --- a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AsyncRawAuthenticationClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AsyncRawAuthenticationClient.java @@ -1,22 +1,23 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.resources.authentication; +package com.skyflow.generated.auth.rest.resources.authentication; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.skyflow.generated.rest.core.ApiClientApiException; -import com.skyflow.generated.rest.core.ApiClientException; -import com.skyflow.generated.rest.core.ApiClientHttpResponse; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.MediaTypes; -import com.skyflow.generated.rest.core.ObjectMappers; -import com.skyflow.generated.rest.core.RequestOptions; -import com.skyflow.generated.rest.errors.BadRequestError; -import com.skyflow.generated.rest.errors.NotFoundError; -import com.skyflow.generated.rest.errors.UnauthorizedError; -import com.skyflow.generated.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.errors.BadRequestError; +import com.skyflow.generated.auth.rest.errors.NotFoundError; +import com.skyflow.generated.auth.rest.errors.UnauthorizedError; +import com.skyflow.generated.auth.rest.resources.authentication.requests.V1GetAuthTokenRequest; +import com.skyflow.generated.auth.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.core.ApiClientException; +import com.skyflow.generated.auth.rest.core.ApiClientHttpResponse; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.MediaTypes; +import com.skyflow.generated.auth.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.RequestOptions; + import java.io.IOException; import java.util.Map; import java.util.concurrent.CompletableFuture; diff --git a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AuthenticationClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AuthenticationClient.java similarity index 82% rename from common/src/main/java/com/skyflow/generated/rest/resources/authentication/AuthenticationClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AuthenticationClient.java index 662bfb3d..a0ba860a 100644 --- a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/AuthenticationClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/AuthenticationClient.java @@ -1,12 +1,12 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.resources.authentication; +package com.skyflow.generated.auth.rest.resources.authentication; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.RequestOptions; -import com.skyflow.generated.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.RequestOptions; +import com.skyflow.generated.auth.rest.resources.authentication.requests.V1GetAuthTokenRequest; +import com.skyflow.generated.auth.rest.types.V1GetAuthTokenResponse; public class AuthenticationClient { protected final ClientOptions clientOptions; diff --git a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/RawAuthenticationClient.java b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/RawAuthenticationClient.java similarity index 84% rename from common/src/main/java/com/skyflow/generated/rest/resources/authentication/RawAuthenticationClient.java rename to common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/RawAuthenticationClient.java index c30cf003..f3211ad1 100644 --- a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/RawAuthenticationClient.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/RawAuthenticationClient.java @@ -1,22 +1,23 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.resources.authentication; +package com.skyflow.generated.auth.rest.resources.authentication; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; -import com.skyflow.generated.rest.core.ApiClientApiException; -import com.skyflow.generated.rest.core.ApiClientException; -import com.skyflow.generated.rest.core.ApiClientHttpResponse; -import com.skyflow.generated.rest.core.ClientOptions; -import com.skyflow.generated.rest.core.MediaTypes; -import com.skyflow.generated.rest.core.ObjectMappers; -import com.skyflow.generated.rest.core.RequestOptions; -import com.skyflow.generated.rest.errors.BadRequestError; -import com.skyflow.generated.rest.errors.NotFoundError; -import com.skyflow.generated.rest.errors.UnauthorizedError; -import com.skyflow.generated.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.errors.BadRequestError; +import com.skyflow.generated.auth.rest.errors.NotFoundError; +import com.skyflow.generated.auth.rest.errors.UnauthorizedError; +import com.skyflow.generated.auth.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.core.ApiClientException; +import com.skyflow.generated.auth.rest.core.ApiClientHttpResponse; +import com.skyflow.generated.auth.rest.core.ClientOptions; +import com.skyflow.generated.auth.rest.core.MediaTypes; +import com.skyflow.generated.auth.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.RequestOptions; +import com.skyflow.generated.auth.rest.resources.authentication.requests.V1GetAuthTokenRequest; + import java.io.IOException; import java.util.Map; import okhttp3.Headers; diff --git a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/requests/V1GetAuthTokenRequest.java b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/requests/V1GetAuthTokenRequest.java similarity index 98% rename from common/src/main/java/com/skyflow/generated/rest/resources/authentication/requests/V1GetAuthTokenRequest.java rename to common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/requests/V1GetAuthTokenRequest.java index 8c4961b1..cd1d22c1 100644 --- a/common/src/main/java/com/skyflow/generated/rest/resources/authentication/requests/V1GetAuthTokenRequest.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/resources/authentication/requests/V1GetAuthTokenRequest.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.resources.authentication.requests; +package com.skyflow.generated.auth.rest.resources.authentication.requests; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -11,7 +11,8 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.skyflow.generated.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.ObjectMappers; + import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/common/src/main/java/com/skyflow/generated/rest/types/GooglerpcStatus.java b/common/src/main/java/com/skyflow/generated/auth/rest/types/GooglerpcStatus.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/types/GooglerpcStatus.java rename to common/src/main/java/com/skyflow/generated/auth/rest/types/GooglerpcStatus.java index d0290573..06a9373c 100644 --- a/common/src/main/java/com/skyflow/generated/rest/types/GooglerpcStatus.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/types/GooglerpcStatus.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.types; +package com.skyflow.generated.auth.rest.types; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -11,7 +11,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.skyflow.generated.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.ObjectMappers; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/common/src/main/java/com/skyflow/generated/rest/types/ProtobufAny.java b/common/src/main/java/com/skyflow/generated/auth/rest/types/ProtobufAny.java similarity index 96% rename from common/src/main/java/com/skyflow/generated/rest/types/ProtobufAny.java rename to common/src/main/java/com/skyflow/generated/auth/rest/types/ProtobufAny.java index 37555aae..6f5df772 100644 --- a/common/src/main/java/com/skyflow/generated/rest/types/ProtobufAny.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/types/ProtobufAny.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.types; +package com.skyflow.generated.auth.rest.types; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -11,7 +11,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.skyflow.generated.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.ObjectMappers; import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/common/src/main/java/com/skyflow/generated/rest/types/V1GetAuthTokenResponse.java b/common/src/main/java/com/skyflow/generated/auth/rest/types/V1GetAuthTokenResponse.java similarity index 97% rename from common/src/main/java/com/skyflow/generated/rest/types/V1GetAuthTokenResponse.java rename to common/src/main/java/com/skyflow/generated/auth/rest/types/V1GetAuthTokenResponse.java index 9b6be70b..3c10fbd3 100644 --- a/common/src/main/java/com/skyflow/generated/rest/types/V1GetAuthTokenResponse.java +++ b/common/src/main/java/com/skyflow/generated/auth/rest/types/V1GetAuthTokenResponse.java @@ -1,7 +1,7 @@ /** * This file was auto-generated by Fern from our API Definition. */ -package com.skyflow.generated.rest.types; +package com.skyflow.generated.auth.rest.types; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; @@ -11,7 +11,7 @@ import com.fasterxml.jackson.annotation.JsonSetter; import com.fasterxml.jackson.annotation.Nulls; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.skyflow.generated.rest.core.ObjectMappers; +import com.skyflow.generated.auth.rest.core.ObjectMappers; import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/common/src/main/java/com/skyflow/logs/ErrorLogs.java b/common/src/main/java/com/skyflow/logs/ErrorLogs.java index 3415860c..9b329ac8 100644 --- a/common/src/main/java/com/skyflow/logs/ErrorLogs.java +++ b/common/src/main/java/com/skyflow/logs/ErrorLogs.java @@ -126,6 +126,10 @@ public enum ErrorLogs { OUTPUT_DIRECTORY_NOT_FOUND("Invalid %s1 request. The output directory does not exist. Please specify a valid output directory."), INVALID_PERMISSIONS_FOR_OUTPUT_DIRECTORY("Invalid %s1 request. The output directory is not writable. Please check the permissions or specify a valid output directory."), EMPTY_FILE_AND_FILE_PATH_IN_DEIDENTIFY_FILE("Invalid %s1 request. The file and file path fields are both empty. Specify a valid file object or file path."), + + UNEXPECTED_ERROR_DURING_BATCH_PROCESSING("Unexpected error occurred during batch processing. Error: %s1"), + + PROCESSING_ERROR_RESPONSE("Processing error response.") ; private final String log; diff --git a/common/src/main/java/com/skyflow/logs/InfoLogs.java b/common/src/main/java/com/skyflow/logs/InfoLogs.java index efd81a49..e54df467 100644 --- a/common/src/main/java/com/skyflow/logs/InfoLogs.java +++ b/common/src/main/java/com/skyflow/logs/InfoLogs.java @@ -90,6 +90,8 @@ public enum InfoLogs { GET_DETECT_RUN_TRIGGERED("Get detect run method triggered."), VALIDATE_GET_DETECT_RUN_REQUEST("Validating get detect run request."), REIDENTIFY_TEXT_SUCCESS("Text data re-identified."), + + PROCESSING_BATCHES("Processing batch") ; diff --git a/common/src/main/java/com/skyflow/logs/WarningLogs.java b/common/src/main/java/com/skyflow/logs/WarningLogs.java index 8d49f056..8905ad12 100644 --- a/common/src/main/java/com/skyflow/logs/WarningLogs.java +++ b/common/src/main/java/com/skyflow/logs/WarningLogs.java @@ -1,7 +1,9 @@ package com.skyflow.logs; public enum WarningLogs { - OVERRIDING_EXISTING_VAULT_CONFIG("New vault config identified. Overriding existing vault config"); + INVALID_BATCH_SIZE_PROVIDED("Invalid value for batch size provided, switching to default value."), + INVALID_CONCURRENCY_LIMIT_PROVIDED("Invalid value for concurrency limit provided, switching to default value."), + ; private final String log; diff --git a/common/src/main/java/com/skyflow/serviceaccount/util/BearerToken.java b/common/src/main/java/com/skyflow/serviceaccount/util/BearerToken.java index f2c15a6f..9f0fc0e3 100644 --- a/common/src/main/java/com/skyflow/serviceaccount/util/BearerToken.java +++ b/common/src/main/java/com/skyflow/serviceaccount/util/BearerToken.java @@ -4,12 +4,12 @@ import com.skyflow.errors.ErrorCode; import com.skyflow.errors.ErrorMessage; import com.skyflow.errors.SkyflowException; -import com.skyflow.generated.rest.ApiClient; -import com.skyflow.generated.rest.ApiClientBuilder; -import com.skyflow.generated.rest.core.ApiClientApiException; -import com.skyflow.generated.rest.resources.authentication.AuthenticationClient; -import com.skyflow.generated.rest.resources.authentication.requests.V1GetAuthTokenRequest; -import com.skyflow.generated.rest.types.V1GetAuthTokenResponse; +import com.skyflow.generated.auth.rest.ApiClient; +import com.skyflow.generated.auth.rest.ApiClientBuilder; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; +import com.skyflow.generated.auth.rest.resources.authentication.AuthenticationClient; +import com.skyflow.generated.auth.rest.resources.authentication.requests.V1GetAuthTokenRequest; +import com.skyflow.generated.auth.rest.types.V1GetAuthTokenResponse; import com.skyflow.logs.ErrorLogs; import com.skyflow.logs.InfoLogs; import com.skyflow.utils.BaseConstants; diff --git a/common/src/main/java/com/skyflow/utils/BaseConstants.java b/common/src/main/java/com/skyflow/utils/BaseConstants.java index f38be8e8..0880034f 100644 --- a/common/src/main/java/com/skyflow/utils/BaseConstants.java +++ b/common/src/main/java/com/skyflow/utils/BaseConstants.java @@ -7,10 +7,13 @@ public class BaseConstants { public static final String ORDER_ASCENDING = "ASCENDING"; public static final String ENV_CREDENTIALS_KEY_NAME = "SKYFLOW_CREDENTIALS"; public static final String SECURE_PROTOCOL = "https://"; - public static final String DEV_DOMAIN = ".vault.skyflowapis.dev"; - public static final String STAGE_DOMAIN = ".vault.skyflowapis.tech"; - public static final String SANDBOX_DOMAIN = ".vault.skyflowapis-preview.com"; - public static final String PROD_DOMAIN = ".vault.skyflowapis.com"; + + public static final String V2_VAULT_DOMAIN = ".vault."; + public static final String V3_VAULT_DOMAIN = ".skyvault."; + public static final String DEV_DOMAIN = "skyflowapis.dev"; + public static final String STAGE_DOMAIN = "skyflowapis.tech"; + public static final String SANDBOX_DOMAIN = "skyflowapis-preview.com"; + public static final String PROD_DOMAIN = "skyflowapis.com"; public static final String PKCS8_PRIVATE_HEADER = "-----BEGIN PRIVATE KEY-----"; public static final String PKCS8_PRIVATE_FOOTER = "-----END PRIVATE KEY-----"; public static final String GRANT_TYPE = "urn:ietf:params:oauth:grant-type:jwt-bearer"; diff --git a/common/src/main/java/com/skyflow/utils/BaseUtils.java b/common/src/main/java/com/skyflow/utils/BaseUtils.java index b51aaaf3..c9278517 100644 --- a/common/src/main/java/com/skyflow/utils/BaseUtils.java +++ b/common/src/main/java/com/skyflow/utils/BaseUtils.java @@ -22,9 +22,8 @@ import java.security.spec.PKCS8EncodedKeySpec; public class BaseUtils { - public static String getVaultURL(String clusterId, Env env) { - StringBuilder sb = new StringBuilder(BaseConstants.SECURE_PROTOCOL); - sb.append(clusterId); + public static String getVaultURL(String clusterId, Env env, String vaultDomain) { + StringBuilder sb = buildBaseUrl(clusterId, vaultDomain); switch (env) { case DEV: sb.append(BaseConstants.DEV_DOMAIN); @@ -160,4 +159,10 @@ private static PrivateKey parsePkcs8PrivateKey(byte[] pkcs8Bytes) throws Skyflow return privateKey; } + private static StringBuilder buildBaseUrl(String clusterId, String vaultDomain) { + StringBuilder sb = new StringBuilder(BaseConstants.SECURE_PROTOCOL); + sb.append(clusterId); + sb.append(vaultDomain); + return sb; + } } diff --git a/common/src/main/java/com/skyflow/vault/data/BaseGetRequest.java b/common/src/main/java/com/skyflow/vault/data/BaseGetRequest.java deleted file mode 100644 index c6d828cc..00000000 --- a/common/src/main/java/com/skyflow/vault/data/BaseGetRequest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.skyflow.vault.data; - -import com.skyflow.utils.BaseConstants; - -import java.util.ArrayList; - -class BaseGetRequest { - private final BaseGetRequestBuilder builder; - - protected BaseGetRequest(BaseGetRequestBuilder builder) { - this.builder = builder; - } - - public String getTable() { - return this.builder.table; - } - - public ArrayList getIds() { - return this.builder.ids; - } - - public Boolean getReturnTokens() { - return this.builder.returnTokens; - } - - public ArrayList getFields() { - return this.builder.fields; - } - - public String getOffset() { - return this.builder.offset; - } - - public String getLimit() { - return this.builder.limit; - } - - public Boolean getDownloadURL() { - return this.builder.downloadURL; - } - - public String getColumnName() { - return this.builder.columnName; - } - - public ArrayList getColumnValues() { - return this.builder.columnValues; - } - - public String getOrderBy() { - return this.builder.orderBy; - } - - static class BaseGetRequestBuilder { - protected String table; - protected ArrayList ids; - protected Boolean returnTokens; - protected ArrayList fields; - protected String offset; - protected String limit; - protected Boolean downloadURL; - protected String columnName; - protected ArrayList columnValues; - protected String orderBy; - - protected BaseGetRequestBuilder() { - this.downloadURL = true; - this.orderBy = BaseConstants.ORDER_ASCENDING; - } - - public BaseGetRequestBuilder table(String table) { - this.table = table; - return this; - } - - public BaseGetRequestBuilder ids(ArrayList ids) { - this.ids = ids; - return this; - } - - public BaseGetRequestBuilder returnTokens(Boolean returnTokens) { - this.returnTokens = returnTokens; - return this; - } - - public BaseGetRequestBuilder fields(ArrayList fields) { - this.fields = fields; - return this; - } - - public BaseGetRequestBuilder offset(String offset) { - this.offset = offset; - return this; - } - - public BaseGetRequestBuilder limit(String limit) { - this.limit = limit; - return this; - } - - public BaseGetRequestBuilder downloadURL(Boolean downloadURL) { - this.downloadURL = downloadURL == null || downloadURL; - return this; - } - - public BaseGetRequestBuilder columnName(String columnName) { - this.columnName = columnName; - return this; - } - - public BaseGetRequestBuilder columnValues(ArrayList columnValues) { - this.columnValues = columnValues; - return this; - } - - public BaseGetRequestBuilder orderBy(String orderBy) { - this.orderBy = orderBy == null ? BaseConstants.ORDER_ASCENDING : orderBy; - return this; - } - - } -} diff --git a/common/src/main/java/com/skyflow/vault/data/BaseGetResponse.java b/common/src/main/java/com/skyflow/vault/data/BaseGetResponse.java deleted file mode 100644 index 7a985fe6..00000000 --- a/common/src/main/java/com/skyflow/vault/data/BaseGetResponse.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.skyflow.vault.data; - -import com.google.gson.Gson; - -import java.util.ArrayList; -import java.util.HashMap; - -public class BaseGetResponse { - private final ArrayList> data; - private final ArrayList> errors; - - public BaseGetResponse(ArrayList> data, ArrayList> errors) { - this.data = data; - this.errors = errors; - } - - public ArrayList> getData() { - return data; - } - - public ArrayList> getErrors() { - return errors; - } - - @Override - public String toString() { - Gson gson = new Gson(); - return gson.toJson(this); - } -} diff --git a/common/src/main/java/com/skyflow/vault/data/BaseInsertRequest.java b/common/src/main/java/com/skyflow/vault/data/BaseInsertRequest.java index 64e61153..b856492d 100644 --- a/common/src/main/java/com/skyflow/vault/data/BaseInsertRequest.java +++ b/common/src/main/java/com/skyflow/vault/data/BaseInsertRequest.java @@ -18,27 +18,12 @@ public ArrayList> getValues() { return this.builder.values; } - public ArrayList> getTokens() { - return this.builder.tokens; - } - - public Boolean getReturnTokens() { - return this.builder.returnTokens; - } - - public String getUpsert() { - return this.builder.upsert; - } - static class BaseInsertRequestBuilder { protected String table; protected ArrayList> values; - protected ArrayList> tokens; - protected Boolean returnTokens; protected String upsert; protected BaseInsertRequestBuilder() { - this.returnTokens = false; } public BaseInsertRequestBuilder table(String table) { @@ -51,20 +36,6 @@ public BaseInsertRequestBuilder values(ArrayList> values return this; } - public BaseInsertRequestBuilder tokens(ArrayList> tokens) { - this.tokens = tokens; - return this; - } - - public BaseInsertRequestBuilder returnTokens(Boolean returnTokens) { - this.returnTokens = returnTokens != null && returnTokens; - return this; - } - - public BaseInsertRequestBuilder upsert(String upsert) { - this.upsert = upsert; - return this; - } } } diff --git a/common/src/main/java/com/skyflow/vault/data/BaseInsertResponse.java b/common/src/main/java/com/skyflow/vault/data/BaseInsertResponse.java deleted file mode 100644 index b3eead69..00000000 --- a/common/src/main/java/com/skyflow/vault/data/BaseInsertResponse.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.skyflow.vault.data; - -import com.google.gson.Gson; - -import java.util.ArrayList; -import java.util.HashMap; - -public class BaseInsertResponse { - private final ArrayList> insertedFields; - private final ArrayList> errors; - - public BaseInsertResponse(ArrayList> insertedFields, ArrayList> errors) { - this.insertedFields = insertedFields; - this.errors = errors; - } - - public ArrayList> getInsertedFields() { - return insertedFields; - } - - public ArrayList> getErrors() { - return errors; - } - - @Override - public String toString() { - Gson gson = new Gson(); - return gson.toJson(this); - } -} diff --git a/common/src/test/java/com/skyflow/config/CredentialsTests.java b/common/test/java/com/skyflow/config/CredentialsTests.java similarity index 100% rename from common/src/test/java/com/skyflow/config/CredentialsTests.java rename to common/test/java/com/skyflow/config/CredentialsTests.java diff --git a/common/src/test/java/com/skyflow/config/VaultConfigTests.java b/common/test/java/com/skyflow/config/VaultConfigTests.java similarity index 100% rename from common/src/test/java/com/skyflow/config/VaultConfigTests.java rename to common/test/java/com/skyflow/config/VaultConfigTests.java diff --git a/common/src/test/java/com/skyflow/errors/SkyflowExceptionTest.java b/common/test/java/com/skyflow/errors/SkyflowExceptionTest.java similarity index 100% rename from common/src/test/java/com/skyflow/errors/SkyflowExceptionTest.java rename to common/test/java/com/skyflow/errors/SkyflowExceptionTest.java diff --git a/common/src/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java b/common/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java similarity index 94% rename from common/src/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java rename to common/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java index 3e8a7696..cf5bbcd5 100644 --- a/common/src/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java +++ b/common/test/java/com/skyflow/serviceaccount/util/BearerTokenTests.java @@ -25,7 +25,7 @@ public class BearerTokenTests { @BeforeClass public static void setup() { credentialsFilePath = "./credentials.json"; - invalidJsonFilePath = "./src/test/resources/notJson.txt"; + invalidJsonFilePath = "./test/resources/notJson.txt"; invalidFilePath = "./src/test/credentials.json"; credentialsString = "{\"key\":\"value\"}"; context = "test_context"; @@ -128,7 +128,7 @@ public void testInvalidCredentialsString() { @Test public void testNoPrivateKeyInCredentialsForCredentials() { - String filePath = "./src/test/resources/noPrivateKeyCredentials.json"; + String filePath = "./test/resources/noPrivateKeyCredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); @@ -142,7 +142,7 @@ public void testNoPrivateKeyInCredentialsForCredentials() { @Test public void testNoClientIDInCredentialsForCredentials() { - String filePath = "./src/test/resources/noClientIDCredentials.json"; + String filePath = "./test/resources/noClientIDCredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); @@ -156,7 +156,7 @@ public void testNoClientIDInCredentialsForCredentials() { @Test public void testNoKeyIDInCredentialsForCredentials() { - String filePath = "./src/test/resources/noKeyIDCredentials.json"; + String filePath = "./test/resources/noKeyIDCredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); @@ -170,7 +170,7 @@ public void testNoKeyIDInCredentialsForCredentials() { @Test public void testNoTokenURIInCredentialsForCredentials() { - String filePath = "./src/test/resources/noTokenURICredentials.json"; + String filePath = "./test/resources/noTokenURICredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); @@ -184,7 +184,7 @@ public void testNoTokenURIInCredentialsForCredentials() { @Test public void testInvalidPrivateKeyInCredentialsForCredentials() { - String filePath = "./src/test/resources/invalidPrivateKeyCredentials.json"; + String filePath = "./test/resources/invalidPrivateKeyCredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); @@ -211,7 +211,7 @@ public void testInvalidKeySpecInCredentialsForCredentials() { @Test public void testInvalidTokenURIInCredentialsForCredentials() throws SkyflowException { - String filePath = "./src/test/resources/invalidTokenURICredentials.json"; + String filePath = "./test/resources/invalidTokenURICredentials.json"; File file = new File(filePath); try { BearerToken bearerToken = BearerToken.builder().setCredentials(file).build(); diff --git a/common/src/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java b/common/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java similarity index 95% rename from common/src/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java rename to common/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java index bd46cc9f..dcda716e 100644 --- a/common/src/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java +++ b/common/test/java/com/skyflow/serviceaccount/util/SignedDataTokensTests.java @@ -26,7 +26,7 @@ public class SignedDataTokensTests { @BeforeClass public static void setup() { credentialsFilePath = "./credentials.json"; - invalidJsonFilePath = "./src/test/resources/notJson.txt"; + invalidJsonFilePath = "./test/resources/notJson.txt"; invalidFilePath = "./src/test/credentials.json"; credentialsString = "{\"key\":\"value\"}"; context = "test_context"; @@ -138,7 +138,7 @@ public void testInvalidCredentialsString() { @Test public void testNoPrivateKeyInCredentials() { - String filePath = "./src/test/resources/noPrivateKeyCredentials.json"; + String filePath = "./test/resources/noPrivateKeyCredentials.json"; File file = new File(filePath); try { SignedDataTokens signedTokens = SignedDataTokens.builder().setCredentials(file).build(); @@ -152,7 +152,7 @@ public void testNoPrivateKeyInCredentials() { @Test public void testNoClientIDInCredentials() { - String filePath = "./src/test/resources/noClientIDCredentials.json"; + String filePath = "./test/resources/noClientIDCredentials.json"; File file = new File(filePath); try { SignedDataTokens signedTokens = SignedDataTokens.builder().setCredentials(file).build(); @@ -166,7 +166,7 @@ public void testNoClientIDInCredentials() { @Test public void testNoKeyIDInCredentials() { - String filePath = "./src/test/resources/noKeyIDCredentials.json"; + String filePath = "./test/resources/noKeyIDCredentials.json"; File file = new File(filePath); try { SignedDataTokens signedTokens = SignedDataTokens.builder().setCredentials(file).build(); @@ -180,7 +180,7 @@ public void testNoKeyIDInCredentials() { @Test public void testInvalidPrivateKeyInCredentials() { - String filePath = "./src/test/resources/invalidPrivateKeyCredentials.json"; + String filePath = "./test/resources/invalidPrivateKeyCredentials.json"; File file = new File(filePath); try { SignedDataTokens signedTokens = SignedDataTokens.builder().setCredentials(file).build(); diff --git a/common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java b/common/test/java/com/skyflow/serviceaccount/util/TokenTests.java similarity index 100% rename from common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java rename to common/test/java/com/skyflow/serviceaccount/util/TokenTests.java diff --git a/common/src/test/java/com/skyflow/utils/BaseUtilsTests.java b/common/test/java/com/skyflow/utils/BaseUtilsTests.java similarity index 100% rename from common/src/test/java/com/skyflow/utils/BaseUtilsTests.java rename to common/test/java/com/skyflow/utils/BaseUtilsTests.java diff --git a/common/src/test/resources/invalidPrivateKeyCredentials.json b/common/test/resources/invalidPrivateKeyCredentials.json similarity index 100% rename from common/src/test/resources/invalidPrivateKeyCredentials.json rename to common/test/resources/invalidPrivateKeyCredentials.json diff --git a/common/src/test/resources/invalidTokenURICredentials.json b/common/test/resources/invalidTokenURICredentials.json similarity index 100% rename from common/src/test/resources/invalidTokenURICredentials.json rename to common/test/resources/invalidTokenURICredentials.json diff --git a/common/src/test/resources/noClientIDCredentials.json b/common/test/resources/noClientIDCredentials.json similarity index 100% rename from common/src/test/resources/noClientIDCredentials.json rename to common/test/resources/noClientIDCredentials.json diff --git a/common/src/test/resources/noKeyIDCredentials.json b/common/test/resources/noKeyIDCredentials.json similarity index 100% rename from common/src/test/resources/noKeyIDCredentials.json rename to common/test/resources/noKeyIDCredentials.json diff --git a/common/src/test/resources/noPrivateKeyCredentials.json b/common/test/resources/noPrivateKeyCredentials.json similarity index 100% rename from common/src/test/resources/noPrivateKeyCredentials.json rename to common/test/resources/noPrivateKeyCredentials.json diff --git a/common/src/test/resources/noTokenURICredentials.json b/common/test/resources/noTokenURICredentials.json similarity index 100% rename from common/src/test/resources/noTokenURICredentials.json rename to common/test/resources/noTokenURICredentials.json diff --git a/common/src/test/resources/notJson.txt b/common/test/resources/notJson.txt similarity index 100% rename from common/src/test/resources/notJson.txt rename to common/test/resources/notJson.txt diff --git a/pom.xml b/pom.xml index f36b6778..372a9525 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.skyflow - skyflow-java + skyflow 1.0.0 pom @@ -37,6 +37,7 @@ + true 8 8 4.12.0 @@ -169,7 +170,8 @@ 3.2.0 - com.skyflow.generated.rest.* + com.skyflow.generated.rest.*: + com.skyflow.generated.auth.*: @@ -178,6 +180,7 @@ jar + @@ -256,36 +259,7 @@ https://repo.maven.apache.org/maven2/ - - - maven-central - - - central - https://central.sonatype.com/api/v1/publisher/upload - - - central-snapshots - https://central.sonatype.com/api/v1/publisher/upload - - - - - - org.sonatype.central - central-publishing-maven-plugin - 0.4.0 - true - - central - true - true - - - - - jfrog @@ -302,4 +276,4 @@ - + \ No newline at end of file diff --git a/scripts/bump_version.sh b/scripts/bump_version.sh index d729ea63..c82f6bbf 100755 --- a/scripts/bump_version.sh +++ b/scripts/bump_version.sh @@ -1,7 +1,7 @@ # Input Arguments Version=$1 CommitHash=$2 -PomFile="$GITHUB_WORKSPACE/pom.xml" +PomFile="$GITHUB_WORKSPACE/v3/pom.xml" if [ -z "$Version" ]; then echo "Error: Version argument is required." @@ -26,15 +26,16 @@ if [ -z "$CommitHash" ]; then else echo "Bumping main project version to $Version-dev-$CommitHash" - awk -v version="$Version-dev.$CommitHash" ' - BEGIN { updated = 0 } - // && updated == 0 { - sub(/.*<\/version>/, "" version "") - updated = 1 - } - { print } - ' "$PomFile" > tempfile && cat tempfile > "$PomFile" && rm -f tempfile + awk -v version="$Version${CommitHash:+-dev.$CommitHash}" ' + BEGIN { updated = 0 } + //,/<\/parent>/ { print; next } + // && updated == 0 { + sub(/.*<\/version>/, "" version "") + updated = 1 + } + { print } + ' "$PomFile" > tempfile && cat tempfile > "$PomFile" && rm -f tempfile - echo "--------------------------" - echo "Done. Main project version now at $Version-dev.$CommitHash" +echo "--------------------------" +echo "Done. v3 module version now at $Version${CommitHash:+-dev.$CommitHash}" fi diff --git a/v2/pom.xml b/v2/pom.xml index 943b7c80..e239bc85 100644 --- a/v2/pom.xml +++ b/v2/pom.xml @@ -5,14 +5,14 @@ 4.0.0 com.skyflow - skyflow-java + skyflow 1.0.0 ../pom.xml skyflow-java - 2.0.0-beta.2 - jar + 2.0.0-beta.3 + com.skyflow 8 @@ -26,30 +26,6 @@ common 1.0.0 - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.6.0 - - - package - - shade - - - - - com.skyflow:common - - - - - - - - + \ No newline at end of file diff --git a/v2/src/main/java/com/skyflow/utils/Constants.java b/v2/src/main/java/com/skyflow/utils/Constants.java index c0f2a778..9b850bf0 100644 --- a/v2/src/main/java/com/skyflow/utils/Constants.java +++ b/v2/src/main/java/com/skyflow/utils/Constants.java @@ -4,6 +4,7 @@ public final class Constants extends BaseConstants { public static final String SDK_NAME = "Skyflow Java SDK "; public static final String SDK_VERSION = "2.0.0-beta.2"; public static final String SDK_PREFIX = SDK_NAME + SDK_VERSION; + public static final String VAULT_DOMAIN = ".vault."; public static final String PROCESSED_FILE_NAME_PREFIX = "processed-"; public static final String DEIDENTIFIED_FILE_PREFIX = "deidentified"; } diff --git a/v2/src/main/java/com/skyflow/utils/Utils.java b/v2/src/main/java/com/skyflow/utils/Utils.java index c1b34102..ff43c089 100644 --- a/v2/src/main/java/com/skyflow/utils/Utils.java +++ b/v2/src/main/java/com/skyflow/utils/Utils.java @@ -2,6 +2,7 @@ import com.google.gson.JsonObject; import com.skyflow.config.ConnectionConfig; +import com.skyflow.enums.Env; import com.skyflow.vault.connection.InvokeConnectionRequest; import java.util.HashMap; @@ -9,6 +10,10 @@ public final class Utils extends BaseUtils { + public static String getVaultURL(String clusterId, Env env) { + return getVaultURL(clusterId, env, Constants.VAULT_DOMAIN); + } + public static String constructConnectionURL(ConnectionConfig config, InvokeConnectionRequest invokeConnectionRequest) { StringBuilder filledURL = new StringBuilder(config.getConnectionUrl()); diff --git a/v2/src/main/java/com/skyflow/vault/data/GetRequest.java b/v2/src/main/java/com/skyflow/vault/data/GetRequest.java index ef4edea2..5fa59852 100644 --- a/v2/src/main/java/com/skyflow/vault/data/GetRequest.java +++ b/v2/src/main/java/com/skyflow/vault/data/GetRequest.java @@ -1,15 +1,15 @@ package com.skyflow.vault.data; import com.skyflow.enums.RedactionType; +import com.skyflow.utils.BaseConstants; import com.skyflow.utils.Constants; import java.util.ArrayList; -public class GetRequest extends BaseGetRequest { +public class GetRequest { private final GetRequestBuilder builder; private GetRequest(GetRequestBuilder builder) { - super(builder); this.builder = builder; } @@ -17,28 +17,75 @@ public static GetRequestBuilder builder() { return new GetRequestBuilder(); } + public String getTable() { + return this.builder.table; + } + + public ArrayList getIds() { + return this.builder.ids; + } + + public Boolean getReturnTokens() { + return this.builder.returnTokens; + } + + public ArrayList getFields() { + return this.builder.fields; + } + + public String getOffset() { + return this.builder.offset; + } + + public String getLimit() { + return this.builder.limit; + } + + public Boolean getDownloadURL() { + return this.builder.downloadURL; + } + + public String getColumnName() { + return this.builder.columnName; + } + + public ArrayList getColumnValues() { + return this.builder.columnValues; + } + + public String getOrderBy() { + return this.builder.orderBy; + } + public RedactionType getRedactionType() { return this.builder.redactionType; } - public static final class GetRequestBuilder extends BaseGetRequestBuilder { + public static final class GetRequestBuilder { + private String table; + private ArrayList ids; + private Boolean returnTokens; + private ArrayList fields; + private String offset; + private String limit; + private Boolean downloadURL; + private String columnName; + private ArrayList columnValues; + private String orderBy; private RedactionType redactionType; private GetRequestBuilder() { - super(); this.downloadURL = true; this.orderBy = Constants.ORDER_ASCENDING; } - @Override public GetRequestBuilder table(String table) { - super.table(table); + this.table = table; return this; } - @Override public GetRequestBuilder ids(ArrayList ids) { - super.ids(ids); + this.ids = ids; return this; } @@ -47,51 +94,43 @@ public GetRequestBuilder redactionType(RedactionType redactionType) { return this; } - @Override public GetRequestBuilder returnTokens(Boolean returnTokens) { - super.returnTokens(returnTokens); + this.returnTokens = returnTokens; return this; } - @Override public GetRequestBuilder fields(ArrayList fields) { - super.fields(fields); + this.fields = fields; return this; } - @Override public GetRequestBuilder offset(String offset) { - super.offset(offset); + this.offset = offset; return this; } - @Override public GetRequestBuilder limit(String limit) { - super.limit(limit); + this.limit = limit; return this; } - @Override public GetRequestBuilder downloadURL(Boolean downloadURL) { - super.downloadURL(downloadURL); + this.downloadURL = downloadURL == null || downloadURL; return this; } - @Override public GetRequestBuilder columnName(String columnName) { - super.columnName(columnName); + this.columnName = columnName; return this; } - @Override public GetRequestBuilder columnValues(ArrayList columnValues) { - super.columnValues(columnValues); + this.columnValues = columnValues; return this; } - @Override public GetRequestBuilder orderBy(String orderBy) { - super.orderBy(orderBy); + this.orderBy = orderBy == null ? BaseConstants.ORDER_ASCENDING : orderBy; return this; } diff --git a/v2/src/main/java/com/skyflow/vault/data/InsertRequest.java b/v2/src/main/java/com/skyflow/vault/data/InsertRequest.java index 01655f02..9758e98f 100644 --- a/v2/src/main/java/com/skyflow/vault/data/InsertRequest.java +++ b/v2/src/main/java/com/skyflow/vault/data/InsertRequest.java @@ -28,16 +28,30 @@ public Boolean getContinueOnError() { public TokenMode getTokenMode() { return this.builder.tokenMode; } + public String getUpsert() { + return this.builder.upsert; + } + public Boolean getReturnTokens() { + return this.builder.returnTokens; + } + + public ArrayList> getTokens() { + return this.builder.tokens; + } public static final class InsertRequestBuilder extends BaseInsertRequestBuilder { private Boolean homogeneous; private Boolean continueOnError; private TokenMode tokenMode; + private ArrayList> tokens; + private Boolean returnTokens; + private InsertRequestBuilder() { super(); this.continueOnError = false; this.tokenMode = TokenMode.DISABLE; + this.returnTokens = false; } @Override @@ -52,21 +66,18 @@ public InsertRequestBuilder values(ArrayList> values) { return this; } - @Override public InsertRequestBuilder tokens(ArrayList> tokens) { - super.tokens(tokens); + this.tokens = tokens ; return this; } - @Override public InsertRequestBuilder returnTokens(Boolean returnTokens) { - super.returnTokens(returnTokens); + this.returnTokens = returnTokens != null && returnTokens; return this; } - @Override public InsertRequestBuilder upsert(String upsert) { - super.upsert(upsert); + this.upsert = upsert; return this; } diff --git a/v3/pom.xml b/v3/pom.xml index a2ba27a5..a3f72b89 100644 --- a/v3/pom.xml +++ b/v3/pom.xml @@ -5,21 +5,24 @@ 4.0.0 com.skyflow - skyflow-java + skyflow 1.0.0 ../pom.xml skyflow-java - 3.0.0-beta.0 + 2.0.0-beta.2-dev.a7b8e55 jar 8 8 UTF-8 + false + + com.skyflow @@ -29,27 +32,28 @@ - - - org.apache.maven.plugins - maven-shade-plugin - 3.6.0 - - - package - - shade - - - - - com.skyflow:common - - - - - - - - - \ No newline at end of file + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + + + com.skyflow:common + + + + + + + + + + diff --git a/v3/src/main/java/com/skyflow/Skyflow.java b/v3/src/main/java/com/skyflow/Skyflow.java index 09b16a2c..7973c747 100644 --- a/v3/src/main/java/com/skyflow/Skyflow.java +++ b/v3/src/main/java/com/skyflow/Skyflow.java @@ -43,6 +43,7 @@ public VaultController vault() throws SkyflowException { LogUtil.printErrorLog(ErrorLogs.VAULT_CONFIG_DOES_NOT_EXIST.getLog()); throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.VaultIdNotInConfigList.getMessage()); } + return controller; } diff --git a/v3/src/main/java/com/skyflow/VaultClient.java b/v3/src/main/java/com/skyflow/VaultClient.java index cbeedb94..8e1147a6 100644 --- a/v3/src/main/java/com/skyflow/VaultClient.java +++ b/v3/src/main/java/com/skyflow/VaultClient.java @@ -2,12 +2,17 @@ import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; +import com.skyflow.enums.UpdateType; import com.skyflow.errors.ErrorCode; import com.skyflow.errors.ErrorMessage; import com.skyflow.errors.SkyflowException; import com.skyflow.generated.rest.ApiClient; import com.skyflow.generated.rest.ApiClientBuilder; import com.skyflow.generated.rest.resources.recordservice.RecordserviceClient; +import com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest; +import com.skyflow.generated.rest.types.EnumUpdateType; +import com.skyflow.generated.rest.types.InsertRecordData; +import com.skyflow.generated.rest.types.Upsert; import com.skyflow.logs.InfoLogs; import com.skyflow.serviceaccount.util.Token; import com.skyflow.utils.Constants; @@ -16,6 +21,12 @@ import com.skyflow.utils.validations.Validations; import io.github.cdimascio.dotenv.Dotenv; import io.github.cdimascio.dotenv.DotenvException; +import okhttp3.OkHttpClient; +import okhttp3.Request; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; public class VaultClient { @@ -61,7 +72,10 @@ protected void setBearerToken() throws SkyflowException { } else { LogUtil.printInfoLog(InfoLogs.REUSE_BEARER_TOKEN.getLog()); } + updateExecutorInHTTP(); // update executor + // token need to add in http client this.apiClient = this.apiClientBuilder.build(); + } private void updateVaultURL() { @@ -98,4 +112,47 @@ private void prioritiseCredentials() throws SkyflowException { throw new RuntimeException(e); } } + + protected void updateExecutorInHTTP() { + OkHttpClient httpClient = new OkHttpClient.Builder() + .addInterceptor(chain -> { + Request original = chain.request(); + Request requestWithAuth = original.newBuilder() + .header("Authorization", "Bearer " + this.token) + .build(); + return chain.proceed(requestWithAuth); + }) + .build(); + apiClientBuilder.httpClient(httpClient); + } + protected InsertRequest getBulkInsertRequestBody(com.skyflow.vault.data.InsertRequest request, VaultConfig config) throws SkyflowException { + List> values = request.getValues(); + List insertRecordDataList = new ArrayList<>(); + for (HashMap value : values) { + InsertRecordData data = InsertRecordData.builder().data(value).build(); + insertRecordDataList.add(data); + } + InsertRequest.Builder builder = InsertRequest.builder() + .vaultId(config.getVaultId()) + .records(insertRecordDataList) + .tableName(request.getTable()); + if(request.getUpsert() != null && !request.getUpsert().isEmpty()){ + if (request.getUpsertType() != null) { + EnumUpdateType updateType = null; + if(request.getUpsertType() == UpdateType.REPLACE){ + updateType = EnumUpdateType.REPLACE; + } else if (request.getUpsertType() == UpdateType.REPLACE) { + updateType = EnumUpdateType.UPDATE; + } + Upsert upsert = Upsert.builder().uniqueColumns(request.getUpsert()).updateType(updateType).build(); + builder.upsert(upsert); + } else { + Upsert upsert = Upsert.builder().uniqueColumns(request.getUpsert()).build(); + builder.upsert(upsert); + } + } + return builder.build(); + + } + } diff --git a/v3/src/main/java/com/skyflow/enums/UpdateType.java b/v3/src/main/java/com/skyflow/enums/UpdateType.java new file mode 100644 index 00000000..7fcb6f65 --- /dev/null +++ b/v3/src/main/java/com/skyflow/enums/UpdateType.java @@ -0,0 +1,20 @@ +package com.skyflow.enums; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum UpdateType { + UPDATE("UPDATE"), + + REPLACE("REPLACE"); + + private final String value; + + UpdateType(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } +} diff --git a/v3/src/main/java/com/skyflow/utils/Constants.java b/v3/src/main/java/com/skyflow/utils/Constants.java index a8fd82ab..f295ab52 100644 --- a/v3/src/main/java/com/skyflow/utils/Constants.java +++ b/v3/src/main/java/com/skyflow/utils/Constants.java @@ -1,7 +1,14 @@ package com.skyflow.utils; -public class Constants extends BaseConstants { +public final class Constants extends BaseConstants { public static final String SDK_NAME = "Skyflow Java SDK "; public static final String SDK_VERSION = "3.0.0-beta.0"; + public static final String VAULT_DOMAIN = ".skyvault."; public static final String SDK_PREFIX = SDK_NAME + SDK_VERSION; + public static final Integer INSERT_BATCH_SIZE = 50; + public static final Integer MAX_INSERT_BATCH_SIZE = 1000; + public static final Integer INSERT_CONCURRENCY_LIMIT = 10; + public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10; + public static final Integer DETOKENIZE_BATCH_SIZE = 100; + public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 10; } diff --git a/v3/src/main/java/com/skyflow/utils/Utils.java b/v3/src/main/java/com/skyflow/utils/Utils.java index 915c7e55..2be9cb78 100644 --- a/v3/src/main/java/com/skyflow/utils/Utils.java +++ b/v3/src/main/java/com/skyflow/utils/Utils.java @@ -1,13 +1,139 @@ package com.skyflow.utils; import com.google.gson.JsonObject; +import com.skyflow.enums.Env; +import com.skyflow.generated.rest.core.ApiClientApiException; +import com.skyflow.generated.rest.types.InsertRecordData; +import com.skyflow.generated.rest.types.InsertResponse; +import com.skyflow.generated.rest.types.RecordResponseObject; +import com.skyflow.vault.data.ErrorRecord; +import com.skyflow.vault.data.Success; +import com.skyflow.vault.data.Token; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public final class Utils extends BaseUtils { + public static String getVaultURL(String clusterId, Env env) { + return getVaultURL(clusterId, env, Constants.VAULT_DOMAIN); + } + public static JsonObject getMetrics() { JsonObject details = getCommonMetrics(); String sdkVersion = Constants.SDK_VERSION; details.addProperty(Constants.SDK_METRIC_NAME_VERSION, Constants.SDK_METRIC_NAME_VERSION_PREFIX + sdkVersion); return details; } + + public static List> createBatches(List records, int batchSize) { + List> batches = new ArrayList<>(); + for (int i = 0; i < records.size(); i += batchSize) { + batches.add(records.subList(i, Math.min(i + batchSize, records.size()))); + } + return batches; + } + + public static ErrorRecord createErrorRecord(Map recordMap, int indexNumber) { + ErrorRecord err = null; + if (recordMap != null) { + int code = recordMap.containsKey("http_code") ? (Integer) recordMap.get("http_code") : 500; + String message = recordMap.containsKey("error") ? (String) recordMap.get("error") : + recordMap.containsKey("message") ? (String) recordMap.get("message") : "Unknown error"; + err = new ErrorRecord(indexNumber, message, code); + } + return err; + } + + public static List handleBatchException( + Throwable ex, List batch, int batchNumber, List> batches + ) { + List errorRecords = new ArrayList<>(); + Throwable cause = ex.getCause(); + if (cause instanceof ApiClientApiException) { + ApiClientApiException apiException = (ApiClientApiException) cause; + Map responseBody = (Map) apiException.body(); + int indexNumber = batchNumber > 0 ? batchNumber * batch.size() : 0; + if (responseBody != null) { + if (responseBody.containsKey("records")) { + Object recordss = responseBody.get("records"); + if (recordss instanceof List) { + List recordsList = (List) recordss; + for (Object record : recordsList) { + if (record instanceof Map) { + Map recordMap = (Map) record; + ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber); + errorRecords.add(err); + indexNumber++; + } + } + } + } else if (responseBody.containsKey("error")) { + Map recordMap = (Map) responseBody.get("error"); + for (int j = 0; j < batch.size(); j++) { + ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber); + errorRecords.add(err); + indexNumber++; + } + } + } + } else { + int indexNumber = batchNumber > 0 ? batchNumber * batch.size() : 0; + for (int j = 0; j < batch.size(); j++) { + ErrorRecord err = new ErrorRecord(indexNumber, ex.getMessage(), 500); + errorRecords.add(err); + indexNumber++; + } + } + return errorRecords; + } + + public static com.skyflow.vault.data.InsertResponse formatResponse(InsertResponse response, int batch, int batchSize) { + com.skyflow.vault.data.InsertResponse formattedResponse = null; + List successRecords = new ArrayList<>(); + List errorRecords = new ArrayList<>(); + if (response != null) { + List record = response.getRecords().get(); + int indexNumber = batch * batchSize; + int recordsSize = response.getRecords().get().size(); + for (int index = 0; index < recordsSize; index++) { + if (record.get(index).getError().isPresent()) { + ErrorRecord errorRecord = new ErrorRecord(indexNumber, record.get(index).getError().get(), record.get(index).getHttpCode().get()); + errorRecords.add(errorRecord); + } else { + Map> tokensMap = null; + + if (record.get(index).getTokens().isPresent()) { + tokensMap = new HashMap<>(); + Map tok = record.get(index).getTokens().get(); + for (Map.Entry entry : tok.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + List tokenList = new ArrayList<>(); + if (value instanceof List) { + List valueList = (List) value; + for (Object item : valueList) { + if(item instanceof Map) { + Map tokenMap = (Map) item; + Token token = new Token((String) tokenMap.get("token"), (String) tokenMap.get("tokenGroupName")); + tokenList.add(token); + } + } + } + tokensMap.put(key, tokenList); + } + } + Success success = new Success(indexNumber, record.get(index).getSkyflowId().get(), tokensMap, record.get(index).getData().isPresent() ? record.get(index).getData().get() : null); + successRecords.add(success); + } + indexNumber++; + } + + formattedResponse = new com.skyflow.vault.data.InsertResponse(successRecords, errorRecords); + } + return formattedResponse; + } + } diff --git a/v3/src/main/java/com/skyflow/utils/validations/Validations.java b/v3/src/main/java/com/skyflow/utils/validations/Validations.java index 374e84d4..247cc701 100644 --- a/v3/src/main/java/com/skyflow/utils/validations/Validations.java +++ b/v3/src/main/java/com/skyflow/utils/validations/Validations.java @@ -1,9 +1,75 @@ package com.skyflow.utils.validations; +import com.skyflow.enums.InterfaceName; +import com.skyflow.errors.ErrorCode; +import com.skyflow.errors.ErrorMessage; +import com.skyflow.errors.SkyflowException; +import com.skyflow.logs.ErrorLogs; +import com.skyflow.utils.Utils; +import com.skyflow.utils.logger.LogUtil; +import com.skyflow.vault.data.InsertRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + public class Validations extends BaseValidations { private Validations() { super(); } // add validations specific to v3 SDK + public static void validateInsertRequest(InsertRequest insertRequest) throws SkyflowException { + String table = insertRequest.getTable(); + ArrayList> values = insertRequest.getValues(); + List upsert = insertRequest.getUpsert(); + + if (table == null) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.TABLE_IS_REQUIRED.getLog(), InterfaceName.INSERT.getName() + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.TableKeyError.getMessage()); + } else if (table.trim().isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.EMPTY_TABLE_NAME.getLog(), InterfaceName.INSERT.getName() + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyTable.getMessage()); + } else if (values == null) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.VALUES_IS_REQUIRED.getLog(), InterfaceName.INSERT.getName() + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.ValuesKeyError.getMessage()); + } else if (values.isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.EMPTY_VALUES.getLog(), InterfaceName.INSERT.getName() + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyValues.getMessage()); + } else if (upsert != null && upsert.isEmpty()){ + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.EMPTY_UPSERT.getLog(), InterfaceName.INSERT.getName() + )); + } + // upsert + + for (HashMap valuesMap : values) { + for (String key : valuesMap.keySet()) { + if (key == null || key.trim().isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.EMPTY_OR_NULL_KEY_IN_VALUES.getLog(), InterfaceName.INSERT.getName() + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyKeyInValues.getMessage()); + } else { + Object value = valuesMap.get(key); + if (value == null || value.toString().trim().isEmpty()) { + LogUtil.printErrorLog(Utils.parameterizedString( + ErrorLogs.EMPTY_OR_NULL_VALUE_IN_VALUES.getLog(), + InterfaceName.INSERT.getName(), key + )); + throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyValueInValues.getMessage()); + } + } + } + } + } + } diff --git a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java index b5181f07..a1843ff6 100644 --- a/v3/src/main/java/com/skyflow/vault/controller/VaultController.java +++ b/v3/src/main/java/com/skyflow/vault/controller/VaultController.java @@ -5,30 +5,222 @@ import com.skyflow.VaultClient; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; +import com.skyflow.errors.SkyflowException; +import com.skyflow.generated.rest.core.ApiClientApiException; +import com.skyflow.generated.rest.types.InsertRecordData; import com.skyflow.generated.rest.types.InsertResponse; +import com.skyflow.logs.ErrorLogs; import com.skyflow.logs.InfoLogs; +import com.skyflow.logs.WarningLogs; +import com.skyflow.utils.Constants; +import com.skyflow.utils.Utils; import com.skyflow.utils.logger.LogUtil; -import com.skyflow.vault.RequestOptions; -import com.skyflow.vault.data.BaseInsertRequest; +import com.skyflow.utils.validations.Validations; +import com.skyflow.vault.data.ErrorRecord; +import com.skyflow.vault.data.InsertRequest; +import com.skyflow.vault.data.Success; +import io.github.cdimascio.dotenv.Dotenv; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.skyflow.utils.Utils.formatResponse; +import static com.skyflow.utils.Utils.handleBatchException; public final class VaultController extends VaultClient { private static final Gson gson = new GsonBuilder().serializeNulls().create(); + private int insertBatchSize; + private int insertConcurrencyLimit; + private int detokenizeBatchSize; + private int detokenizeConcurrencyLimit; public VaultController(VaultConfig vaultConfig, Credentials credentials) { super(vaultConfig, credentials); + this.insertBatchSize = Constants.INSERT_BATCH_SIZE; + this.insertConcurrencyLimit = Constants.INSERT_CONCURRENCY_LIMIT; + this.detokenizeBatchSize = Constants.DETOKENIZE_BATCH_SIZE; + this.detokenizeConcurrencyLimit = Constants.DETOKENIZE_CONCURRENCY_LIMIT; + } + + public com.skyflow.vault.data.InsertResponse bulkInsert(InsertRequest insertRequest) throws SkyflowException { + com.skyflow.vault.data.InsertResponse response; + LogUtil.printInfoLog(InfoLogs.INSERT_TRIGGERED.getLog()); + try { + LogUtil.printInfoLog(InfoLogs.VALIDATE_INSERT_REQUEST.getLog()); + Validations.validateInsertRequest(insertRequest); + setBearerToken(); + configureInsertConcurrencyAndBatchSize(insertRequest.getValues().size()); + com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest request = super.getBulkInsertRequestBody(insertRequest, super.getVaultConfig()); + + response = this.processSync(request, insertRequest.getValues()); + return response; + } catch (ApiClientApiException e) { + String bodyString = gson.toJson(e.body()); + LogUtil.printErrorLog(ErrorLogs.INSERT_RECORDS_REJECTED.getLog()); + throw new SkyflowException(e.statusCode(), e, e.headers(), bodyString); + } catch (ExecutionException | InterruptedException e) { + LogUtil.printErrorLog(ErrorLogs.INSERT_RECORDS_REJECTED.getLog()); + throw new SkyflowException(e.getMessage()); + } } - // add methods for v3 SDK - public InsertResponse bulkInsert(BaseInsertRequest insertRequest, RequestOptions requestOptions) { - InsertResponse response = null; + public CompletableFuture bulkInsertAsync(InsertRequest insertRequest) throws SkyflowException { LogUtil.printInfoLog(InfoLogs.INSERT_TRIGGERED.getLog()); + try { + LogUtil.printInfoLog(InfoLogs.VALIDATE_INSERT_REQUEST.getLog()); + Validations.validateInsertRequest(insertRequest); + setBearerToken(); + configureInsertConcurrencyAndBatchSize(insertRequest.getValues().size()); + com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest request = super.getBulkInsertRequestBody(insertRequest, super.getVaultConfig()); + List errorRecords = Collections.synchronizedList(new ArrayList<>()); + List> futures = this.insertBatchFutures(request, errorRecords); + + return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply(v -> { + List successRecords = new ArrayList<>(); +// List errorRecords = new ArrayList<>(); + + for (CompletableFuture future : futures) { + com.skyflow.vault.data.InsertResponse futureResponse = future.join(); + if (futureResponse != null) { + if (futureResponse.getSuccess() != null) { + successRecords.addAll(futureResponse.getSuccess()); + } + if (futureResponse.getErrors() != null) { + errorRecords.addAll(futureResponse.getErrors()); + } + } + } + + return new com.skyflow.vault.data.InsertResponse(successRecords, errorRecords, insertRequest.getValues()); + }); + } catch (ApiClientApiException e) { + String bodyString = gson.toJson(e.body()); + LogUtil.printErrorLog(ErrorLogs.INSERT_RECORDS_REJECTED.getLog()); + throw new SkyflowException(e.statusCode(), e, e.headers(), bodyString); + } + } + + private com.skyflow.vault.data.InsertResponse processSync( + com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest insertRequest, + ArrayList> originalPayload + ) throws ExecutionException, InterruptedException { + LogUtil.printInfoLog(InfoLogs.PROCESSING_BATCHES.getLog()); +// List errorRecords = new ArrayList<>(); + List successRecords = new ArrayList<>(); + List errorRecords = Collections.synchronizedList(new ArrayList<>()); + List> futures = this.insertBatchFutures(insertRequest, errorRecords); + + CompletableFuture allFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + allFutures.join(); + + for (CompletableFuture future : futures) { + com.skyflow.vault.data.InsertResponse futureResponse = future.get(); + if (futureResponse != null) { + if (futureResponse.getSuccess() != null) { + successRecords.addAll(futureResponse.getSuccess()); + } + if (futureResponse.getErrors() != null) { + errorRecords.addAll(futureResponse.getErrors()); + } + } + } + com.skyflow.vault.data.InsertResponse response = new com.skyflow.vault.data.InsertResponse(successRecords, errorRecords, originalPayload); + LogUtil.printInfoLog(InfoLogs.INSERT_REQUEST_RESOLVED.getLog()); return response; } - public CompletableFuture bulkInsertAsync(BaseInsertRequest insertRequest, RequestOptions requestOptions){ - CompletableFuture future = null; - return future; + + private List> insertBatchFutures( + com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest insertRequest, + List errorRecords) { + List records = insertRequest.getRecords().get(); + + ExecutorService executor = Executors.newFixedThreadPool(insertConcurrencyLimit); + List> batches = Utils.createBatches(records, insertBatchSize); + List> futures = new ArrayList<>(); + + try { + for (int batchIndex = 0; batchIndex < batches.size(); batchIndex++) { + List batch = batches.get(batchIndex); + int batchNumber = batchIndex; + CompletableFuture future = CompletableFuture + .supplyAsync(() -> insertBatch(batch, insertRequest.getTableName().get()), executor) + .thenApply(response -> formatResponse(response, batchNumber, insertBatchSize)) + .exceptionally(ex -> { + errorRecords.addAll(handleBatchException(ex, batch, batchNumber, batches)); + return null; + }); + futures.add(future); + } + } finally { + executor.shutdown(); + } + return futures; + } + + private InsertResponse insertBatch(List batch, String tableName) { + com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest req = com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest.builder() + .vaultId(this.getVaultConfig().getVaultId()) + .tableName(tableName) + .records(batch) + .build(); + return this.getRecordsApi().insert(req); + } + + private void configureInsertConcurrencyAndBatchSize(int totalRequests) { + try { + Dotenv dotenv = Dotenv.load(); + String userProvidedBatchSize = dotenv.get("INSERT_BATCH_SIZE"); + String userProvidedConcurrencyLimit = dotenv.get("INSERT_CONCURRENCY_LIMIT"); + + if (userProvidedBatchSize != null) { + try { + int batchSize = Integer.parseInt(userProvidedBatchSize); + int maxBatchSize = Math.min(batchSize, Constants.MAX_INSERT_BATCH_SIZE); + if (maxBatchSize > 0) { + this.insertBatchSize = maxBatchSize; + } else { + LogUtil.printWarningLog(WarningLogs.INVALID_BATCH_SIZE_PROVIDED.getLog()); + this.insertBatchSize = Constants.INSERT_BATCH_SIZE; + } + } catch (NumberFormatException e) { + LogUtil.printWarningLog(WarningLogs.INVALID_BATCH_SIZE_PROVIDED.getLog()); + this.insertBatchSize = Constants.INSERT_BATCH_SIZE; + } + } + + // Max no of threads required to run all batches concurrently at once + int maxConcurrencyNeeded = (totalRequests + this.insertBatchSize - 1) / this.insertBatchSize; + + if (userProvidedConcurrencyLimit != null) { + try { + int concurrencyLimit = Integer.parseInt(userProvidedConcurrencyLimit); + int maxConcurrencyLimit = Math.min(concurrencyLimit, Constants.MAX_INSERT_CONCURRENCY_LIMIT); + + if (maxConcurrencyLimit > 0) { + this.insertConcurrencyLimit = Math.min(maxConcurrencyLimit, maxConcurrencyNeeded); + } else { + LogUtil.printWarningLog(WarningLogs.INVALID_CONCURRENCY_LIMIT_PROVIDED.getLog()); + this.insertConcurrencyLimit = Math.min(Constants.INSERT_CONCURRENCY_LIMIT, maxConcurrencyNeeded); + } + } catch (NumberFormatException e) { + LogUtil.printWarningLog(WarningLogs.INVALID_CONCURRENCY_LIMIT_PROVIDED.getLog()); + this.insertConcurrencyLimit = Math.min(Constants.INSERT_CONCURRENCY_LIMIT, maxConcurrencyNeeded); + } + } else { + this.insertConcurrencyLimit = Math.min(Constants.INSERT_CONCURRENCY_LIMIT, maxConcurrencyNeeded); + } + } catch (Exception e) { + this.insertBatchSize = Constants.INSERT_BATCH_SIZE; + int maxConcurrencyNeeded = (totalRequests + this.insertBatchSize - 1) / this.insertBatchSize; + this.insertConcurrencyLimit = Math.min(Constants.INSERT_CONCURRENCY_LIMIT, maxConcurrencyNeeded); + } } -} +} \ No newline at end of file diff --git a/v3/src/main/java/com/skyflow/vault/data/ErrorRecord.java b/v3/src/main/java/com/skyflow/vault/data/ErrorRecord.java new file mode 100644 index 00000000..9044f189 --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/ErrorRecord.java @@ -0,0 +1,39 @@ +package com.skyflow.vault.data; + +import com.google.gson.Gson; +import com.google.gson.annotations.Expose; + +public class ErrorRecord { + @Expose(serialize = true) + private int index; + @Expose(serialize = true) + private String error; + @Expose(serialize = true) + private int code; +// public ErrorRecord() { +// } + + public ErrorRecord(int index, String error, int code) { + this.index = index; + this.error = error; + this.code = code; + } + public String getError() { + return error; + } + + public int getCode() { + return code; + } + + public int getIndex() { + return index; + } + + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } +} \ No newline at end of file diff --git a/v3/src/main/java/com/skyflow/vault/data/InsertRequest.java b/v3/src/main/java/com/skyflow/vault/data/InsertRequest.java new file mode 100644 index 00000000..732df960 --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/InsertRequest.java @@ -0,0 +1,64 @@ +package com.skyflow.vault.data; + +import com.skyflow.enums.UpdateType; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class InsertRequest extends BaseInsertRequest { + private final InsertRequestBuilder builder; + + private InsertRequest(InsertRequestBuilder builder) { + super(builder); + this.builder = builder; + } + + public static InsertRequestBuilder builder() { + return new InsertRequestBuilder(); + } + + public List getUpsert() { + return this.builder.upsert; + } + + public UpdateType getUpsertType() { + return this.builder.upsertType; + } + + public static final class InsertRequestBuilder extends BaseInsertRequestBuilder { + private List upsert; + + private UpdateType upsertType; + + private InsertRequestBuilder() { + super(); + } + + @Override + public InsertRequestBuilder table(String table) { + super.table(table); + return this; + } + + @Override + public InsertRequestBuilder values(ArrayList> values) { + super.values(values); + return this; + } + + public InsertRequestBuilder upsert(List upsert) { + this.upsert = upsert; + return this; + } + + public InsertRequestBuilder upsertType(UpdateType upsertType) { + this.upsertType = upsertType; + return this; + } + + public InsertRequest build() { + return new InsertRequest(this); + } + } +} diff --git a/v3/src/main/java/com/skyflow/vault/data/InsertResponse.java b/v3/src/main/java/com/skyflow/vault/data/InsertResponse.java new file mode 100644 index 00000000..a84e121c --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/InsertResponse.java @@ -0,0 +1,69 @@ +package com.skyflow.vault.data; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.annotations.Expose; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +public class InsertResponse { + // These members will be included in the toString() output + @Expose(serialize = true) + private Summary summary; + @Expose(serialize = true) + private List success; + @Expose(serialize = true) + private List errors; + + // Internal fields. Should not be included in toString() output + private ArrayList> originalPayload; + private ArrayList> recordsToRetry; + + public InsertResponse(List successRecords, List errorRecords) { + this.success = successRecords; + this.errors = errorRecords; + } + + public InsertResponse( + List successRecords, + List errorRecords, + ArrayList> originalPayload + ) { + this.success = successRecords; + this.errors = errorRecords; + this.originalPayload = originalPayload; + this.summary = new Summary(this.originalPayload.size(), this.success.size(), this.errors.size()); + } + + public Summary getSummary() { + return this.summary; + } + + public List getSuccess() { + return this.success; + } + + public List getErrors() { + return this.errors; + } + + public ArrayList> getRecordsToRetry() { + if (recordsToRetry == null) { + recordsToRetry = new ArrayList<>(); + recordsToRetry = errors.stream() + .filter(error -> (error.getCode() >= 500 && error.getCode() <= 599) && error.getCode() != 529) + .map(errorRecord -> originalPayload.get(errorRecord.getIndex())) + .collect(Collectors.toCollection(ArrayList::new)); + } + return recordsToRetry; + } + + @Override + public String toString() { + Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().create(); + return gson.toJson(this); + } +} \ No newline at end of file diff --git a/v3/src/main/java/com/skyflow/vault/data/Success.java b/v3/src/main/java/com/skyflow/vault/data/Success.java new file mode 100644 index 00000000..db302e6c --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/Success.java @@ -0,0 +1,46 @@ +package com.skyflow.vault.data; +import com.google.gson.Gson; +import com.google.gson.annotations.Expose; + +import java.util.List; +import java.util.Map; + +public class Success { + @Expose(serialize = true) + private int index; + @Expose(serialize = true) + private String skyflow_id; + @Expose(serialize = true) + private Map> tokens; + @Expose(serialize = true) + private Map data; + + public int getIndex() { + return index; + } + + public Success(int index, String skyflow_id, Map> tokens, Map data) { + this.index = index; + this.skyflow_id = skyflow_id; + this.tokens = tokens; + this.data = data; + } + + public String getSkyflowId() { + return skyflow_id; + } + + public Map> getTokens() { + return tokens; + } + + public Map getData() { + return data; + } + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } +} \ No newline at end of file diff --git a/v3/src/main/java/com/skyflow/vault/data/Summary.java b/v3/src/main/java/com/skyflow/vault/data/Summary.java new file mode 100644 index 00000000..c15db6cb --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/Summary.java @@ -0,0 +1,42 @@ +package com.skyflow.vault.data; + +import com.google.gson.Gson; +import com.google.gson.annotations.Expose; + +public class Summary { + @Expose(serialize = true) + private int totalRecords; + @Expose(serialize = true) + private int totalInserted; + @Expose(serialize = true) + private int totalFailed; + + public Summary() { + } + + public Summary(int totalRecords, int totalInserted, int totalFailed) { + this.totalRecords = totalRecords; + this.totalInserted = totalInserted; + this.totalFailed = totalFailed; + } + + public int getTotalRecords() { + return totalRecords; + } + + + public int getTotalInserted() { + return totalInserted; + } + + public int getTotalFailed() { + return totalFailed; + } + + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } +} \ No newline at end of file diff --git a/v3/src/main/java/com/skyflow/vault/data/Token.java b/v3/src/main/java/com/skyflow/vault/data/Token.java new file mode 100644 index 00000000..642d62c4 --- /dev/null +++ b/v3/src/main/java/com/skyflow/vault/data/Token.java @@ -0,0 +1,23 @@ +package com.skyflow.vault.data; + +import com.google.gson.annotations.Expose; + +public class Token { + @Expose(serialize = true) + private String token; + @Expose(serialize = true) + private String tokenGroupName; + + public String getToken() { + return token; + } + + public String getTokenGroupName() { + return tokenGroupName; + } + + public Token(String token, String tokenGroupName) { + this.token = token; + this.tokenGroupName = tokenGroupName; + } +} \ No newline at end of file diff --git a/v3/test/java/com/skyflow/utils/UtilsTests.java b/v3/test/java/com/skyflow/utils/UtilsTests.java index 096a4ec6..74d4f227 100644 --- a/v3/test/java/com/skyflow/utils/UtilsTests.java +++ b/v3/test/java/com/skyflow/utils/UtilsTests.java @@ -1,20 +1,127 @@ package com.skyflow.utils; import com.google.gson.JsonObject; +import com.skyflow.config.Credentials; +import com.skyflow.enums.Env; +import com.skyflow.errors.ErrorCode; +import com.skyflow.errors.ErrorMessage; +import com.skyflow.errors.SkyflowException; +import com.skyflow.generated.auth.rest.core.ApiClientApiException; +import com.skyflow.generated.rest.types.InsertRecordData; +import com.skyflow.generated.rest.types.InsertResponse; +import com.skyflow.generated.rest.types.RecordResponseObject; +import com.skyflow.vault.data.ErrorRecord; +import com.skyflow.vault.data.Success; +import com.skyflow.vault.data.Token; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; +import java.util.*; + public class UtilsTests { private static final String INVALID_EXCEPTION_THROWN = "Should not have thrown any exception"; - private static final String EXCEPTION_NOT_THROWN = "Should have thrown an exception"; + private static final String EXCEPTIONNOTTHROWN = "Should have thrown an exception"; + private static String filePath = null; + private static String credentialsString = null; + private static String token = null; + private static String context = null; + private static ArrayList roles = null; + + @BeforeClass + public static void setup() { + filePath = "invalid/file/path/credentials.json"; + credentialsString = "invalid credentials string"; + token = "invalid-token"; + context = "testcontext"; + roles = new ArrayList<>(); + String role = "testrole"; + roles.add(role); + SdkVersion.setSdkPrefix(Constants.SDK_PREFIX); + } + + @Test + public void testGetVaultURL() { + // Test with production environment + String prodUrl = Utils.getVaultURL("abc123", Env.PROD); + Assert.assertEquals( + "https://abc123.skyvault.skyflowapis.com", + prodUrl + ); + + // Test with development environment + String devUrl = Utils.getVaultURL("xyz789", Env.DEV); + Assert.assertEquals( + "https://xyz789.skyvault.skyflowapis.dev", + devUrl + ); + } + @Test(expected = NullPointerException.class) + public void testGetVaultURLWithNullEnv() { + Utils.getVaultURL("abc123", null); + } + + @Test + public void testGetVaultURLWithEmptyClusterId() { + String url = Utils.getVaultURL("", Env.PROD); + Assert.assertEquals( + "https://.skyvault.skyflowapis.com", + url + ); + } + + @Test + public void testGenerateBearerTokenWithCredentialsFile() { + try { + Credentials credentials = new Credentials(); + credentials.setPath(filePath); + credentials.setContext(context); + credentials.setRoles(roles); + Utils.generateBearerToken(credentials); + Assert.fail(EXCEPTIONNOTTHROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.FileNotFound.getMessage(), filePath), + e.getMessage() + ); + } + } + + @Test + public void testGenerateBearerTokenWithCredentialsString() { + try { + Credentials credentials = new Credentials(); + credentials.setCredentialsString(credentialsString); + credentials.setContext(context); + credentials.setRoles(roles); + Utils.generateBearerToken(credentials); + Assert.fail(EXCEPTIONNOTTHROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals(ErrorMessage.CredentialsStringInvalidJson.getMessage(), e.getMessage()); + } + } + + @Test + public void testGenerateBearerTokenWithToken() { + try { + Credentials credentials = new Credentials(); + credentials.setToken(token); + credentials.setContext(context); + credentials.setRoles(roles); + String bearerToken = Utils.generateBearerToken(credentials); + Assert.assertEquals(token, bearerToken); + } catch (SkyflowException e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } @Test public void testGetMetrics() { try { JsonObject metrics = Utils.getMetrics(); - String sdkVersion = Constants.SDK_VERSION; Assert.assertNotNull(metrics.get(Constants.SDK_METRIC_NAME_VERSION)); - Assert.assertEquals("skyflow-java@" + sdkVersion, metrics.get(Constants.SDK_METRIC_NAME_VERSION).getAsString()); Assert.assertNotNull(metrics.get(Constants.SDK_METRIC_CLIENT_DEVICE_MODEL)); Assert.assertNotNull(metrics.get(Constants.SDK_METRIC_CLIENT_OS_DETAILS)); Assert.assertNotNull(metrics.get(Constants.SDK_METRIC_RUNTIME_DETAILS)); @@ -22,4 +129,398 @@ public void testGetMetrics() { Assert.fail(INVALID_EXCEPTION_THROWN); } } -} + + @Test + public void testGetMetricsWithException() { + try { + System.clearProperty("os.name"); + System.clearProperty("os.version"); + System.clearProperty("java.version"); + + String sdkVersion = Constants.SDK_VERSION; + JsonObject metrics = Utils.getMetrics(); + Assert.assertEquals("skyflow-java@" + sdkVersion, metrics.get(Constants.SDK_METRIC_NAME_VERSION).getAsString()); + Assert.assertEquals("Java@", metrics.get(Constants.SDK_METRIC_RUNTIME_DETAILS).getAsString()); + Assert.assertTrue(metrics.get(Constants.SDK_METRIC_CLIENT_DEVICE_MODEL).getAsString().isEmpty()); + Assert.assertTrue(metrics.get(Constants.SDK_METRIC_CLIENT_OS_DETAILS).getAsString().isEmpty()); + } catch (Exception e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } + + @Test + public void testCreateBatchesMultipleBatches() { + List records = new ArrayList<>(); + for (int i = 0; i < 125; i++) { + records.add(InsertRecordData.builder().build()); + } + List> batches = Utils.createBatches(records, 50); + + Assert.assertEquals(3, batches.size()); + Assert.assertEquals(50, batches.get(0).size()); + Assert.assertEquals(50, batches.get(1).size()); + Assert.assertEquals(25, batches.get(2).size()); + } + + @Test + public void testCreateBatchesWithEmptyList() { + List records = new ArrayList<>(); + List> batches = Utils.createBatches(records, 50); + Assert.assertTrue("Batches should be empty for empty input", batches.isEmpty()); + } + + @Test + public void testCreateBatchesWithSmallerSizeThanBatch() { + List records = new ArrayList<>(); + for (int i = 0; i < 25; i++) { + records.add(InsertRecordData.builder().build()); + } + List> batches = Utils.createBatches(records, 50); + + Assert.assertEquals("Should create single batch", 1, batches.size()); + Assert.assertEquals("Batch should contain all records", 25, batches.get(0).size()); + } + + @Test + public void testCreateBatchesWithExactBatchSize() { + List records = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + records.add(InsertRecordData.builder().build()); + } + List> batches = Utils.createBatches(records, 50); + + Assert.assertEquals("Should create single batch", 1, batches.size()); + Assert.assertEquals("Batch should have exact size", 50, batches.get(0).size()); + } + + @Test + public void testCreateBatchesPreservesOrder() { + List records = new ArrayList<>(); + for (int i = 0; i < 75; i++) { + InsertRecordData record = InsertRecordData.builder() + .data(Optional.of(Collections.singletonMap("id", String.valueOf(i)))) + .build(); + records.add(record); + } + + List> batches = Utils.createBatches(records, 50); + + Assert.assertEquals("Should create two batches", 2, batches.size()); + Assert.assertEquals("First record in first batch should be 0", + "0", batches.get(0).get(0).getData().get().get("id")); + Assert.assertEquals("First record in second batch should be 50", + "50", batches.get(1).get(0).getData().get().get("id")); + } + + @Test(expected = NullPointerException.class) + public void testCreateBatchesWithNullList() { + Utils.createBatches(null, 50); + } + + @Test + public void testCreateErrorRecord() { + Map recordMap = new HashMap<>(); + recordMap.put("error", "Test error"); + recordMap.put("message", "Test message"); + recordMap.put("http_code", 400); + + ErrorRecord error = Utils.createErrorRecord(recordMap, 1); + + Assert.assertEquals(1, error.getIndex()); + Assert.assertEquals("Test error", error.getError()); + Assert.assertEquals(400, error.getCode()); + } + + @Test + public void testHandleBatchExceptionApiClientExceptionWithSingleError() { + List batch = Arrays.asList(InsertRecordData.builder().build(), InsertRecordData.builder().build()); + List> batches = Collections.singletonList(batch); + + Map errorMap = new HashMap<>(); + errorMap.put("error", "Common error"); + errorMap.put("http_code", 403); + + Map responseBody = new HashMap<>(); + responseBody.put("error", errorMap); + + ApiClientApiException apiException = new ApiClientApiException("Forbidden", 403, responseBody); + Exception exception = new Exception("Test exception", apiException); + + List errors = Utils.handleBatchException(exception, batch, 0, batches); + + Assert.assertEquals("Should have errors for all records", 2, errors.size()); + Assert.assertEquals("Error message should be same", "Test exception", errors.get(0).getError()); + Assert.assertEquals("Error code should be same", 500, errors.get(0).getCode()); + Assert.assertEquals("First error index", 0, errors.get(0).getIndex()); + Assert.assertEquals("Second error index", 1, errors.get(1).getIndex()); + } + + @Test + public void testHandleBatchExceptionWithNonApiClientException() { + List batch = Arrays.asList(InsertRecordData.builder().build(), InsertRecordData.builder().build()); + List> batches = Collections.singletonList(batch); + + RuntimeException exception = new RuntimeException("Unexpected error"); + + List errors = Utils.handleBatchException(exception, batch, 0, batches); + + Assert.assertEquals("Should have errors for all records", 2, errors.size()); + Assert.assertEquals("Error message should match", "Unexpected error", errors.get(0).getError()); + Assert.assertEquals("Error code should be 500", 500, errors.get(0).getCode()); + Assert.assertEquals("First error index", 0, errors.get(0).getIndex()); + Assert.assertEquals("Second error index", 1, errors.get(1).getIndex()); + } + + @Test + public void testHandleBatchExceptionWithNonZeroBatchNumber() { + List batch = Arrays.asList(InsertRecordData.builder().build(), InsertRecordData.builder().build()); + List> batches = Arrays.asList(new ArrayList<>(), batch); + + RuntimeException exception = new RuntimeException("Batch error"); + + List errors = Utils.handleBatchException(exception, batch, 1, batches); + + Assert.assertEquals("Should have errors for all records", 2, errors.size()); + Assert.assertEquals("First error index should be offset", 2, errors.get(0).getIndex()); + Assert.assertEquals("Second error index should be offset", 3, errors.get(1).getIndex()); + } + + @Test + public void testHandleBatchExceptionWithNullResponseBody() { + List batch = Arrays.asList(InsertRecordData.builder().build(), InsertRecordData.builder().build()); + List> batches = Collections.singletonList(batch); + + ApiClientApiException apiException = new ApiClientApiException("Bad Request", 400, null); + Exception exception = new Exception("Test exception", apiException); + + List errors = Utils.handleBatchException(exception, batch, 0, batches); + Assert.assertEquals("Should return empty list for null response body", 2, errors.size()); + } + + @Test + public void testFormatResponseWithSuccessAndErrorRecords() { + RecordResponseObject successRecord = RecordResponseObject.builder() + .skyflowId(Optional.of("testId1")) + .error(Optional.empty()) + .build(); + RecordResponseObject errorRecord = RecordResponseObject.builder() + .error(Optional.of("Test error")) + .httpCode(Optional.of(400)) + .build(); + + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Arrays.asList(successRecord, errorRecord))) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + Assert.assertNotNull(result.getSuccess()); + Assert.assertEquals(1, result.getSuccess().size()); + Assert.assertEquals("testId1", result.getSuccess().get(0).getSkyflowId()); + + Assert.assertNotNull(result.getErrors()); + Assert.assertEquals(1, result.getErrors().size()); + Assert.assertEquals("Test error", result.getErrors().get(0).getError()); + Assert.assertEquals(400, result.getErrors().get(0).getCode()); + } + + @Test + public void testFormatResponseWithNullResponse() { + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(null, 0, 50); + Assert.assertNull(result); + } + + @Test + public void testFormatResponseWithSuccessRecordsOnly() { + RecordResponseObject successRecord1 = RecordResponseObject.builder() + .skyflowId(Optional.of("id1")) + .error(Optional.empty()) + .build(); + RecordResponseObject successRecord2 = RecordResponseObject.builder() + .skyflowId(Optional.of("id2")) + .error(Optional.empty()) + .build(); + + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Arrays.asList(successRecord1, successRecord2))) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + Assert.assertNotNull("Response should not be null", result); + Assert.assertEquals("Should have two success records", 2, result.getSuccess().size()); + Assert.assertEquals("First skyflow ID should match", "id1", result.getSuccess().get(0).getSkyflowId()); + Assert.assertEquals("Second skyflow ID should match", "id2", result.getSuccess().get(1).getSkyflowId()); + Assert.assertTrue("Error list should be empty", result.getErrors().isEmpty()); + } + + @Test + public void testFormatResponseWithErrorRecordsOnly() { + RecordResponseObject errorRecord1 = RecordResponseObject.builder() + .error(Optional.of("Error 1")) + .httpCode(Optional.of(400)) + .build(); + RecordResponseObject errorRecord2 = RecordResponseObject.builder() + .error(Optional.of("Error 2")) + .httpCode(Optional.of(500)) + .build(); + + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Arrays.asList(errorRecord1, errorRecord2))) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + Assert.assertNotNull("Response should not be null", result); + Assert.assertEquals("Should have two error records", 2, result.getErrors().size()); + Assert.assertEquals("First error message should match", "Error 1", result.getErrors().get(0).getError()); + Assert.assertEquals("First error code should match", 400, result.getErrors().get(0).getCode()); + Assert.assertEquals("Second error message should match", "Error 2", result.getErrors().get(1).getError()); + Assert.assertEquals("Second error code should match", 500, result.getErrors().get(1).getCode()); + Assert.assertTrue("Success list should be empty", result.getSuccess().isEmpty()); + } + + @Test + public void testFormatResponseWithBatchOffset() { + RecordResponseObject successRecord = RecordResponseObject.builder() + .skyflowId(Optional.of("id1")) + .error(Optional.empty()) + .build(); + RecordResponseObject errorRecord = RecordResponseObject.builder() + .error(Optional.of("Error")) + .httpCode(Optional.of(400)) + .build(); + + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Arrays.asList(successRecord, errorRecord))) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 1, 50); + + Assert.assertNotNull("Response should not be null", result); + Assert.assertEquals("Should have correct index for error record", 51, result.getErrors().get(0).getIndex()); + } + + @Test + public void testFormatResponseWithEmptyRecords() { + InsertResponse response = InsertResponse.builder() + .records(Optional.of(new ArrayList<>())) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + Assert.assertNotNull("Response should not be null", result); + Assert.assertTrue("Success list should be empty", result.getSuccess().isEmpty()); + Assert.assertTrue("Error list should be empty", result.getErrors().isEmpty()); + } + + @Test + public void testFormatResponseWithTokens() { + Map tokens = new HashMap<>(); + tokens.put("field1", "token1"); + tokens.put("field2", "token2"); + + RecordResponseObject successRecord = RecordResponseObject.builder() + .skyflowId(Optional.of("id1")) + .tokens(Optional.of(tokens)) + .error(Optional.empty()) + .build(); + + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Collections.singletonList(successRecord))) + .build(); + + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + Assert.assertNotNull("Response should not be null", result); + Assert.assertEquals("Should have one success record", 1, result.getSuccess().size()); + Assert.assertEquals("Skyflow ID should match", "id1", result.getSuccess().get(0).getSkyflowId()); + } + @Test + public void testFormatResponseWithTokenListMapping() { + // Prepare test data + Map tokenData = new HashMap<>(); + List> tokenList = new ArrayList<>(); + Map tokenMap = new HashMap<>(); + tokenMap.put("token", "token123"); + tokenMap.put("tokenGroupName", "group1"); + tokenList.add(tokenMap); + tokenData.put("field1", tokenList); + + // Create success record with tokens + RecordResponseObject successRecord = RecordResponseObject.builder() + .skyflowId(Optional.of("id1")) + .tokens(Optional.of(tokenData)) + .error(Optional.empty()) + .build(); + + // Create response object + InsertResponse response = InsertResponse.builder() + .records(Optional.of(Collections.singletonList(successRecord))) + .build(); + + // Format response + com.skyflow.vault.data.InsertResponse result = Utils.formatResponse(response, 0, 50); + + // Assertions + Assert.assertNotNull("Response should not be null", result); + Assert.assertEquals("Should have one success record", 1, result.getSuccess().size()); + + Success successResult = result.getSuccess().get(0); + Assert.assertEquals("Skyflow ID should match", "id1", successResult.getSkyflowId()); + + Map> tokens = successResult.getTokens(); + Assert.assertNotNull("Tokens map should not be null", tokens); + Assert.assertTrue("Should contain field1", tokens.containsKey("field1")); + + List tokensList = tokens.get("field1"); + Assert.assertEquals("Should have one token", 1, tokensList.size()); + Assert.assertEquals("Token value should match", "token123", tokensList.get(0).getToken()); + Assert.assertEquals("Token group name should match", "group1", tokensList.get(0).getTokenGroupName()); + } + @Test + public void testHandleBatchExceptionWithRecordsInResponseBody() { + // Prepare test data + List batch = Arrays.asList( + InsertRecordData.builder().build(), + InsertRecordData.builder().build() + ); + List> batches = Collections.singletonList(batch); + + // Create nested records with errors + List> recordsList = new ArrayList<>(); + Map record1 = new HashMap<>(); + record1.put("error", "Error 1"); + record1.put("http_code", 400); + Map record2 = new HashMap<>(); + record2.put("error", "Error 2"); + record2.put("http_code", 401); + recordsList.add(record1); + recordsList.add(record2); + + // Create response body + Map responseBody = new HashMap<>(); + responseBody.put("records", recordsList); + + // Create API exception + ApiClientApiException apiException = new ApiClientApiException("Bad Request", 400, responseBody); + Exception exception = new Exception("Test exception", apiException); + + // Test the method + List errors = Utils.handleBatchException(exception, batch, 0, batches); + + // Assertions + Assert.assertNotNull("Errors list should not be null", errors); + Assert.assertEquals("Should have two error records", 2, errors.size()); + + // Verify first error + Assert.assertEquals("First error message should match", "Test exception", errors.get(0).getError()); + Assert.assertEquals("First error code should match", 500, errors.get(0).getCode()); + Assert.assertEquals("First error index should be 0", 0, errors.get(0).getIndex()); + + // Verify second error + Assert.assertEquals("Second error message should match", "Test exception", errors.get(1).getError()); + Assert.assertEquals("Second error code should match", 500, errors.get(1).getCode()); + Assert.assertEquals("Second error index should be 1", 1, errors.get(1).getIndex()); + } +} \ No newline at end of file diff --git a/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java b/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java new file mode 100644 index 00000000..50e14087 --- /dev/null +++ b/v3/test/java/com/skyflow/vault/controller/VaultControllerTests.java @@ -0,0 +1,218 @@ +package com.skyflow.vault.controller; + +import com.skyflow.config.Credentials; +import com.skyflow.config.VaultConfig; +import com.skyflow.enums.Env; +import com.skyflow.errors.ErrorCode; +import com.skyflow.errors.ErrorMessage; +import com.skyflow.errors.SkyflowException; +import com.skyflow.utils.Constants; +import com.skyflow.utils.SdkVersion; +import com.skyflow.utils.Utils; +import com.skyflow.vault.data.InsertRequest; +import okhttp3.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; + +public class VaultControllerTests { + private static final String INVALID_EXCEPTION_THROWN = "Should not have thrown any exception"; + private static final String EXCEPTION_NOT_THROWN = "Should have thrown an exception"; + private static String vaultID; + private static String clusterID; + private static VaultConfig vaultConfig; + private VaultController vaultController; + + private OkHttpClient mockClient; + + @BeforeClass + public static void setupClass() { + vaultID = "vault123"; + clusterID = "cluster123"; + SdkVersion.setSdkPrefix(Constants.SDK_PREFIX); + } + + @Before + public void setup() { +// Create mock interceptor + Interceptor mockInterceptor = chain -> { + // Create mock response + String mockResponseBody = "{\"records\":[{\"skyflowId\":\"test-id-123\",\"tokens\":{}}]}"; + return new Response.Builder() + .code(200) + .message("OK") + .protocol(Protocol.HTTP_1_1) + .request(chain.request()) + .body(ResponseBody.create( + MediaType.parse("application/json"), + mockResponseBody + )) + .build(); + }; + + // Create client with mock interceptor + mockClient = new OkHttpClient.Builder() + .addInterceptor(mockInterceptor) + .build(); + vaultConfig = new VaultConfig(); + vaultConfig.setVaultId(vaultID); + vaultConfig.setClusterId(clusterID); + vaultConfig.setEnv(Env.DEV); + + Credentials credentials = new Credentials(); + credentials.setToken("valid-token"); + vaultConfig.setCredentials(credentials); + this.vaultController = new VaultController(vaultConfig, credentials); +// spyController = Mockito.spy(vaultController); + // Create mock response +// String mockResponseBody = "{\"records\":[{\"skyflowId\":\"test-id-123\",\"tokens\":{}}]}"; +// ResponseBody responseBody = ResponseBody.create( +// MediaType.parse("application/json"), +// mockResponseBody +// ); +// +// Response mockResponse = new Response.Builder() +// .code(200) +// .message("OK") +// .protocol(Protocol.HTTP_1_1) +// .request(new Request.Builder().url("https://test.com").build()) +// .body(responseBody) +// .build(); +// +// // Mock Call +// Call mockCall = PowerMockito.mock(Call.class); +// try { +// PowerMockito.when(mockCall.execute()).thenReturn(mockResponse); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// +// // Mock OkHttpClient +// mockClient = PowerMockito.mock(OkHttpClient.class); +// PowerMockito.when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + + + } + +// @Test +// public void testBulkInsertSuccess() throws SkyflowException { +// // Prepare test data +// ArrayList> records = new ArrayList<>(); +// HashMap record = new HashMap<>(); +// record.put("field1", "value1"); +// records.add(record); +// +// InsertRequest request = InsertRequest.builder() +// .values(records) +// .table("test_table") +// .build(); +// +// // Create mock response +// List responseObjects = new ArrayList<>(); +// RecordResponseObject responseObject = RecordResponseObject.builder() +// .skyflowId("test-id-123") +// .data(record) +// .build(); +// responseObjects.add(responseObject); +// +// InsertResponse mockResponse = InsertResponse.builder() +// .records(responseObjects) +// .build(); +// +// InsertResponse resp = InsertResponse.builder().records(responseObjects).build(); +// // Mock insertBatch method +// when(vaultController.bulkInsert(any())); +// +// // Execute test +// com.skyflow.vault.data.InsertResponse response = vaultController.bulkInsert(request); +// +// // Verify response +// Assert.assertNotNull(response); +// Assert.assertNotNull(response.getSuccess()); +// Assert.assertEquals(1, response.getSuccess().size()); +// Assert.assertEquals("test-id-123", response.getSuccess().get(0).getSkyflowId()); +// +// // Verify method was called +//// verify(vaultController).insertBatch(any(), eq("test_table")); +// } + @Test + public void testInvalidRequestInInsertMethod() { + try { + InsertRequest request = InsertRequest.builder().build(); + vaultController.bulkInsert(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.TableKeyError.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } +// @Test +// public void testBulkInsertWithApiError3() throws SkyflowException { +// // Prepare test data +// ArrayList> records = new ArrayList<>(); +// HashMap record = new HashMap<>(); +// record.put("field1", "value1"); +// records.add(record); +// +// InsertRequest request = InsertRequest.builder() +// .values(records) +// .table("test_table") +// .build(); +// +// try { +// com.skyflow.vault.data.InsertResponse res = vaultController.bulkInsert(request); +// String resp = "{\"summary\":{\"total_records\":1,\"total_inserted\":0,\"total_failed\":1},\"errors\":[{\"index\":0,\"error\":\"com.skyflow.generated.rest.core.ApiClientException: Network error executing HTTP request\",\"code\":500}]}"; +// Assert.assertEquals(res.toString(), resp); +// } catch (SkyflowException e) { +// Assert.assertEquals(400, e.getHttpCode()); +// } +// } + +// @Test +// public void testInsert(){ +// // Prepare test data +// ArrayList> records = new ArrayList<>(); +// HashMap record = new HashMap<>(); +// record.put("field1", "value1"); +// records.add(record); +// +// InsertRequest request = InsertRequest.builder() +// .values(records) +// .table("test_table") +// .build(); +// List recordDataList = new ArrayList<>(); +// InsertRecordData recordData = InsertRecordData.builder().data(record).build(); +// recordDataList.add(recordData); +// +// com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest request1 = com.skyflow.generated.rest.resources.recordservice.requests.InsertRequest.builder() +// .records(recordDataList).vaultId("id").tableName("test_table").build(); +// RecordResponseObject recordResponseObject = RecordResponseObject.builder().data(record).build(); +// List recordResponseObjects = new ArrayList<>(); +// recordResponseObjects.add(recordResponseObject); +// +//// ApiClient apiClient = PowerMockito.mock(ApiClient.class); +////// ApiClientBuilder apiClientBuilder = PowerMockito.mock(ApiClientBuilder.class); +//// RecordserviceClient recordserviceClient = PowerMockito.mock(RecordserviceClient.class); +//// apiClient = ApiClient.builder().url("https://demo.com").httpClient(new OkHttpClient()).build(); +//// when(recordserviceClient.insert(request1)).thenReturn(apiClient.recordservice().insert(request1)); +// +//// PowerMockito.when(OkHttpClient.class).thenReturn(this.mockClient); +// PowerMockito.mock(OkHttpClient.class); +// +// try { +// com.skyflow.vault.data.InsertResponse res = vaultController.bulkInsert(request); +// String resp = "{\"summary\":{\"total_records\":1,\"total_inserted\":0,\"total_failed\":1},\"errors\":[{\"index\":0,\"error\":\"com.skyflow.generated.rest.core.ApiClientException: Network error executing HTTP request\",\"code\":500}]}"; +// Assert.assertEquals(res.toString(), resp); +// System.out.println("resppp=>"+ res); +// } catch (SkyflowException e) { +// Assert.assertEquals(400, e.getHttpCode()); +// } +// } +} \ No newline at end of file diff --git a/v3/test/java/com/skyflow/vault/data/InsertTests.java b/v3/test/java/com/skyflow/vault/data/InsertTests.java new file mode 100644 index 00000000..af8b050f --- /dev/null +++ b/v3/test/java/com/skyflow/vault/data/InsertTests.java @@ -0,0 +1,269 @@ +package com.skyflow.vault.data; + +import com.skyflow.Skyflow; +import com.skyflow.config.Credentials; +import com.skyflow.config.VaultConfig; +import com.skyflow.enums.Env; +import com.skyflow.errors.ErrorCode; +import com.skyflow.errors.ErrorMessage; +import com.skyflow.errors.SkyflowException; +import com.skyflow.utils.Constants; +import com.skyflow.utils.SdkVersion; +import com.skyflow.utils.Utils; +import com.skyflow.utils.validations.Validations; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class InsertTests { + private static final String INVALID_EXCEPTION_THROWN = "Should not have thrown any exception"; + private static final String EXCEPTION_NOT_THROWN = "Should have thrown an exception"; + private static final String requestId = "95be08fc-4d13-4335-8b8d-24e85d53ed1d"; + private static String vaultID = null; + private static String clusterID = null; + private static String table = null; + private static ArrayList> values = null; + private static ArrayList> tokens = null; + private static HashMap valueMap = null; + private static HashMap tokenMap = null; + private static List upsert = new ArrayList<>(); + private static Skyflow skyflowClient = null; + + @BeforeClass + public static void setup() { + + vaultID = "vault123"; + clusterID = "cluster123"; + + Credentials credentials = new Credentials(); + credentials.setToken("valid-token"); + + VaultConfig vaultConfig = new VaultConfig(); + vaultConfig.setVaultId(vaultID); + vaultConfig.setClusterId(clusterID); + vaultConfig.setEnv(Env.DEV); + vaultConfig.setCredentials(credentials); + + table = "test_table"; + values = new ArrayList<>(); + tokens = new ArrayList<>(); + valueMap = new HashMap<>(); + tokenMap = new HashMap<>(); + upsert.add("upsert_column"); + SdkVersion.setSdkPrefix(Constants.SDK_PREFIX); + } + + @Before + public void setupTest() { + values.clear(); + tokens.clear(); + valueMap.clear(); + valueMap.put("test_column_1", "test_value_1"); + valueMap.put("test_column_2", "test_value_2"); + tokenMap.clear(); + tokenMap.put("test_column_1", "test_token_1"); + } + + @Test + public void testValidInputInInsertRequestValidations() { + try { + values.add(valueMap); + tokens.add(tokenMap); + + InsertRequest request = InsertRequest.builder() + .table(table) + .upsert(upsert) + .values(values) + .build(); + Validations.validateInsertRequest(request); + + Assert.assertEquals(table, request.getTable()); + Assert.assertEquals(upsert, request.getUpsert()); + Assert.assertEquals(1, request.getValues().size()); + } catch (SkyflowException e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } + + @Test + public void testValidInputInInsertRequestValidationsWithTokenModeDisable() { + try { + values.add(valueMap); + tokens.add(tokenMap); + InsertRequest request = InsertRequest.builder() + .table(table) + .upsert(upsert) + .values(values) + .build(); + Validations.validateInsertRequest(request); + + Assert.assertEquals(table, request.getTable()); + Assert.assertEquals(upsert, request.getUpsert()); + Assert.assertEquals(1, request.getValues().size()); + } catch (SkyflowException e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } + + @Test + public void testNoTableInInsertRequestValidations() { + InsertRequest request = InsertRequest.builder().build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.TableKeyError.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testEmptyTableInInsertRequestValidations() { + InsertRequest request = InsertRequest.builder().table("").build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.EmptyTable.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testNoValuesInInsertRequestValidations() { + InsertRequest request = InsertRequest.builder().table(table).build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.ValuesKeyError.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testEmptyValuesInInsertRequestValidations() { + InsertRequest request = InsertRequest.builder().table(table).values(values).build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.EmptyValues.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testEmptyKeyInValuesInInsertRequestValidations() { + valueMap.put("", "test_value_3"); + values.add(valueMap); + InsertRequest request = InsertRequest.builder().table(table).values(values).build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.EmptyKeyInValues.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testEmptyValueInValuesInInsertRequestValidations() { + valueMap.put("test_column_3", ""); + values.add(valueMap); + InsertRequest request = InsertRequest.builder().table(table).values(values).build(); + try { + Validations.validateInsertRequest(request); + Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.EmptyValueInValues.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testEmptyUpsertInInsertRequestValidations() { + values.add(valueMap); + InsertRequest request = InsertRequest.builder().table(table).values(values).upsert(new ArrayList<>()).build(); + try { + Validations.validateInsertRequest(request); +// Assert.fail(EXCEPTION_NOT_THROWN); + } catch (SkyflowException e) { + Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); + Assert.assertEquals( + Utils.parameterizedString(ErrorMessage.EmptyUpsert.getMessage(), Constants.SDK_PREFIX), + e.getMessage() + ); + } + } + + @Test + public void testInsertResponse() { + try { + Map value = new HashMap<>(); + value.put("test_column_1", "test_value_1"); + Success success = new Success(0, "id", null, null); + + List successList = new ArrayList<>(); + successList.add(success); + InsertResponse response = new InsertResponse(successList, null); + String responseString = "{\"success\":[{\"index\":0,\"skyflow_id\":\"id\"}]}"; + Assert.assertEquals(1, response.getSuccess().size()); + Assert.assertNull(response.getErrors()); + Assert.assertEquals(responseString, response.toString()); + } catch (Exception e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } + + @Test + public void testInsertErrorResponse() { + try { + HashMap value = new HashMap<>(); + value.put("test_column_1", "test_value_1"); + Success success = new Success(0, "id", null, value); + + List successList = new ArrayList<>(); + successList.add(success); + + List errorList = new ArrayList<>(); + ErrorRecord error = new ErrorRecord(1, "Bad Request", 400); + errorList.add(error); + + InsertResponse response1 = new InsertResponse(successList, errorList); + System.out.println("response: " + response1.getErrors()); + String responseString = "{\"success\":[{\"index\":0,\"skyflow_id\":\"id\",\"data\":{\"test_column_1\":\"test_value_1\"}}],\"errors\":[{\"index\":1,\"error\":\"Bad Request\",\"code\":400}]}"; + Assert.assertEquals(1, response1.getSuccess().size()); + Assert.assertEquals(1, response1.getErrors().size()); + System.out.println("response: " + response1.toString()); + Assert.assertEquals(responseString, response1.toString()); + } catch (Exception e) { + Assert.fail(INVALID_EXCEPTION_THROWN); + } + } + +}