SK-2274 detokenize bulk async and sync method

common/src/main/java/com/skyflow/errors/ErrorMessage.java

@@ -150,7 +150,16 @@ public enum ErrorMessage {
     FailedToSaveProcessedFile("%s0 Validation error. Failed to save the processed file. Ensure the output directory is valid and writable."),
     InvalidAudioFileType("%s0 Validation error. The file type is not supported. Specify a valid file type mp3 or wav."),
     // Generic
-    ErrorOccurred("%s0 API error. Error occurred.")
+    ErrorOccurred("%s0 API error. Error occurred."),
+
+    DetokenizeRequestNull("%s0 Validation error. DetokenizeRequest object is null. Specify a valid DetokenizeRequest object."),
+
+    NullTokenGroupRedactions("%s0 Validation error. TokenGroupRedaction in the list is null. Specify a valid TokenGroupRedactions object."),
+
+    NullRedactionInTokenGroup("%s0 Validation error. Redaction in TokenGroupRedactions is null or empty. Specify a valid redaction."),
+
+    NullTokenGroupNameInTokenGroup("%s0 Validation error. TokenGroupName in TokenGroupRedactions is null or empty. Specify a valid tokenGroupName."),
+    ;
     ;
     private final String message;
 

common/src/main/java/com/skyflow/logs/ErrorLogs.java

@@ -66,6 +66,15 @@ public enum ErrorLogs {
     EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA("Invalid %s1 request. Token can not be null or empty in detokenize data at index %s2."),
     REDACTION_IS_REQUIRED("Invalid %s1 request. Redaction is required."),
     DETOKENIZE_REQUEST_REJECTED("Detokenize request resulted in failure."),
+    DETOKENIZE_REQUEST_NULL("Invalid %s1 request. Detokenize request can not be null."),
+
+    NULL_TOKEN_REDACTION_GROUP_OBJECT("Invalid %s1 request. Token Redaction group object can not be null or empty."),
+
+    NULL_REDACTION_IN_TOKEN_GROUP("Invalid %s1 request. Redaction can not be null in token redaction group"),
+
+    NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP("Invalid %s1 request. Token group name can not be null in token redaction group"),
+
+    EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP("Invalid %s1 request. Redaction can not be null or empty in token redaction group"),
     IDS_IS_REQUIRED("Invalid %s1 request. Ids are required."),
     EMPTY_IDS("Invalid %s1 request. Ids can not be empty."),
     EMPTY_OR_NULL_ID_IN_IDS("Invalid %s1 request. Id can not be null or empty in ids at index %s2."),

v3/src/main/java/com/skyflow/VaultClient.java

@@ -19,6 +19,7 @@
 import com.skyflow.utils.Utils;
 import com.skyflow.utils.logger.LogUtil;
 import com.skyflow.utils.validations.Validations;
+import com.skyflow.vault.data.DetokenizeRequest;
 import io.github.cdimascio.dotenv.Dotenv;
 import io.github.cdimascio.dotenv.DotenvException;
 import okhttp3.OkHttpClient;
@@ -155,4 +156,25 @@ protected InsertRequest getBulkInsertRequestBody(com.skyflow.vault.data.InsertRe
 
     }
 
+    protected com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest getDetokenizeRequestBody(DetokenizeRequest request) {
+        List<String> tokens = request.getTokens();
+        com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.Builder builder =
+                com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest.builder()
+                        .vaultId(this.vaultConfig.getVaultId())
+                        .tokens(tokens);
+        if (request.getTokenGroupRedactions() != null){
+            List<com.skyflow.generated.rest.types.TokenGroupRedactions> tokenGroupRedactionsList = new ArrayList<>();
+            for (com.skyflow.vault.data.TokenGroupRedactions tokenGroupRedactions : request.getTokenGroupRedactions()) {
+                com.skyflow.generated.rest.types.TokenGroupRedactions redactions =
+                        com.skyflow.generated.rest.types.TokenGroupRedactions.builder()
+                                .tokenGroupName(tokenGroupRedactions.getTokenGroupName())
+                                .redaction(tokenGroupRedactions.getRedaction())
+                                .build();
+                tokenGroupRedactionsList.add(redactions);
+            }
+
+            builder.tokenGroupRedactions(tokenGroupRedactionsList);
+        }
+        return builder.build();
+    }
 }

v3/src/main/java/com/skyflow/utils/Constants.java

@@ -9,6 +9,10 @@ public final class Constants extends BaseConstants {
     public static final Integer MAX_INSERT_BATCH_SIZE = 1000;
     public static final Integer INSERT_CONCURRENCY_LIMIT = 10;
     public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10;
-    public static final Integer DETOKENIZE_BATCH_SIZE = 100;
+    public static final Integer DETOKENIZE_BATCH_SIZE = 50;
     public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 10;
+
+    public static final Integer MAX_DETOKENIZE_BATCH_SIZE = 1000;
+    public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 10;
+
 }

v3/src/main/java/com/skyflow/utils/Utils.java

@@ -3,9 +3,12 @@
 import com.google.gson.JsonObject;
 import com.skyflow.enums.Env;
 import com.skyflow.generated.rest.core.ApiClientApiException;
+import com.skyflow.generated.rest.resources.recordservice.requests.DetokenizeRequest;
 import com.skyflow.generated.rest.types.InsertRecordData;
 import com.skyflow.generated.rest.types.InsertResponse;
 import com.skyflow.generated.rest.types.RecordResponseObject;
+import com.skyflow.generated.rest.types.TokenGroupRedactions;
+import com.skyflow.vault.data.DetokenizeResponse;
 import com.skyflow.vault.data.ErrorRecord;
 import com.skyflow.vault.data.Success;
 import com.skyflow.vault.data.Token;
@@ -36,6 +39,29 @@ public static List<List<InsertRecordData>> createBatches(List<InsertRecordData>
         return batches;
     }
 
+    public static List<DetokenizeRequest> createDetokenizeBatches(DetokenizeRequest request, int batchSize) {
+        List<DetokenizeRequest> detokenizeRequests = new ArrayList<>();
+        List<String> tokens = request.getTokens().get();
+
+        for (int i = 0; i < tokens.size(); i += batchSize) {
+            // Create a sublist for the current batch
+            List<String> batchTokens = tokens.subList(i, Math.min(i + batchSize, tokens.size()));
+            List<TokenGroupRedactions> tokenGroupRedactions = null;
+            if (request.getTokenGroupRedactions().isPresent() && !request.getTokenGroupRedactions().get().isEmpty() && i < request.getTokenGroupRedactions().get().size()) {
+                tokenGroupRedactions = request.getTokenGroupRedactions().get().subList(i, Math.min(i + batchSize, request.getTokenGroupRedactions().get().size()));            }
+            // Build a new DetokenizeRequest for the current batch
+            DetokenizeRequest batchRequest = DetokenizeRequest.builder()
+                    .vaultId(request.getVaultId())
+                    .tokens(new ArrayList<>(batchTokens))
+                    .tokenGroupRedactions(tokenGroupRedactions)
+                    .build();
+
+            detokenizeRequests.add(batchRequest);
+        }
+
+        return detokenizeRequests;
+    }
+
     public static ErrorRecord createErrorRecord(Map<String, Object> recordMap, int indexNumber) {
         ErrorRecord err = null;
         if (recordMap != null) {
@@ -48,7 +74,7 @@ public static ErrorRecord createErrorRecord(Map<String, Object> recordMap, int i
     }
 
     public static List<ErrorRecord> handleBatchException(
-            Throwable ex, List<InsertRecordData> batch, int batchNumber, List<List<InsertRecordData>> batches
+            Throwable ex, List<InsertRecordData> batch, int batchNumber
     ) {
         List<ErrorRecord> errorRecords = new ArrayList<>();
         Throwable cause = ex.getCause();
@@ -90,6 +116,72 @@ public static List<ErrorRecord> handleBatchException(
         return errorRecords;
     }
 
+    public static List<ErrorRecord> handleDetokenizeBatchException(
+            Throwable ex, DetokenizeRequest batch, int batchNumber, int batchSize
+    ) {
+        List<ErrorRecord> errorRecords = new ArrayList<>();
+        Throwable cause = ex.getCause();
+        if (cause instanceof ApiClientApiException) {
+            ApiClientApiException apiException = (ApiClientApiException) cause;
+            Map<String, Object> responseBody = (Map<String, Object>) apiException.body();
+            int indexNumber = batchNumber * batchSize;
+            if (responseBody != null) {
+                if (responseBody.containsKey("records")) {
+                    Object recordss = responseBody.get("records");
+                    if (recordss instanceof List) {
+                        List<?> recordsList = (List<?>) recordss;
+                        for (Object record : recordsList) {
+                            if (record instanceof Map) {
+                                Map<String, Object> recordMap = (Map<String, Object>) record;
+                                ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber);
+                                errorRecords.add(err);
+                                indexNumber++;
+                            }
+                        }
+                    }
+                } else if (responseBody.containsKey("error")) {
+                    Map<String, Object> recordMap = (Map<String, Object>) responseBody.get("error");
+                    for (int j = 0; j < batch.getTokens().get().size(); j++) {
+                        ErrorRecord err = Utils.createErrorRecord(recordMap, indexNumber);
+                        errorRecords.add(err);
+                        indexNumber++;
+                    }
+                }
+            }
+        } else {
+            int indexNumber = batchNumber * batchSize;
+            for (int j = 0; j < batch.getTokens().get().size(); j++) {
+                ErrorRecord err = new ErrorRecord(indexNumber, ex.getMessage(), 500);
+                errorRecords.add(err);
+                indexNumber++;
+            }
+        }
+        return errorRecords;
+    }
+
+    public static DetokenizeResponse formatDetokenizeResponse(com.skyflow.generated.rest.types.DetokenizeResponse response, int batch, int batchSize) {
+        if (response != null) {
+            List<com.skyflow.generated.rest.types.DetokenizeResponseObject> record = response.getResponse().get();
+            List<ErrorRecord> errorRecords = new ArrayList<>();
+            List<com.skyflow.vault.data.DetokenizeResponseObject> successRecords = new ArrayList<>();
+            int indexNumber = batch * batchSize;
+            int recordsSize = response.getResponse().get().size();
+            for (int index = 0; index < recordsSize; index++) {
+                if (record.get(index).getError().isPresent()) {
+                    ErrorRecord errorRecord = new ErrorRecord(indexNumber, record.get(index).getError().get(), record.get(index).getHttpCode().get());
+                    errorRecords.add(errorRecord);
+                } else {
+                    com.skyflow.vault.data.DetokenizeResponseObject success = new com.skyflow.vault.data.DetokenizeResponseObject(indexNumber, record.get(index).getToken().orElse(null), record.get(index).getValue().orElse(null), record.get(index).getTokenGroupName().orElse(null), record.get(index).getError().orElse(null), record.get(index).getMetadata().orElse(null));
+                    successRecords.add(success);
+                }
+                indexNumber++;
+            }
+            DetokenizeResponse formattedResponse = new DetokenizeResponse(successRecords, errorRecords);
+            return formattedResponse;
+        }
+        return null;
+    }
+
     public static com.skyflow.vault.data.InsertResponse formatResponse(InsertResponse response, int batch, int batchSize) {
         com.skyflow.vault.data.InsertResponse formattedResponse = null;
         List<Success> successRecords = new ArrayList<>();

v3/src/main/java/com/skyflow/utils/validations/Validations.java

@@ -7,7 +7,9 @@
 import com.skyflow.logs.ErrorLogs;
 import com.skyflow.utils.Utils;
 import com.skyflow.utils.logger.LogUtil;
+import com.skyflow.vault.data.DetokenizeRequest;
 import com.skyflow.vault.data.InsertRequest;
+import com.skyflow.vault.data.TokenGroupRedactions;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -72,4 +74,48 @@ public static void validateInsertRequest(InsertRequest insertRequest) throws Sky
         }
     }
 
+    public static void validateDetokenizeRequest(DetokenizeRequest request) throws SkyflowException {
+        if (request == null) {
+            LogUtil.printErrorLog(Utils.parameterizedString(
+                    ErrorLogs.DETOKENIZE_REQUEST_NULL.getLog(), InterfaceName.DETOKENIZE.getName()
+            ));
+            throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.DetokenizeRequestNull.getMessage());
+        }
+        ArrayList<String> tokens = request.getTokens();
+        if (tokens == null || tokens.isEmpty()) {
+            LogUtil.printErrorLog(Utils.parameterizedString(
+                    ErrorLogs.EMPTY_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
+            ));
+            throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyDetokenizeData.getMessage());
+        }
+        for (String token : tokens) {
+            if (token == null || token.trim().isEmpty()) {
+                LogUtil.printErrorLog(Utils.parameterizedString(
+                        ErrorLogs.EMPTY_OR_NULL_TOKEN_IN_DETOKENIZE_DATA.getLog(), InterfaceName.DETOKENIZE.getName()
+                ));
+                throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.EmptyTokenInDetokenizeData.getMessage());
+            }
+        }
+        ArrayList<TokenGroupRedactions> groupRedactions = request.getTokenGroupRedactions();
+        if (groupRedactions != null && !groupRedactions.isEmpty()) {
+            for (TokenGroupRedactions group : groupRedactions) {
+                if (group == null) {
+                    LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_REDACTION_GROUP_OBJECT.getLog(), InterfaceName.DETOKENIZE.getName()));
+                    throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupRedactions.getMessage());
+                }
+                String groupName = group.getTokenGroupName();
+                String redaction = group.getRedaction();
+                if (groupName == null || groupName.trim().isEmpty()) {
+                    LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.NULL_TOKEN_GROUP_NAME_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
+                    throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullTokenGroupNameInTokenGroup.getMessage());
+                }
+                if (redaction == null || redaction.trim().isEmpty()) {
+                    LogUtil.printErrorLog(Utils.parameterizedString(ErrorLogs.EMPTY_OR_NULL_REDACTION_IN_TOKEN_GROUP.getLog(), InterfaceName.DETOKENIZE.getName()));
+                    throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), ErrorMessage.NullRedactionInTokenGroup.getMessage());
+                }
+            }
+        }
+
+    }
+
 }