From c986e8817680a41e203b2152794bda2b549cedcf Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Mon, 1 Sep 2025 17:16:20 +0530 Subject: [PATCH 1/6] SK-971: add fern generated code with upload file v2 and latest detect --- skyflow/generated/rest/__init__.py | 6 +- skyflow/generated/rest/core/client_wrapper.py | 2 +- skyflow/generated/rest/files/client.py | 16 +- skyflow/generated/rest/files/raw_client.py | 16 +- ...deidentify_image_request_masking_method.py | 2 +- skyflow/generated/rest/guardrails/client.py | 12 +- skyflow/generated/rest/records/client.py | 141 +++++++++++ skyflow/generated/rest/records/raw_client.py | 239 ++++++++++++++++++ skyflow/generated/rest/types/__init__.py | 6 +- .../rest/types/deidentify_status_response.py | 4 +- .../deidentify_status_response_output_type.py | 2 +- .../deidentify_status_response_status.py | 2 +- skyflow/generated/rest/types/entity_type.py | 4 +- skyflow/generated/rest/types/error_string.py | 3 - .../rest/types/reidentify_file_response.py | 3 +- .../reidentify_file_response_output_type.py | 5 + .../types/reidentify_file_response_status.py | 2 +- .../rest/types/reidentify_string_response.py | 2 +- .../rest/types/upload_file_v_2_response.py | 34 +++ 19 files changed, 459 insertions(+), 42 deletions(-) delete mode 100644 skyflow/generated/rest/types/error_string.py create mode 100644 skyflow/generated/rest/types/reidentify_file_response_output_type.py create mode 100644 skyflow/generated/rest/types/upload_file_v_2_response.py diff --git a/skyflow/generated/rest/__init__.py b/skyflow/generated/rest/__init__.py index bad57c24..b8309d05 100644 --- a/skyflow/generated/rest/__init__.py +++ b/skyflow/generated/rest/__init__.py @@ -28,12 +28,12 @@ EntityTypes, ErrorResponse, ErrorResponseError, - ErrorString, GooglerpcStatus, ProtobufAny, RedactionEnumRedaction, ReidentifyFileResponse, ReidentifyFileResponseOutput, + ReidentifyFileResponseOutputType, ReidentifyFileResponseStatus, ReidentifyStringResponse, RequestActionType, @@ -46,6 +46,7 @@ Transformations, TransformationsShiftDates, TransformationsShiftDatesEntityTypesItem, + UploadFileV2Response, Uuid, V1AuditAfterOptions, V1AuditEventResponse, @@ -175,7 +176,6 @@ "EntityTypes", "ErrorResponse", "ErrorResponseError", - "ErrorString", "GooglerpcStatus", "InternalServerError", "NotFoundError", @@ -189,6 +189,7 @@ "ReidentifyFileRequestFormat", "ReidentifyFileResponse", "ReidentifyFileResponseOutput", + "ReidentifyFileResponseOutputType", "ReidentifyFileResponseStatus", "ReidentifyStringRequestFormat", "ReidentifyStringResponse", @@ -205,6 +206,7 @@ "TransformationsShiftDates", "TransformationsShiftDatesEntityTypesItem", "UnauthorizedError", + "UploadFileV2Response", "Uuid", "V1AuditAfterOptions", "V1AuditEventResponse", diff --git a/skyflow/generated/rest/core/client_wrapper.py b/skyflow/generated/rest/core/client_wrapper.py index a3210a7e..5179f373 100644 --- a/skyflow/generated/rest/core/client_wrapper.py +++ b/skyflow/generated/rest/core/client_wrapper.py @@ -24,7 +24,7 @@ def get_headers(self) -> typing.Dict[str, str]: headers: typing.Dict[str, str] = { "X-Fern-Language": "Python", "X-Fern-SDK-Name": "skyflow_vault", - "X-Fern-SDK-Version": "0.0.252", + "X-Fern-SDK-Version": "0.0.323", **(self.get_custom_headers() or {}), } headers["Authorization"] = f"Bearer {self._get_token()}" diff --git a/skyflow/generated/rest/files/client.py b/skyflow/generated/rest/files/client.py index 654789de..4d5d548b 100644 --- a/skyflow/generated/rest/files/client.py +++ b/skyflow/generated/rest/files/client.py @@ -200,8 +200,8 @@ def deidentify_pdf( vault_id: VaultId, file: DeidentifyPdfRequestFile, configuration_id: typing.Optional[ConfigurationId] = OMIT, - density: typing.Optional[int] = OMIT, - max_resolution: typing.Optional[int] = OMIT, + density: typing.Optional[float] = OMIT, + max_resolution: typing.Optional[float] = OMIT, entity_types: typing.Optional[EntityTypes] = OMIT, token_type: typing.Optional[TokenTypeWithoutVault] = OMIT, allow_regex: typing.Optional[AllowRegex] = OMIT, @@ -221,10 +221,10 @@ def deidentify_pdf( configuration_id : typing.Optional[ConfigurationId] - density : typing.Optional[int] + density : typing.Optional[float] Pixel density at which to process the PDF file. - max_resolution : typing.Optional[int] + max_resolution : typing.Optional[float] Max resolution at which to process the PDF file. entity_types : typing.Optional[EntityTypes] @@ -1020,8 +1020,8 @@ async def deidentify_pdf( vault_id: VaultId, file: DeidentifyPdfRequestFile, configuration_id: typing.Optional[ConfigurationId] = OMIT, - density: typing.Optional[int] = OMIT, - max_resolution: typing.Optional[int] = OMIT, + density: typing.Optional[float] = OMIT, + max_resolution: typing.Optional[float] = OMIT, entity_types: typing.Optional[EntityTypes] = OMIT, token_type: typing.Optional[TokenTypeWithoutVault] = OMIT, allow_regex: typing.Optional[AllowRegex] = OMIT, @@ -1041,10 +1041,10 @@ async def deidentify_pdf( configuration_id : typing.Optional[ConfigurationId] - density : typing.Optional[int] + density : typing.Optional[float] Pixel density at which to process the PDF file. - max_resolution : typing.Optional[int] + max_resolution : typing.Optional[float] Max resolution at which to process the PDF file. entity_types : typing.Optional[EntityTypes] diff --git a/skyflow/generated/rest/files/raw_client.py b/skyflow/generated/rest/files/raw_client.py index 5a67292f..c0e535ea 100644 --- a/skyflow/generated/rest/files/raw_client.py +++ b/skyflow/generated/rest/files/raw_client.py @@ -287,8 +287,8 @@ def deidentify_pdf( vault_id: VaultId, file: DeidentifyPdfRequestFile, configuration_id: typing.Optional[ConfigurationId] = OMIT, - density: typing.Optional[int] = OMIT, - max_resolution: typing.Optional[int] = OMIT, + density: typing.Optional[float] = OMIT, + max_resolution: typing.Optional[float] = OMIT, entity_types: typing.Optional[EntityTypes] = OMIT, token_type: typing.Optional[TokenTypeWithoutVault] = OMIT, allow_regex: typing.Optional[AllowRegex] = OMIT, @@ -308,10 +308,10 @@ def deidentify_pdf( configuration_id : typing.Optional[ConfigurationId] - density : typing.Optional[int] + density : typing.Optional[float] Pixel density at which to process the PDF file. - max_resolution : typing.Optional[int] + max_resolution : typing.Optional[float] Max resolution at which to process the PDF file. entity_types : typing.Optional[EntityTypes] @@ -1575,8 +1575,8 @@ async def deidentify_pdf( vault_id: VaultId, file: DeidentifyPdfRequestFile, configuration_id: typing.Optional[ConfigurationId] = OMIT, - density: typing.Optional[int] = OMIT, - max_resolution: typing.Optional[int] = OMIT, + density: typing.Optional[float] = OMIT, + max_resolution: typing.Optional[float] = OMIT, entity_types: typing.Optional[EntityTypes] = OMIT, token_type: typing.Optional[TokenTypeWithoutVault] = OMIT, allow_regex: typing.Optional[AllowRegex] = OMIT, @@ -1596,10 +1596,10 @@ async def deidentify_pdf( configuration_id : typing.Optional[ConfigurationId] - density : typing.Optional[int] + density : typing.Optional[float] Pixel density at which to process the PDF file. - max_resolution : typing.Optional[int] + max_resolution : typing.Optional[float] Max resolution at which to process the PDF file. entity_types : typing.Optional[EntityTypes] diff --git a/skyflow/generated/rest/files/types/deidentify_image_request_masking_method.py b/skyflow/generated/rest/files/types/deidentify_image_request_masking_method.py index d1ff8c83..bc0c338c 100644 --- a/skyflow/generated/rest/files/types/deidentify_image_request_masking_method.py +++ b/skyflow/generated/rest/files/types/deidentify_image_request_masking_method.py @@ -2,4 +2,4 @@ import typing -DeidentifyImageRequestMaskingMethod = typing.Union[typing.Literal["blackout", "blur"], typing.Any] +DeidentifyImageRequestMaskingMethod = typing.Union[typing.Literal["blackbox", "blur"], typing.Any] diff --git a/skyflow/generated/rest/guardrails/client.py b/skyflow/generated/rest/guardrails/client.py index 169f7de1..e7fe1e05 100644 --- a/skyflow/generated/rest/guardrails/client.py +++ b/skyflow/generated/rest/guardrails/client.py @@ -68,10 +68,8 @@ def check_guardrails( token="YOUR_TOKEN", ) client.guardrails.check_guardrails( - vault_id="VAULT_ID", - text="I love to play cricket.", - check_toxicity=True, - deny_topics=["sports"], + vault_id="vault_id", + text="text", ) """ _response = self._raw_client.check_guardrails( @@ -145,10 +143,8 @@ async def check_guardrails( async def main() -> None: await client.guardrails.check_guardrails( - vault_id="VAULT_ID", - text="I love to play cricket.", - check_toxicity=True, - deny_topics=["sports"], + vault_id="vault_id", + text="text", ) diff --git a/skyflow/generated/rest/records/client.py b/skyflow/generated/rest/records/client.py index 1f727bfc..cfe15a1c 100644 --- a/skyflow/generated/rest/records/client.py +++ b/skyflow/generated/rest/records/client.py @@ -5,6 +5,7 @@ from .. import core from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ..core.request_options import RequestOptions +from ..types.upload_file_v_2_response import UploadFileV2Response from ..types.v_1_batch_operation_response import V1BatchOperationResponse from ..types.v_1_batch_record import V1BatchRecord from ..types.v_1_bulk_delete_record_response import V1BulkDeleteRecordResponse @@ -700,6 +701,72 @@ def file_service_get_file_scan_status( ) return _response.data + def upload_file_v_2( + self, + vault_id: str, + *, + table_name: str, + column_name: str, + file: core.File, + skyflow_id: typing.Optional[str] = OMIT, + return_file_metadata: typing.Optional[bool] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> UploadFileV2Response: + """ + Uploads the specified file to a record. If an existing record isn't specified, creates a new record and uploads the file to that record. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to upload the file to. + + column_name : str + Name of the column to upload the file to. The column must have a `file` data type. + + file : core.File + See core.File for more documentation + + skyflow_id : typing.Optional[str] + Skyflow ID of the record to upload the file to. If `skyflowID` isn't specified, a new record will be created. + + return_file_metadata : typing.Optional[bool] + If `true`, returns metadata about the uploaded file. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + UploadFileV2Response + File uploaded successfully. + + Examples + -------- + from skyflow import Skyflow + + client = Skyflow( + token="YOUR_TOKEN", + ) + client.records.upload_file_v_2( + vault_id="d4410ea01d83473ca09a24c6b03096d4", + table_name="tableName", + column_name="columnName", + ) + """ + _response = self._raw_client.upload_file_v_2( + vault_id, + table_name=table_name, + column_name=column_name, + file=file, + skyflow_id=skyflow_id, + return_file_metadata=return_file_metadata, + request_options=request_options, + ) + return _response.data + class AsyncRecordsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -1455,3 +1522,77 @@ async def main() -> None: vault_id, table_name, id, column_name, request_options=request_options ) return _response.data + + async def upload_file_v_2( + self, + vault_id: str, + *, + table_name: str, + column_name: str, + file: core.File, + skyflow_id: typing.Optional[str] = OMIT, + return_file_metadata: typing.Optional[bool] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> UploadFileV2Response: + """ + Uploads the specified file to a record. If an existing record isn't specified, creates a new record and uploads the file to that record. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to upload the file to. + + column_name : str + Name of the column to upload the file to. The column must have a `file` data type. + + file : core.File + See core.File for more documentation + + skyflow_id : typing.Optional[str] + Skyflow ID of the record to upload the file to. If `skyflowID` isn't specified, a new record will be created. + + return_file_metadata : typing.Optional[bool] + If `true`, returns metadata about the uploaded file. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + UploadFileV2Response + File uploaded successfully. + + Examples + -------- + import asyncio + + from skyflow import AsyncSkyflow + + client = AsyncSkyflow( + token="YOUR_TOKEN", + ) + + + async def main() -> None: + await client.records.upload_file_v_2( + vault_id="d4410ea01d83473ca09a24c6b03096d4", + table_name="tableName", + column_name="columnName", + ) + + + asyncio.run(main()) + """ + _response = await self._raw_client.upload_file_v_2( + vault_id, + table_name=table_name, + column_name=column_name, + file=file, + skyflow_id=skyflow_id, + return_file_metadata=return_file_metadata, + request_options=request_options, + ) + return _response.data diff --git a/skyflow/generated/rest/records/raw_client.py b/skyflow/generated/rest/records/raw_client.py index e2bfdc92..b42e0bc9 100644 --- a/skyflow/generated/rest/records/raw_client.py +++ b/skyflow/generated/rest/records/raw_client.py @@ -11,7 +11,12 @@ from ..core.pydantic_utilities import parse_obj_as from ..core.request_options import RequestOptions from ..core.serialization import convert_and_respect_annotation_metadata +from ..errors.bad_request_error import BadRequestError +from ..errors.internal_server_error import InternalServerError from ..errors.not_found_error import NotFoundError +from ..errors.unauthorized_error import UnauthorizedError +from ..types.error_response import ErrorResponse +from ..types.upload_file_v_2_response import UploadFileV2Response from ..types.v_1_batch_operation_response import V1BatchOperationResponse from ..types.v_1_batch_record import V1BatchRecord from ..types.v_1_bulk_delete_record_response import V1BulkDeleteRecordResponse @@ -804,6 +809,123 @@ def file_service_get_file_scan_status( raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + def upload_file_v_2( + self, + vault_id: str, + *, + table_name: str, + column_name: str, + file: core.File, + skyflow_id: typing.Optional[str] = OMIT, + return_file_metadata: typing.Optional[bool] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> HttpResponse[UploadFileV2Response]: + """ + Uploads the specified file to a record. If an existing record isn't specified, creates a new record and uploads the file to that record. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to upload the file to. + + column_name : str + Name of the column to upload the file to. The column must have a `file` data type. + + file : core.File + See core.File for more documentation + + skyflow_id : typing.Optional[str] + Skyflow ID of the record to upload the file to. If `skyflowID` isn't specified, a new record will be created. + + return_file_metadata : typing.Optional[bool] + If `true`, returns metadata about the uploaded file. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + HttpResponse[UploadFileV2Response] + File uploaded successfully. + """ + _response = self._client_wrapper.httpx_client.request( + f"v2/vaults/{jsonable_encoder(vault_id)}/files/upload", + method="POST", + data={ + "tableName": table_name, + "columnName": column_name, + "skyflowID": skyflow_id, + "returnFileMetadata": return_file_metadata, + }, + files={ + "file": file, + }, + request_options=request_options, + omit=OMIT, + force_multipart=True, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + UploadFileV2Response, + parse_obj_as( + type_=UploadFileV2Response, # type: ignore + object_=_response.json(), + ), + ) + return HttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + class AsyncRawRecordsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -1577,3 +1699,120 @@ async def file_service_get_file_scan_status( except JSONDecodeError: raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) + + async def upload_file_v_2( + self, + vault_id: str, + *, + table_name: str, + column_name: str, + file: core.File, + skyflow_id: typing.Optional[str] = OMIT, + return_file_metadata: typing.Optional[bool] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> AsyncHttpResponse[UploadFileV2Response]: + """ + Uploads the specified file to a record. If an existing record isn't specified, creates a new record and uploads the file to that record. + + Parameters + ---------- + vault_id : str + ID of the vault. + + table_name : str + Name of the table to upload the file to. + + column_name : str + Name of the column to upload the file to. The column must have a `file` data type. + + file : core.File + See core.File for more documentation + + skyflow_id : typing.Optional[str] + Skyflow ID of the record to upload the file to. If `skyflowID` isn't specified, a new record will be created. + + return_file_metadata : typing.Optional[bool] + If `true`, returns metadata about the uploaded file. + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + AsyncHttpResponse[UploadFileV2Response] + File uploaded successfully. + """ + _response = await self._client_wrapper.httpx_client.request( + f"v2/vaults/{jsonable_encoder(vault_id)}/files/upload", + method="POST", + data={ + "tableName": table_name, + "columnName": column_name, + "skyflowID": skyflow_id, + "returnFileMetadata": return_file_metadata, + }, + files={ + "file": file, + }, + request_options=request_options, + omit=OMIT, + force_multipart=True, + ) + try: + if 200 <= _response.status_code < 300: + _data = typing.cast( + UploadFileV2Response, + parse_obj_as( + type_=UploadFileV2Response, # type: ignore + object_=_response.json(), + ), + ) + return AsyncHttpResponse(response=_response, data=_data) + if _response.status_code == 400: + raise BadRequestError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 401: + raise UnauthorizedError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 404: + raise NotFoundError( + headers=dict(_response.headers), + body=typing.cast( + typing.Optional[typing.Any], + parse_obj_as( + type_=typing.Optional[typing.Any], # type: ignore + object_=_response.json(), + ), + ), + ) + if _response.status_code == 500: + raise InternalServerError( + headers=dict(_response.headers), + body=typing.cast( + ErrorResponse, + parse_obj_as( + type_=ErrorResponse, # type: ignore + object_=_response.json(), + ), + ), + ) + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text) + raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json) diff --git a/skyflow/generated/rest/types/__init__.py b/skyflow/generated/rest/types/__init__.py index 5a48e4f4..92d826c9 100644 --- a/skyflow/generated/rest/types/__init__.py +++ b/skyflow/generated/rest/types/__init__.py @@ -27,12 +27,12 @@ from .entity_types import EntityTypes from .error_response import ErrorResponse from .error_response_error import ErrorResponseError -from .error_string import ErrorString from .googlerpc_status import GooglerpcStatus from .protobuf_any import ProtobufAny from .redaction_enum_redaction import RedactionEnumRedaction from .reidentify_file_response import ReidentifyFileResponse from .reidentify_file_response_output import ReidentifyFileResponseOutput +from .reidentify_file_response_output_type import ReidentifyFileResponseOutputType from .reidentify_file_response_status import ReidentifyFileResponseStatus from .reidentify_string_response import ReidentifyStringResponse from .request_action_type import RequestActionType @@ -45,6 +45,7 @@ from .transformations import Transformations from .transformations_shift_dates import TransformationsShiftDates from .transformations_shift_dates_entity_types_item import TransformationsShiftDatesEntityTypesItem +from .upload_file_v_2_response import UploadFileV2Response from .uuid_ import Uuid from .v_1_audit_after_options import V1AuditAfterOptions from .v_1_audit_event_response import V1AuditEventResponse @@ -105,12 +106,12 @@ "EntityTypes", "ErrorResponse", "ErrorResponseError", - "ErrorString", "GooglerpcStatus", "ProtobufAny", "RedactionEnumRedaction", "ReidentifyFileResponse", "ReidentifyFileResponseOutput", + "ReidentifyFileResponseOutputType", "ReidentifyFileResponseStatus", "ReidentifyStringResponse", "RequestActionType", @@ -123,6 +124,7 @@ "Transformations", "TransformationsShiftDates", "TransformationsShiftDatesEntityTypesItem", + "UploadFileV2Response", "Uuid", "V1AuditAfterOptions", "V1AuditEventResponse", diff --git a/skyflow/generated/rest/types/deidentify_status_response.py b/skyflow/generated/rest/types/deidentify_status_response.py index a276963c..712a85b2 100644 --- a/skyflow/generated/rest/types/deidentify_status_response.py +++ b/skyflow/generated/rest/types/deidentify_status_response.py @@ -24,7 +24,7 @@ class DeidentifyStatusResponse(UniversalBaseModel): How the input file was specified. """ - output_type: typing.Optional[DeidentifyStatusResponseOutputType] = pydantic.Field(default=None) + output_type: DeidentifyStatusResponseOutputType = pydantic.Field() """ How the output file is specified. """ @@ -49,7 +49,7 @@ class DeidentifyStatusResponse(UniversalBaseModel): Size of the processed text in kilobytes (KB). """ - duration: typing.Optional[int] = pydantic.Field(default=None) + duration: typing.Optional[float] = pydantic.Field(default=None) """ Duration of the processed audio in seconds. """ diff --git a/skyflow/generated/rest/types/deidentify_status_response_output_type.py b/skyflow/generated/rest/types/deidentify_status_response_output_type.py index 571801c1..051cc31a 100644 --- a/skyflow/generated/rest/types/deidentify_status_response_output_type.py +++ b/skyflow/generated/rest/types/deidentify_status_response_output_type.py @@ -2,4 +2,4 @@ import typing -DeidentifyStatusResponseOutputType = typing.Union[typing.Literal["base64", "efs_path"], typing.Any] +DeidentifyStatusResponseOutputType = typing.Union[typing.Literal["BASE64", "UNKNOWN"], typing.Any] diff --git a/skyflow/generated/rest/types/deidentify_status_response_status.py b/skyflow/generated/rest/types/deidentify_status_response_status.py index 40262092..9ec2931b 100644 --- a/skyflow/generated/rest/types/deidentify_status_response_status.py +++ b/skyflow/generated/rest/types/deidentify_status_response_status.py @@ -2,4 +2,4 @@ import typing -DeidentifyStatusResponseStatus = typing.Union[typing.Literal["failed", "in_progress", "success"], typing.Any] +DeidentifyStatusResponseStatus = typing.Union[typing.Literal["FAILED", "IN_PROGRESS", "SUCCESS", "UNKNOWN"], typing.Any] diff --git a/skyflow/generated/rest/types/entity_type.py b/skyflow/generated/rest/types/entity_type.py index 20195417..1a343410 100644 --- a/skyflow/generated/rest/types/entity_type.py +++ b/skyflow/generated/rest/types/entity_type.py @@ -15,8 +15,8 @@ "credit_card_expiration", "cvv", "date", - "day", "date_interval", + "day", "dob", "dose", "driver_license", @@ -58,10 +58,10 @@ "passport_number", "password", "phone_number", - "project", "physical_attribute", "political_affiliation", "product", + "project", "religion", "routing_number", "sexuality", diff --git a/skyflow/generated/rest/types/error_string.py b/skyflow/generated/rest/types/error_string.py deleted file mode 100644 index 068b4a84..00000000 --- a/skyflow/generated/rest/types/error_string.py +++ /dev/null @@ -1,3 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -ErrorString = str diff --git a/skyflow/generated/rest/types/reidentify_file_response.py b/skyflow/generated/rest/types/reidentify_file_response.py index c67b41ac..bd90fb49 100644 --- a/skyflow/generated/rest/types/reidentify_file_response.py +++ b/skyflow/generated/rest/types/reidentify_file_response.py @@ -5,6 +5,7 @@ import pydantic from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel from .reidentify_file_response_output import ReidentifyFileResponseOutput +from .reidentify_file_response_output_type import ReidentifyFileResponseOutputType from .reidentify_file_response_status import ReidentifyFileResponseStatus @@ -18,7 +19,7 @@ class ReidentifyFileResponse(UniversalBaseModel): Status of the re-identify operation. """ - output_type: typing.Literal["BASE64"] = pydantic.Field(default="BASE64") + output_type: ReidentifyFileResponseOutputType = pydantic.Field() """ Format of the output file. """ diff --git a/skyflow/generated/rest/types/reidentify_file_response_output_type.py b/skyflow/generated/rest/types/reidentify_file_response_output_type.py new file mode 100644 index 00000000..03048c85 --- /dev/null +++ b/skyflow/generated/rest/types/reidentify_file_response_output_type.py @@ -0,0 +1,5 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +ReidentifyFileResponseOutputType = typing.Union[typing.Literal["BASE64", "UNKNOWN"], typing.Any] diff --git a/skyflow/generated/rest/types/reidentify_file_response_status.py b/skyflow/generated/rest/types/reidentify_file_response_status.py index c640c3a6..8bdfa1e0 100644 --- a/skyflow/generated/rest/types/reidentify_file_response_status.py +++ b/skyflow/generated/rest/types/reidentify_file_response_status.py @@ -2,4 +2,4 @@ import typing -ReidentifyFileResponseStatus = typing.Union[typing.Literal["failed", "in_progress", "success"], typing.Any] +ReidentifyFileResponseStatus = typing.Union[typing.Literal["FAILED", "IN_PROGRESS", "SUCCESS", "UNKNOWN"], typing.Any] diff --git a/skyflow/generated/rest/types/reidentify_string_response.py b/skyflow/generated/rest/types/reidentify_string_response.py index 8284806b..cbb1b836 100644 --- a/skyflow/generated/rest/types/reidentify_string_response.py +++ b/skyflow/generated/rest/types/reidentify_string_response.py @@ -11,7 +11,7 @@ class ReidentifyStringResponse(UniversalBaseModel): Re-identify string response. """ - processed_text: typing.Optional[str] = pydantic.Field(default=None) + text: typing.Optional[str] = pydantic.Field(default=None) """ Re-identified text. """ diff --git a/skyflow/generated/rest/types/upload_file_v_2_response.py b/skyflow/generated/rest/types/upload_file_v_2_response.py new file mode 100644 index 00000000..f1bcc215 --- /dev/null +++ b/skyflow/generated/rest/types/upload_file_v_2_response.py @@ -0,0 +1,34 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing + +import pydantic +import typing_extensions +from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel +from ..core.serialization import FieldMetadata + + +class UploadFileV2Response(UniversalBaseModel): + """ + Response schema for uploading a file, optionally creating a new record. + """ + + skyflow_id: typing_extensions.Annotated[typing.Optional[str], FieldMetadata(alias="skyflowID")] = pydantic.Field( + default=None + ) + """ + Skyflow ID of the record the file was uploaded to. + """ + + file_metadata: typing_extensions.Annotated[ + typing.Optional[typing.Optional[typing.Any]], FieldMetadata(alias="fileMetadata") + ] = None + + if IS_PYDANTIC_V2: + model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2 + else: + + class Config: + frozen = True + smart_union = True + extra = pydantic.Extra.allow From 5fb9e05cecf452e4e5a8d28130da12b71cba3f57 Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Tue, 2 Sep 2025 13:06:37 +0530 Subject: [PATCH 2/6] SK-971: file upload support --- skyflow/utils/_skyflow_messages.py | 7 +++ skyflow/utils/validations/__init__.py | 1 + skyflow/utils/validations/_validations.py | 64 +++++++++++++++++++++ skyflow/vault/controller/_vault.py | 57 +++++++++++++++++- skyflow/vault/data/__init__.py | 4 +- skyflow/vault/data/_file_upload_request.py | 18 ++++++ skyflow/vault/data/_file_upload_response.py | 6 ++ 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 skyflow/vault/data/_file_upload_request.py create mode 100644 skyflow/vault/data/_file_upload_response.py diff --git a/skyflow/utils/_skyflow_messages.py b/skyflow/utils/_skyflow_messages.py index 460ca29e..b040eaf4 100644 --- a/skyflow/utils/_skyflow_messages.py +++ b/skyflow/utils/_skyflow_messages.py @@ -198,6 +198,7 @@ class Error(Enum): INVALID_FILE_OR_ENCODED_FILE= f"{error_prefix} . Error while decoding base64 and saving file" INVALID_FILE_TYPE = f"{error_prefix} Validation error. Invalid file type. Specify a valid file type." INVALID_FILE_NAME= f"{error_prefix} Validation error. Invalid file name. Specify a valid file name." + INVALID_FILE_PATH= f"{error_prefix} Validation error. Invalid file path. Specify a valid file path." INVALID_DEIDENTIFY_FILE_PATH= f"{error_prefix} Validation error. Invalid file path. Specify a valid file path." INVALID_BASE64_HEADER= f"{error_prefix} Validation error. Invalid base64 header. Specify a valid base64 header." INVALID_WAIT_TIME= f"{error_prefix} Validation error. Invalid wait time. Specify a valid wait time as number and should not be greater than 64 secs." @@ -271,6 +272,12 @@ class Info(Enum): TOKENIZE_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Tokenize request resolved." TOKENIZE_SUCCESS = f"{INFO}: [{error_prefix}] Data tokenized." + FILE_UPLOAD_TRIGGERED = f"{INFO}: [{error_prefix}] File upload method triggered." + VALIDATING_FILE_UPLOAD_REQUEST = f"{INFO}: [{error_prefix}] Validating file upload request." + FILE_UPLOAD_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] File upload request resolved." + FILE_UPLOAD_SUCCESS = f"{INFO}: [{error_prefix}] File uploaded successfully." + FILE_UPLOAD_REQUEST_REJECTED = f"{ERROR}: [{error_prefix}] File upload failed." + INVOKE_CONNECTION_TRIGGERED = f"{INFO}: [{error_prefix}] Invoke connection method triggered." VALIDATING_INVOKE_CONNECTION_REQUEST = f"{INFO}: [{error_prefix}] Validating invoke connection request." INVOKE_CONNECTION_REQUEST_RESOLVED = f"{INFO}: [{error_prefix}] Invoke connection request resolved." diff --git a/skyflow/utils/validations/__init__.py b/skyflow/utils/validations/__init__.py index b8ce13c8..2f0bc710 100644 --- a/skyflow/utils/validations/__init__.py +++ b/skyflow/utils/validations/__init__.py @@ -12,6 +12,7 @@ validate_update_request, validate_detokenize_request, validate_tokenize_request, + validate_file_upload_request, validate_invoke_connection_params, validate_deidentify_text_request, validate_reidentify_text_request, diff --git a/skyflow/utils/validations/_validations.py b/skyflow/utils/validations/_validations.py index bbca6e85..3bbcdf6f 100644 --- a/skyflow/utils/validations/_validations.py +++ b/skyflow/utils/validations/_validations.py @@ -692,6 +692,70 @@ def validate_tokenize_request(logger, request): log_error_log(SkyflowMessages.ErrorLogs.EMPTY_COLUMN_GROUP_IN_COLUMN_VALUES.value.format("TOKENIZE"), logger = logger) raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENIZE_PARAMETER_COLUMN_GROUP.value.format(i), invalid_input_error_code) + +def validate_file_upload_request(logger, request): + if request is None: + raise SkyflowError(SkyflowMessages.Error.INVALID_TABLE_VALUE.value, invalid_input_error_code) + + # Table + table = getattr(request, "table", None) + if table is None: + raise SkyflowError(SkyflowMessages.Error.INVALID_TABLE_VALUE.value, invalid_input_error_code) + elif table.strip() == "": + raise SkyflowError(SkyflowMessages.Error.EMPTY_TABLE_VALUE.value, invalid_input_error_code) + + # Skyflow ID + skyflow_id = getattr(request, "skyflow_id", None) + if skyflow_id is None: + raise SkyflowError(SkyflowMessages.Error.IDS_KEY_ERROR.value, invalid_input_error_code) + elif skyflow_id.strip() == "": + raise SkyflowError(SkyflowMessages.Error.EMPTY_SKYFLOW_ID.value.format("FILE_UPLOAD"), invalid_input_error_code) + + # Column Name + column_name = getattr(request, "column_name", None) + if column_name is None: + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) + elif column_name.strip() == "": + logger.error("Empty column name in FILE_UPLOAD") + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) + + # File-related attributes + file_path = getattr(request, "file_path", None) + base64_str = getattr(request, "base64", None) + file_object = getattr(request, "file_object", None) + file_name = getattr(request, "file_name", None) + + # Check file_path first if present + if not is_none_or_empty(file_path): + if not os.path.exists(file_path) or not os.path.isfile(file_path): + raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_PATH.value, invalid_input_error_code) + return + + # Check base64 if present + if not is_none_or_empty(base64_str): + if is_none_or_empty(file_name): + raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_NAME.value, invalid_input_error_code) + try: + import base64 + base64.b64decode(base64_str) + except Exception: + raise SkyflowError(SkyflowMessages.Error.INVALID_BASE64_STRING.value, invalid_input_error_code) + return + + # Check file_object if present + if file_object is not None: + try: + file_object.seek(0, 1) + return + except Exception: + raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_OBJECT.value, invalid_input_error_code) + + # If none of the above, raise missing file source error + raise SkyflowError(SkyflowMessages.Error.MISSING_FILE_SOURCE.value, invalid_input_error_code) + +def is_none_or_empty(value: str) -> bool: + return value is None or (isinstance(value, str) and value.strip() == "") + def validate_invoke_connection_params(logger, query_params, path_params): if not isinstance(path_params, dict): raise SkyflowError(SkyflowMessages.Error.INVALID_PATH_PARAMS.value, invalid_input_error_code) diff --git a/skyflow/vault/controller/_vault.py b/skyflow/vault/controller/_vault.py index 4602cf87..cea6d0b9 100644 --- a/skyflow/vault/controller/_vault.py +++ b/skyflow/vault/controller/_vault.py @@ -1,6 +1,10 @@ +import base64 import json +import os +from typing import BinaryIO, Optional, Tuple from skyflow.generated.rest import V1FieldRecords, V1BatchRecord, V1TokenizeRecordRequest, \ V1DetokenizeRecordRequest +from skyflow.generated.rest.core.file import File from skyflow.utils import SkyflowMessages, parse_insert_response, \ handle_exception, parse_update_record_response, parse_delete_response, parse_detokenize_response, \ parse_tokenize_response, parse_query_response, parse_get_response, encode_column_values, get_metrics @@ -8,8 +12,8 @@ from skyflow.utils.enums import RequestMethod from skyflow.utils.logger import log_info, log_error_log from skyflow.utils.validations import validate_insert_request, validate_delete_request, validate_query_request, \ - validate_get_request, validate_update_request, validate_detokenize_request, validate_tokenize_request -from skyflow.vault.data import InsertRequest, UpdateRequest, DeleteRequest, GetRequest, QueryRequest + validate_get_request, validate_update_request, validate_detokenize_request, validate_tokenize_request, validate_file_upload_request +from skyflow.vault.data import InsertRequest, UpdateRequest, DeleteRequest, GetRequest, QueryRequest, FileUploadRequest, FileUploadResponse from skyflow.vault.tokens import DetokenizeRequest, TokenizeRequest class Vault: @@ -62,7 +66,27 @@ def __build_insert_body(self, request: InsertRequest): else: records_list = self.__build_bulk_field_records(request.values, request.tokens) return records_list + + def __get_file_for_file_upload(self, request: FileUploadRequest) -> Optional[File]: + if request.file_path: + if not request.file_name: + request.file_name = os.path.basename(request.file_path) + with open(request.file_path, "rb") as f: + file_bytes = f.read() + return (request.file_name, file_bytes) + + elif request.base64 and request.file_name: + decoded_bytes = base64.b64decode(request.base64) + return (request.file_name, decoded_bytes) + + elif request.file_object is not None: + if hasattr(request.file_object, "name") and request.file_object.name: + file_name = os.path.basename(request.file_object.name) + return (file_name, request.file_object) + + return None + def __get_headers(self): headers = { SKY_META_DATA_HEADER: json.dumps(get_metrics()) @@ -244,4 +268,31 @@ def tokenize(self, request: TokenizeRequest): return tokenize_response except Exception as e: log_error_log(SkyflowMessages.ErrorLogs.TOKENIZE_REQUEST_REJECTED.value, logger = self.__vault_client.get_logger()) - handle_exception(e, self.__vault_client.get_logger()) \ No newline at end of file + handle_exception(e, self.__vault_client.get_logger()) + + def upload_file(self, request: FileUploadRequest): + log_info(SkyflowMessages.Info.FILE_UPLOAD_TRIGGERED.value, self.__vault_client.get_logger()) + log_info(SkyflowMessages.Info.VALIDATING_FILE_UPLOAD_REQUEST.value, self.__vault_client.get_logger()) + validate_file_upload_request(self.__vault_client.get_logger(), request) + self.__initialize() + file_upload_api = self.__vault_client.get_records_api().with_raw_response + try: + api_response = file_upload_api.upload_file_v_2( + self.__vault_client.get_vault_id(), + table_name=request.table, + column_name=request.column_name, + file=self.__get_file_for_file_upload(request), + skyflow_id=request.skyflow_id, + return_file_metadata= False, + request_options=self.__get_headers() + ) + log_info(SkyflowMessages.Info.FILE_UPLOAD_REQUEST_RESOLVED.value, self.__vault_client.get_logger()) + log_info(SkyflowMessages.Info.FILE_UPLOAD_SUCCESS.value, self.__vault_client.get_logger()) + upload_response = FileUploadResponse( + skyflow_id=api_response.data.skyflow_id, + errors=None + ) + return upload_response + except Exception as e: + log_error_log(SkyflowMessages.ErrorLogs.FILE_UPLOAD_REQUEST_REJECTED.value, logger = self.__vault_client.get_logger()) + handle_exception(e, self.__vault_client.get_logger()) diff --git a/skyflow/vault/data/__init__.py b/skyflow/vault/data/__init__.py index b43b23cf..d711f4f6 100644 --- a/skyflow/vault/data/__init__.py +++ b/skyflow/vault/data/__init__.py @@ -8,4 +8,6 @@ from ._update_response import UpdateResponse from ._upload_file_request import UploadFileRequest from ._query_request import QueryRequest -from ._query_response import QueryResponse \ No newline at end of file +from ._query_response import QueryResponse +from ._file_upload_request import FileUploadRequest +from ._file_upload_response import FileUploadResponse \ No newline at end of file diff --git a/skyflow/vault/data/_file_upload_request.py b/skyflow/vault/data/_file_upload_request.py new file mode 100644 index 00000000..d1bd4a44 --- /dev/null +++ b/skyflow/vault/data/_file_upload_request.py @@ -0,0 +1,18 @@ +from typing import BinaryIO + +class FileUploadRequest: + def __init__(self, + table: str, + skyflow_id: str, + column_name: str, + file_path: str= None, + base64: str= None, + file_object: BinaryIO= None, + file_name: str= None): + self.table = table + self.skyflow_id = skyflow_id + self.column_name = column_name + self.file_path = file_path + self.base64 = base64 + self.file_object = file_object + self.file_name = file_name diff --git a/skyflow/vault/data/_file_upload_response.py b/skyflow/vault/data/_file_upload_response.py new file mode 100644 index 00000000..18218f08 --- /dev/null +++ b/skyflow/vault/data/_file_upload_response.py @@ -0,0 +1,6 @@ +class FileUploadResponse: + def __init__(self, + skyflow_id, + errors): + self.skyflow_id = skyflow_id + self.errors = errors From fff5decbbb51c61de0bc966a70deea4fc95975ae Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Tue, 2 Sep 2025 13:56:03 +0530 Subject: [PATCH 3/6] SK-971: unit tests --- tests/vault/controller/test__vault.py | 315 +++++++++++++++++++++++++- 1 file changed, 312 insertions(+), 3 deletions(-) diff --git a/tests/vault/controller/test__vault.py b/tests/vault/controller/test__vault.py index 0c8a7743..d18d8a06 100644 --- a/tests/vault/controller/test__vault.py +++ b/tests/vault/controller/test__vault.py @@ -1,12 +1,14 @@ import unittest -from unittest.mock import Mock, patch +from unittest.mock import Mock, patch, mock_open as mock_open_func, mock_open from skyflow.generated.rest import V1BatchRecord, V1FieldRecords, V1DetokenizeRecordRequest, V1TokenizeRecordRequest +from skyflow.utils._skyflow_messages import SkyflowMessages from skyflow.utils.enums import RedactionType, TokenMode from skyflow.vault.controller import Vault from skyflow.vault.data import InsertRequest, InsertResponse, UpdateResponse, UpdateRequest, DeleteResponse, \ - DeleteRequest, GetRequest, GetResponse, QueryRequest, QueryResponse + DeleteRequest, GetRequest, GetResponse, QueryRequest, QueryResponse, FileUploadRequest from skyflow.vault.tokens import DetokenizeRequest, DetokenizeResponse, TokenizeResponse, TokenizeRequest - +from skyflow.error import SkyflowError +from skyflow.utils.validations import validate_file_upload_request VAULT_ID = "test_vault_id" TABLE_NAME = "test_table" @@ -598,3 +600,310 @@ def test_tokenize_handles_generic_error(self, mock_validate): self.vault.tokenize(request) tokens_api.record_service_tokenize.assert_called_once() + + @patch("skyflow.vault.controller._vault.validate_file_upload_request") + def test_upload_file_with_file_path_successful(self, mock_validate): + """Test upload_file functionality using file path.""" + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_path="/path/to/test.txt", + ) + + # Mock file open + mocked_open = mock_open_func(read_data=b"test file content") + + # Mock API response + mock_api_response = Mock() + mock_api_response.data = Mock(skyflow_id="123") + + records_api = self.vault_client.get_records_api.return_value + records_api.with_raw_response.upload_file_v_2.return_value = mock_api_response + + with patch('builtins.open', mocked_open): + result = self.vault.upload_file(request) + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + mocked_open.assert_called_once_with("/path/to/test.txt", "rb") + self.assertEqual(result.skyflow_id, "123") + self.assertIsNone(result.errors) + + @patch("skyflow.vault.controller._vault.validate_file_upload_request") + def test_upload_file_with_base64_successful(self, mock_validate): + """Test upload_file functionality using base64 content.""" + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + base64="dGVzdCBmaWxlIGNvbnRlbnQ=", # "test file content" in base64 + file_name="test.txt" + ) + + # Mock API response + mock_api_response = Mock() + mock_api_response.data = Mock(skyflow_id="123") + + records_api = self.vault_client.get_records_api.return_value + records_api.with_raw_response.upload_file_v_2.return_value = mock_api_response + + # Call upload_file + result = self.vault.upload_file(request) + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.assertEqual(result.skyflow_id, "123") + self.assertIsNone(result.errors) + + @patch("skyflow.vault.controller._vault.validate_file_upload_request") + def test_upload_file_with_file_object_successful(self, mock_validate): + """Test upload_file functionality using file object.""" + + # Create mock file object + mock_file = Mock() + mock_file.name = "test.txt" + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_object=mock_file + ) + + # Mock API response + mock_api_response = Mock() + mock_api_response.data = Mock(skyflow_id="123") + + records_api = self.vault_client.get_records_api.return_value + records_api.with_raw_response.upload_file_v_2.return_value = mock_api_response + + # Call upload_file + result = self.vault.upload_file(request) + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + self.assertEqual(result.skyflow_id, "123") + self.assertIsNone(result.errors) + + @patch("skyflow.vault.controller._vault.validate_file_upload_request") + def test_upload_file_handles_api_error(self, mock_validate): + """Test upload_file error handling for API errors.""" + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_path="/path/to/test.txt" + ) + + # Mock API error + records_api = self.vault_client.get_records_api.return_value + records_api.with_raw_response.upload_file_v_2.side_effect = Exception("Upload failed") + + # Assert that the exception is propagated + with patch('builtins.open', mock_open(read_data=b"test content")): + with self.assertRaises(Exception): + self.vault.upload_file(request) + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + + @patch("skyflow.vault.controller._vault.validate_file_upload_request") + def test_upload_file_with_missing_file_source(self, mock_validate): + """Test upload_file with no file source specified.""" + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123" + ) + + mock_validate.side_effect = SkyflowError(SkyflowMessages.Error.MISSING_FILE_SOURCE.value, + SkyflowMessages.ErrorCodes.INVALID_INPUT.value) + + with self.assertRaises(SkyflowError) as error: + self.vault.upload_file(request) + + self.assertEqual(error.exception.message, SkyflowMessages.Error.MISSING_FILE_SOURCE.value) + mock_validate.assert_called_once_with(self.vault_client.get_logger(), request) + +class TestFileUploadValidation(unittest.TestCase): + def setUp(self): + self.logger = Mock() + + def test_validate_missing_table(self): + """Test validation fails when table is missing""" + request = FileUploadRequest( + column_name="file_column", + skyflow_id="123", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_TABLE_VALUE.value) + + def test_validate_empty_table(self): + """Test validation fails when table is empty""" + request = FileUploadRequest( + table="", + column_name="file_column", + skyflow_id="123", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.EMPTY_TABLE_VALUE.value) + + def test_validate_missing_skyflow_id(self): + """Test validation fails when skyflow_id is missing""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.IDS_KEY_ERROR.value) + + def test_validate_empty_skyflow_id(self): + """Test validation fails when skyflow_id is empty""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, + SkyflowMessages.Error.EMPTY_SKYFLOW_ID.value.format("FILE_UPLOAD")) + + def test_validate_missing_column_name(self): + """Test validation fails when column_name is missing""" + request = FileUploadRequest( + table="test_table", + skyflow_id="123", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, + SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(None))) + + def test_validate_empty_column_name(self): + """Test validation fails when column_name is empty""" + request = FileUploadRequest( + table="test_table", + column_name="", + skyflow_id="123", + file_path="/path/to/file.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, + SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(""))) + + @patch('os.path.exists') + @patch('os.path.isfile') + def test_validate_file_path_success(self, mock_isfile, mock_exists): + """Test validation succeeds with valid file path""" + mock_exists.return_value = True + mock_isfile.return_value = True + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_path="/path/to/file.txt" + ) + validate_file_upload_request(self.logger, request) + mock_exists.assert_called_once_with("/path/to/file.txt") + mock_isfile.assert_called_once_with("/path/to/file.txt") + + @patch('os.path.exists') + def test_validate_invalid_file_path(self, mock_exists): + """Test validation fails with invalid file path""" + mock_exists.return_value = False + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_path="/invalid/path.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_FILE_PATH.value) + + def test_validate_base64_success(self): + """Test validation succeeds with valid base64""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + base64="dGVzdCBmaWxlIGNvbnRlbnQ=", + file_name="test.txt" + ) + validate_file_upload_request(self.logger, request) + + def test_validate_base64_without_filename(self): + """Test validation fails with base64 but no filename""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + base64="dGVzdCBmaWxlIGNvbnRlbnQ=" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_FILE_NAME.value) + + def test_validate_invalid_base64(self): + """Test validation fails with invalid base64""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + base64="invalid-base64", + file_name="test.txt" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_BASE64_STRING.value) + + def test_validate_file_object_success(self): + """Test validation succeeds with valid file object""" + mock_file = Mock() + mock_file.seek = Mock() # Add seek method + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_object=mock_file + ) + validate_file_upload_request(self.logger, request) + + def test_validate_invalid_file_object(self): + """Test validation fails with invalid file object""" + mock_file = Mock() + mock_file.seek = Mock(side_effect=Exception()) # Make seek fail + + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123", + file_object=mock_file + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_FILE_OBJECT.value) + + def test_validate_missing_file_source(self): + """Test validation fails when no file source is provided""" + request = FileUploadRequest( + table="test_table", + column_name="file_column", + skyflow_id="123" + ) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.MISSING_FILE_SOURCE.value) + with self.assertRaises(SkyflowError) as error: + validate_file_upload_request(self.logger, request) + self.assertEqual(error.exception.message, SkyflowMessages.Error.MISSING_FILE_SOURCE.value) From e26502d78733b7e0b89d444116a24a74ceea7a81 Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Tue, 2 Sep 2025 15:39:30 +0530 Subject: [PATCH 4/6] SK-971: fix unit tests --- skyflow/utils/_skyflow_messages.py | 2 ++ skyflow/utils/validations/_validations.py | 4 +-- tests/vault/controller/test__vault.py | 41 +++-------------------- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/skyflow/utils/_skyflow_messages.py b/skyflow/utils/_skyflow_messages.py index b040eaf4..cb693966 100644 --- a/skyflow/utils/_skyflow_messages.py +++ b/skyflow/utils/_skyflow_messages.py @@ -100,6 +100,8 @@ class Error(Enum): INVALID_TABLE_VALUE = f"{error_prefix} Validation error. Invalid type of table. Specify table as a string" EMPTY_RECORD_IDS_IN_DELETE = f"{error_prefix} Validation error. 'record ids' array can't be empty. Specify one or more record ids." BULK_DELETE_FAILURE = f"{error_prefix} Delete operation failed." + EMPTY_SKYFLOW_ID= f"{error_prefix} Validation error. Skyflow id can't be empty." + INVALID_FILE_COLUMN_NAME= f"{error_prefix} Validation error. 'column_name' can't be empty." INVALID_QUERY_TYPE = f"{error_prefix} Validation error. Query parameter is of type {{}}. Specify as a string." EMPTY_QUERY = f"{error_prefix} Validation error. Query parameter can't be empty. Specify as a string." diff --git a/skyflow/utils/validations/_validations.py b/skyflow/utils/validations/_validations.py index 3bbcdf6f..cb75d1b6 100644 --- a/skyflow/utils/validations/_validations.py +++ b/skyflow/utils/validations/_validations.py @@ -714,10 +714,10 @@ def validate_file_upload_request(logger, request): # Column Name column_name = getattr(request, "column_name", None) if column_name is None: - raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) + raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) elif column_name.strip() == "": logger.error("Empty column name in FILE_UPLOAD") - raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) + raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code) # File-related attributes file_path = getattr(request, "file_path", None) diff --git a/tests/vault/controller/test__vault.py b/tests/vault/controller/test__vault.py index d18d8a06..8d1d1ab0 100644 --- a/tests/vault/controller/test__vault.py +++ b/tests/vault/controller/test__vault.py @@ -726,18 +726,7 @@ class TestFileUploadValidation(unittest.TestCase): def setUp(self): self.logger = Mock() - def test_validate_missing_table(self): - """Test validation fails when table is missing""" - request = FileUploadRequest( - column_name="file_column", - skyflow_id="123", - file_path="/path/to/file.txt" - ) - with self.assertRaises(SkyflowError) as error: - validate_file_upload_request(self.logger, request) - self.assertEqual(error.exception.message, SkyflowMessages.Error.INVALID_TABLE_VALUE.value) - - def test_validate_empty_table(self): + def test_validate_invalid_table(self): """Test validation fails when table is empty""" request = FileUploadRequest( table="", @@ -749,17 +738,6 @@ def test_validate_empty_table(self): validate_file_upload_request(self.logger, request) self.assertEqual(error.exception.message, SkyflowMessages.Error.EMPTY_TABLE_VALUE.value) - def test_validate_missing_skyflow_id(self): - """Test validation fails when skyflow_id is missing""" - request = FileUploadRequest( - table="test_table", - column_name="file_column", - file_path="/path/to/file.txt" - ) - with self.assertRaises(SkyflowError) as error: - validate_file_upload_request(self.logger, request) - self.assertEqual(error.exception.message, SkyflowMessages.Error.IDS_KEY_ERROR.value) - def test_validate_empty_skyflow_id(self): """Test validation fails when skyflow_id is empty""" request = FileUploadRequest( @@ -773,30 +751,19 @@ def test_validate_empty_skyflow_id(self): self.assertEqual(error.exception.message, SkyflowMessages.Error.EMPTY_SKYFLOW_ID.value.format("FILE_UPLOAD")) - def test_validate_missing_column_name(self): + def test_validate_invalid_column_name(self): """Test validation fails when column_name is missing""" request = FileUploadRequest( table="test_table", skyflow_id="123", + column_name="", file_path="/path/to/file.txt" ) with self.assertRaises(SkyflowError) as error: validate_file_upload_request(self.logger, request) self.assertEqual(error.exception.message, - SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(None))) + SkyflowMessages.Error.INVALID_FILE_COLUMN_NAME.value.format("FILE_UPLOAD")) - def test_validate_empty_column_name(self): - """Test validation fails when column_name is empty""" - request = FileUploadRequest( - table="test_table", - column_name="", - skyflow_id="123", - file_path="/path/to/file.txt" - ) - with self.assertRaises(SkyflowError) as error: - validate_file_upload_request(self.logger, request) - self.assertEqual(error.exception.message, - SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(""))) @patch('os.path.exists') @patch('os.path.isfile') From 05335e7548af1a3bbc408f4147c556cee1cc1284 Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Wed, 10 Sep 2025 14:52:44 +0530 Subject: [PATCH 5/6] SK-971: fix the message and import --- skyflow/utils/_skyflow_messages.py | 2 +- skyflow/utils/validations/_validations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/skyflow/utils/_skyflow_messages.py b/skyflow/utils/_skyflow_messages.py index cb693966..8401aeb6 100644 --- a/skyflow/utils/_skyflow_messages.py +++ b/skyflow/utils/_skyflow_messages.py @@ -100,7 +100,7 @@ class Error(Enum): INVALID_TABLE_VALUE = f"{error_prefix} Validation error. Invalid type of table. Specify table as a string" EMPTY_RECORD_IDS_IN_DELETE = f"{error_prefix} Validation error. 'record ids' array can't be empty. Specify one or more record ids." BULK_DELETE_FAILURE = f"{error_prefix} Delete operation failed." - EMPTY_SKYFLOW_ID= f"{error_prefix} Validation error. Skyflow id can't be empty." + EMPTY_SKYFLOW_ID= f"{error_prefix} Validation error. skyflow_id can't be empty." INVALID_FILE_COLUMN_NAME= f"{error_prefix} Validation error. 'column_name' can't be empty." INVALID_QUERY_TYPE = f"{error_prefix} Validation error. Query parameter is of type {{}}. Specify as a string." diff --git a/skyflow/utils/validations/_validations.py b/skyflow/utils/validations/_validations.py index cb75d1b6..f88388ad 100644 --- a/skyflow/utils/validations/_validations.py +++ b/skyflow/utils/validations/_validations.py @@ -1,3 +1,4 @@ +import base64 import json import os from skyflow.generated.rest import TokenType @@ -736,7 +737,6 @@ def validate_file_upload_request(logger, request): if is_none_or_empty(file_name): raise SkyflowError(SkyflowMessages.Error.INVALID_FILE_NAME.value, invalid_input_error_code) try: - import base64 base64.b64decode(base64_str) except Exception: raise SkyflowError(SkyflowMessages.Error.INVALID_BASE64_STRING.value, invalid_input_error_code) From ae41cabd3e25b3694e78ee184504166a7dc89858 Mon Sep 17 00:00:00 2001 From: Raushan Kumar Gupta Date: Wed, 10 Sep 2025 15:38:07 +0530 Subject: [PATCH 6/6] SK-971: remove unused import --- skyflow/vault/controller/_vault.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skyflow/vault/controller/_vault.py b/skyflow/vault/controller/_vault.py index cea6d0b9..fe921293 100644 --- a/skyflow/vault/controller/_vault.py +++ b/skyflow/vault/controller/_vault.py @@ -1,7 +1,7 @@ import base64 import json import os -from typing import BinaryIO, Optional, Tuple +from typing import Optional from skyflow.generated.rest import V1FieldRecords, V1BatchRecord, V1TokenizeRecordRequest, \ V1DetokenizeRecordRequest from skyflow.generated.rest.core.file import File