diff --git a/flowfile_core/flowfile_core/flowfile/database_connection_manager/db_connections.py b/flowfile_core/flowfile_core/flowfile/database_connection_manager/db_connections.py index 00472e1ea..0115c4ef5 100644 --- a/flowfile_core/flowfile_core/flowfile/database_connection_manager/db_connections.py +++ b/flowfile_core/flowfile_core/flowfile/database_connection_manager/db_connections.py @@ -213,6 +213,14 @@ def store_cloud_connection( ).id else: azure_account_key_ref_id = None + if connection.azure_sas_token is not None: + azure_sas_token_ref_id = store_secret( + db, + SecretInput(name=connection.connection_name + "azure_sas_token", value=connection.azure_sas_token), + user_id, + ).id + else: + azure_sas_token_ref_id = None db_cloud_connection = DBCloudStorageConnection( connection_name=connection.connection_name, @@ -231,6 +239,7 @@ def store_cloud_connection( azure_client_id=connection.azure_client_id, azure_account_key_id=azure_account_key_ref_id, azure_client_secret_id=azure_client_secret_ref_id, + azure_sas_token_id=azure_sas_token_ref_id, # Common fields endpoint_url=connection.endpoint_url, verify_ssl=connection.verify_ssl, @@ -290,6 +299,12 @@ def get_cloud_connection_schema(db: Session, connection_name: str, user_id: int) if secret_record: azure_client_secret = decrypt_secret(secret_record.encrypted_value) + azure_sas_token = None + if db_connection.azure_sas_token_id: + secret_record = db.query(Secret).filter(Secret.id == db_connection.azure_sas_token_id).first() + if secret_record: + azure_sas_token = decrypt_secret(secret_record.encrypted_value) + # Construct the full Pydantic model return FullCloudStorageConnection( connection_name=db_connection.connection_name, @@ -305,6 +320,7 @@ def get_cloud_connection_schema(db: Session, connection_name: str, user_id: int) azure_tenant_id=db_connection.azure_tenant_id, azure_client_id=db_connection.azure_client_id, azure_client_secret=azure_client_secret, + azure_sas_token=azure_sas_token, endpoint_url=db_connection.endpoint_url, verify_ssl=db_connection.verify_ssl, ) diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py index 26b4bddec..0d47cd1b0 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/cloud_storage_reader.py @@ -2,6 +2,7 @@ from typing import Any, Literal import boto3 +from azure.storage.blob import BlobServiceClient, ContainerClient from botocore.exceptions import ClientError from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection @@ -258,3 +259,125 @@ def ensure_path_has_wildcard_pattern(resource_path: str, file_format: Literal["c if not resource_path.endswith(f"*.{file_format}"): resource_path = resource_path.rstrip("/") + f"/**/*.{file_format}" return resource_path + + +def get_first_file_from_adls_dir(source: str, storage_options: dict[str, Any] = None) -> str: + """ + Get the first file from an Azure ADLS directory path. + + Parameters + ---------- + source : str + ADLS path with wildcards (e.g., 'az://container/prefix/**/*.parquet' or + 'abfs://container@account.dfs.core.windows.net/prefix/*.parquet') + + storage_options: dict + Storage options containing authentication details + + Returns + ------- + str + ADLS URI of the first file found + + Raises + ------ + ValueError + If source path is invalid or no files found + Exception + If ADLS access fails + """ + if not (source.startswith("az://") or source.startswith("abfs://")): + raise ValueError("Source must be a valid ADLS URI starting with 'az://' or 'abfs://'") + + container_name, prefix, account_name = _parse_adls_path(source) + file_extension = _get_file_extension(source) + base_prefix = _remove_wildcards_from_prefix(prefix) + + blob_service_client = _create_adls_client(account_name, storage_options) + container_client = blob_service_client.get_container_client(container_name) + + # List blobs with the given prefix + first_file = _get_first_adls_file(container_client, base_prefix, file_extension) + + # Return first file URI in az:// format + return f"az://{container_name}/{first_file['name']}" + + +def _parse_adls_path(source: str) -> tuple[str, str, str]: + """ + Parse ADLS URI into container name, prefix, and account name. + + Supports both formats: + - az://container/prefix/path + - abfs://container@account.dfs.core.windows.net/prefix/path + """ + if source.startswith("az://"): + # Format: az://container/prefix/path + path_parts = source[5:].split("/", 1) # Remove 'az://' + container_name = path_parts[0] + prefix = path_parts[1] if len(path_parts) > 1 else "" + account_name = None # Will be extracted from storage_options + elif source.startswith("abfs://"): + # Format: abfs://container@account.dfs.core.windows.net/prefix/path + path_parts = source[7:].split("/", 1) # Remove 'abfs://' + container_and_account = path_parts[0] + prefix = path_parts[1] if len(path_parts) > 1 else "" + + # Extract container and account + if "@" in container_and_account: + container_name, account_part = container_and_account.split("@", 1) + account_name = account_part.split(".")[0] # Extract account name from FQDN + else: + container_name = container_and_account + account_name = None + else: + raise ValueError("Invalid ADLS URI format") + + return container_name, prefix, account_name + + +def _create_adls_client(account_name: str | None, storage_options: dict[str, Any] | None) -> BlobServiceClient: + """Create Azure Blob Service Client with optional credentials.""" + if storage_options is None: + raise ValueError("Storage options are required for ADLS connections") + + # Extract account name from storage options if not provided + if account_name is None: + account_name = storage_options.get("account_name") + + if not account_name: + raise ValueError("Azure account name is required") + + account_url = f"https://{account_name}.blob.core.windows.net" + + # Authenticate based on available credentials + if "account_key" in storage_options: + return BlobServiceClient(account_url=account_url, credential=storage_options["account_key"]) + elif "sas_token" in storage_options: + return BlobServiceClient(account_url=account_url, credential=storage_options["sas_token"]) + elif "client_id" in storage_options and "client_secret" in storage_options and "tenant_id" in storage_options: + # Service principal authentication + from azure.identity import ClientSecretCredential + + credential = ClientSecretCredential( + tenant_id=storage_options["tenant_id"], + client_id=storage_options["client_id"], + client_secret=storage_options["client_secret"], + ) + return BlobServiceClient(account_url=account_url, credential=credential) + else: + raise ValueError("No valid authentication method found in storage options") + + +def _get_first_adls_file(container_client: ContainerClient, base_prefix: str, file_extension: str) -> dict[str, Any]: + """List all files in ADLS container with given prefix and return the first match.""" + try: + blob_list = container_client.list_blobs(name_starts_with=base_prefix) + + for blob in blob_list: + if blob.name.endswith(f".{file_extension}"): + return {"name": blob.name} + + raise ValueError(f"No {file_extension} files found in container with prefix {base_prefix}") + except Exception as e: + raise ValueError(f"Failed to list files in ADLS container: {e}") diff --git a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py index b2bfdb734..663949d58 100644 --- a/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py +++ b/flowfile_core/flowfile_core/flowfile/flow_data_engine/flow_data_engine.py @@ -29,6 +29,7 @@ from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import ( CloudStorageReader, ensure_path_has_wildcard_pattern, + get_first_file_from_adls_dir, get_first_file_from_s3_dir, ) from flowfile_core.flowfile.flow_data_engine.create import funcs as create_funcs @@ -589,7 +590,14 @@ def _get_schema_from_first_file_in_dir( """Infers the schema by scanning the first file in a cloud directory.""" try: scan_func = getattr(pl, "scan_" + file_format) - first_file_ref = get_first_file_from_s3_dir(source, storage_options=storage_options) + # Determine storage type and use appropriate function + if source.startswith("s3://"): + first_file_ref = get_first_file_from_s3_dir(source, storage_options=storage_options) + elif source.startswith("az://") or source.startswith("abfs://"): + first_file_ref = get_first_file_from_adls_dir(source, storage_options=storage_options) + else: + raise ValueError(f"Unsupported cloud storage URI format: {source}") + return convert_stats_to_column_info( FlowDataEngine._create_schema_stats_from_pl_schema( scan_func(first_file_ref, storage_options=storage_options).collect_schema() diff --git a/flowfile_core/flowfile_core/schemas/cloud_storage_schemas.py b/flowfile_core/flowfile_core/schemas/cloud_storage_schemas.py index c3dd85d8c..114f5e0a4 100644 --- a/flowfile_core/flowfile_core/schemas/cloud_storage_schemas.py +++ b/flowfile_core/flowfile_core/schemas/cloud_storage_schemas.py @@ -58,6 +58,7 @@ class FullCloudStorageConnectionWorkerInterface(AuthSettingsInput): azure_tenant_id: str | None = None azure_client_id: str | None = None azure_client_secret: str | None = None + azure_sas_token: str | None = None # Common endpoint_url: str | None = None @@ -81,6 +82,7 @@ class FullCloudStorageConnection(AuthSettingsInput): azure_tenant_id: str | None = None azure_client_id: str | None = None azure_client_secret: SecretStr | None = None + azure_sas_token: SecretStr | None = None # Common endpoint_url: str | None = None @@ -111,6 +113,7 @@ def get_worker_interface(self, user_id: int) -> "FullCloudStorageConnectionWorke azure_account_key=encrypt_for_worker(self.azure_account_key, user_id), azure_client_id=self.azure_client_id, azure_client_secret=encrypt_for_worker(self.azure_client_secret, user_id), + azure_sas_token=encrypt_for_worker(self.azure_sas_token, user_id), endpoint_url=self.endpoint_url, verify_ssl=self.verify_ssl, ) diff --git a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionSettings.vue b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionSettings.vue index e19b03a0a..ea7cafd41 100644 --- a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionSettings.vue +++ b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionSettings.vue @@ -19,7 +19,7 @@ @@ -200,6 +200,29 @@ + + +
+ +
+ + +
+
@@ -299,6 +322,7 @@ watch( const showAwsSecret = ref(false); const showAzureKey = ref(false); const showAzureSecret = ref(false); +const showAzureSasToken = ref(false); // Computed property for available auth methods based on storage type const availableAuthMethods = computed(() => { @@ -353,6 +377,8 @@ const isValid = computed(() => { !!connection.value.azureClientId && !!connection.value.azureClientSecret ); + } else if (connection.value.authMethod === "sas_token") { + return !!connection.value.azureSasToken; } } diff --git a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionTypes.ts b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionTypes.ts index d8e86e1de..ee987b868 100644 --- a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionTypes.ts +++ b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/CloudConnectionTypes.ts @@ -39,6 +39,7 @@ export interface PythonFullCloudStorageConnection extends PythonAuthSettingsInpu azure_tenant_id?: string; azure_client_id?: string; azure_client_secret?: string; + azure_sas_token?: string; // Common endpoint_url?: string; @@ -59,6 +60,7 @@ export interface FullCloudStorageConnection extends AuthSettingsInput { azureTenantId?: string; azureClientId?: string; azureClientSecret?: string; + azureSasToken?: string; // Common endpointUrl?: string; diff --git a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/api.ts b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/api.ts index 66a339a3e..96fb22ddd 100644 --- a/flowfile_frontend/src/renderer/app/views/CloudConnectionView/api.ts +++ b/flowfile_frontend/src/renderer/app/views/CloudConnectionView/api.ts @@ -34,6 +34,7 @@ const toPythonFormat = ( azure_tenant_id: connection.azureTenantId, azure_client_id: connection.azureClientId, azure_client_secret: connection.azureClientSecret, + azure_sas_token: connection.azureSasToken, // Common endpoint_url: connection.endpointUrl, diff --git a/flowfile_worker/flowfile_worker/external_sources/s3_source/models.py b/flowfile_worker/flowfile_worker/external_sources/s3_source/models.py index 93af2780b..7610f9341 100644 --- a/flowfile_worker/flowfile_worker/external_sources/s3_source/models.py +++ b/flowfile_worker/flowfile_worker/external_sources/s3_source/models.py @@ -70,6 +70,7 @@ class FullCloudStorageConnection(BaseModel): azure_tenant_id: str | None = None azure_client_id: str | None = None azure_client_secret: SecretStr | None = None + azure_sas_token: SecretStr | None = None # Common endpoint_url: str | None = None @@ -84,6 +85,10 @@ def get_storage_options(self) -> dict[str, Any]: """ if self.storage_type == "s3": return self._get_s3_storage_options() + elif self.storage_type == "adls": + return self._get_adls_storage_options() + else: + raise ValueError(f"Unsupported storage type: {self.storage_type}") def _get_s3_storage_options(self) -> dict[str, Any]: """Build S3-specific storage options.""" @@ -127,6 +132,44 @@ def _get_s3_storage_options(self) -> dict[str, Any]: return storage_options + def _get_adls_storage_options(self) -> dict[str, Any]: + """Build Azure ADLS-specific storage options.""" + auth_method = self.auth_method + print(f"Building ADLS storage options for auth_method: '{auth_method}'") + + storage_options = {} + + # Common options + if self.azure_account_name: + storage_options["account_name"] = self.azure_account_name + + if auth_method == "access_key": + # Account key authentication + if self.azure_account_key: + storage_options["account_key"] = decrypt_secret( + self.azure_account_key.get_secret_value() + ).get_secret_value() + + elif auth_method == "service_principal": + # Service principal authentication + if self.azure_tenant_id: + storage_options["tenant_id"] = self.azure_tenant_id + if self.azure_client_id: + storage_options["client_id"] = self.azure_client_id + if self.azure_client_secret: + storage_options["client_secret"] = decrypt_secret( + self.azure_client_secret.get_secret_value() + ).get_secret_value() + + elif auth_method == "sas_token": + # SAS token authentication + if self.azure_sas_token: + storage_options["sas_token"] = decrypt_secret( + self.azure_sas_token.get_secret_value() + ).get_secret_value() + + return storage_options + class WriteSettings(BaseModel): """Settings for writing to cloud storage""" diff --git a/poetry.lock b/poetry.lock index 0b5fe8365..8ec30c98c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,28 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. + +[[package]] +name = "adlfs" +version = "2025.8.0" +description = "Access Azure Datalake Gen1 with fsspec and dask" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "adlfs-2025.8.0-py3-none-any.whl", hash = "sha256:c12a9203a31485cad19599bf2ad30d8efcf4cf0fd2446c60fcdc18604fae07b1"}, + {file = "adlfs-2025.8.0.tar.gz", hash = "sha256:6fe5857866c18990f632598273e6a8b15edc6baf8614272ede25624057b83e64"}, +] + +[package.dependencies] +aiohttp = ">=3.7.0" +azure-core = ">=1.28.0,<2.0.0" +azure-datalake-store = ">=0.0.53,<0.1" +azure-identity = "*" +azure-storage-blob = ">=12.17.0" +fsspec = ">=2023.12.0" + +[package.extras] +docs = ["furo", "myst-parser", "numpydoc", "sphinx"] +tests = ["arrow", "dask[dataframe]", "docker", "pytest", "pytest-mock"] [[package]] name = "aiobotocore" @@ -6,6 +30,7 @@ version = "2.23.1" description = "Async client for aws services using botocore and aiohttp" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiobotocore-2.23.1-py3-none-any.whl", hash = "sha256:d81c54d2eae2406ea9a473fea518fed580cf37bc4fc51ce43ba81546e5305114"}, {file = "aiobotocore-2.23.1.tar.gz", hash = "sha256:a59f2a78629b97d52f10936b79c73de64e481a8c44a62c1871f088df6c1afc4f"}, @@ -31,6 +56,7 @@ version = "24.1.0" description = "File support for asyncio." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5"}, {file = "aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c"}, @@ -42,6 +68,7 @@ version = "2.6.1" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8"}, {file = "aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558"}, @@ -53,6 +80,7 @@ version = "3.13.3" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7"}, {file = "aiohttp-3.13.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821"}, @@ -187,7 +215,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli (>=1.2)", "aiodns (>=3.3.0)", "backports.zstd", "brotlicffi (>=1.2)"] +speedups = ["Brotli (>=1.2) ; platform_python_implementation == \"CPython\"", "aiodns (>=3.3.0)", "backports.zstd ; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", "brotlicffi (>=1.2) ; platform_python_implementation != \"CPython\""] [[package]] name = "aioitertools" @@ -195,6 +223,7 @@ version = "0.13.0" description = "itertools and builtins for AsyncIO and mixed iterables" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be"}, {file = "aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c"}, @@ -206,6 +235,7 @@ version = "1.4.0" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, @@ -221,6 +251,7 @@ version = "0.17.5" description = "Python graph (network) package" optional = false python-versions = "*" +groups = ["build"] files = [ {file = "altgraph-0.17.5-py2.py3-none-any.whl", hash = "sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597"}, {file = "altgraph-0.17.5.tar.gz", hash = "sha256:c87b395dd12fabde9c99573a9749d67da8d29ef9de0125c7f536699b4a9bc9e7"}, @@ -232,6 +263,7 @@ version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, @@ -243,6 +275,7 @@ version = "4.12.1" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c"}, {file = "anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703"}, @@ -254,7 +287,7 @@ idna = ">=2.8" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -trio = ["trio (>=0.31.0)", "trio (>=0.32.0)"] +trio = ["trio (>=0.31.0) ; python_version < \"3.10\"", "trio (>=0.32.0) ; python_version >= \"3.10\""] [[package]] name = "arro3-core" @@ -262,6 +295,7 @@ version = "0.6.5" description = "" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae"}, {file = "arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a"}, @@ -331,7 +365,7 @@ files = [ ] [package.dependencies] -typing-extensions = {version = "*", markers = "python_full_version < \"3.12\""} +typing-extensions = {version = "*", markers = "python_full_version < \"3.12.0\""} [[package]] name = "async-timeout" @@ -339,6 +373,8 @@ version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -350,24 +386,103 @@ version = "25.4.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373"}, {file = "attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11"}, ] +[[package]] +name = "azure-core" +version = "1.37.0" +description = "Microsoft Azure Core Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_core-1.37.0-py3-none-any.whl", hash = "sha256:b3abe2c59e7d6bb18b38c275a5029ff80f98990e7c90a5e646249a56630fcc19"}, + {file = "azure_core-1.37.0.tar.gz", hash = "sha256:7064f2c11e4b97f340e8e8c6d923b822978be3016e46b7bc4aa4b337cfb48aee"}, +] + +[package.dependencies] +requests = ">=2.21.0" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["aiohttp (>=3.0)"] +tracing = ["opentelemetry-api (>=1.26,<2.0)"] + +[[package]] +name = "azure-datalake-store" +version = "0.0.53" +description = "Azure Data Lake Store Filesystem Client Library for Python" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "azure-datalake-store-0.0.53.tar.gz", hash = "sha256:05b6de62ee3f2a0a6e6941e6933b792b800c3e7f6ffce2fc324bc19875757393"}, + {file = "azure_datalake_store-0.0.53-py2.py3-none-any.whl", hash = "sha256:a30c902a6e360aa47d7f69f086b426729784e71c536f330b691647a51dc42b2b"}, +] + +[package.dependencies] +cffi = "*" +msal = ">=1.16.0,<2" +requests = ">=2.20.0" + +[[package]] +name = "azure-identity" +version = "1.25.1" +description = "Microsoft Azure Identity Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_identity-1.25.1-py3-none-any.whl", hash = "sha256:e9edd720af03dff020223cd269fa3a61e8f345ea75443858273bcb44844ab651"}, + {file = "azure_identity-1.25.1.tar.gz", hash = "sha256:87ca8328883de6036443e1c37b40e8dc8fb74898240f61071e09d2e369361456"}, +] + +[package.dependencies] +azure-core = ">=1.31.0" +cryptography = ">=2.5" +msal = ">=1.30.0" +msal-extensions = ">=1.2.0" +typing-extensions = ">=4.0.0" + +[[package]] +name = "azure-storage-blob" +version = "12.28.0" +description = "Microsoft Azure Blob Storage Client Library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "azure_storage_blob-12.28.0-py3-none-any.whl", hash = "sha256:00fb1db28bf6a7b7ecaa48e3b1d5c83bfadacc5a678b77826081304bd87d6461"}, + {file = "azure_storage_blob-12.28.0.tar.gz", hash = "sha256:e7d98ea108258d29aa0efbfd591b2e2075fa1722a2fae8699f0b3c9de11eff41"}, +] + +[package.dependencies] +azure-core = ">=1.30.0" +cryptography = ">=2.1.4" +isodate = ">=0.6.1" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["azure-core[aio] (>=1.30.0)"] + [[package]] name = "babel" version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] [[package]] name = "backrefs" @@ -375,6 +490,7 @@ version = "6.1" description = "A wrapper around re and regex that adds additional back references." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "backrefs-6.1-py310-none-any.whl", hash = "sha256:2a2ccb96302337ce61ee4717ceacfbf26ba4efb1d55af86564b8bbaeda39cac1"}, {file = "backrefs-6.1-py311-none-any.whl", hash = "sha256:e82bba3875ee4430f4de4b6db19429a27275d95a5f3773c57e9e18abc23fd2b7"}, @@ -394,6 +510,7 @@ version = "4.3.0" description = "Modern password hashing for your software and your servers" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "bcrypt-4.3.0-cp313-cp313t-macosx_10_12_universal2.whl", hash = "sha256:f01e060f14b6b57bbb72fc5b4a83ac21c443c9a2ee708e04a10e9192f90a6281"}, {file = "bcrypt-4.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5eeac541cefd0bb887a371ef73c62c3cd78535e4887b310626036a7c0a817bb"}, @@ -458,6 +575,7 @@ version = "1.38.46" description = "The AWS SDK for Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "boto3-1.38.46-py3-none-any.whl", hash = "sha256:9c8e88a32a6465e5905308708cff5b17547117f06982908bdfdb0108b4a65079"}, {file = "boto3-1.38.46.tar.gz", hash = "sha256:d1ca2b53138afd0341e1962bd52be6071ab7a63c5b4f89228c5ef8942c40c852"}, @@ -477,6 +595,7 @@ version = "1.38.46" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "botocore-1.38.46-py3-none-any.whl", hash = "sha256:89ca782ffbf2e8769ca9c89234cfa5ca577f1987d07d913ee3c68c4776b1eb5b"}, {file = "botocore-1.38.46.tar.gz", hash = "sha256:8798e5a418c27cf93195b077153644aea44cb171fcd56edc1ecebaa1e49e226e"}, @@ -496,6 +615,7 @@ version = "5.5.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, @@ -507,6 +627,7 @@ version = "2026.1.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c"}, {file = "certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120"}, @@ -518,6 +639,7 @@ version = "2.0.0" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -614,6 +736,7 @@ version = "3.4.4" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, @@ -736,6 +859,7 @@ version = "8.3.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -750,6 +874,7 @@ version = "3.1.2" description = "Pickler class to extend the standard pickle.Pickler functionality" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a"}, {file = "cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414"}, @@ -761,10 +886,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\""} [[package]] name = "connectorx" @@ -772,6 +899,7 @@ version = "0.4.4" description = "" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "connectorx-0.4.4-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:58a3f9e05a42066dd8e2f1da6c9bbd18662817eba7968eb88031f4b3365831e0"}, {file = "connectorx-0.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcaeb9ccede6bb63e6cc85b8004aef08a2ec4cd128df18390da46bdb2daa378b"}, @@ -801,6 +929,7 @@ version = "45.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.7" +groups = ["main"] files = [ {file = "cryptography-45.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:3be4f21c6245930688bd9e162829480de027f8bf962ede33d4f8ba7d67a00cee"}, {file = "cryptography-45.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:67285f8a611b0ebc0857ced2081e30302909f571a46bfa7a3cc0ad303fe015c6"}, @@ -845,10 +974,10 @@ files = [ cffi = {version = ">=1.14", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] +docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs ; python_full_version >= \"3.8.0\"", "sphinx-rtd-theme (>=3.0.0) ; python_full_version >= \"3.8.0\""] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_full_version >= \"3.8.0\""] +pep8test = ["check-sdist ; python_full_version >= \"3.8.0\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==45.0.7)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -860,6 +989,7 @@ version = "0.9.0" description = "Async database support for Python." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "databases-0.9.0-py3-none-any.whl", hash = "sha256:9ee657c9863b34f8d3a06c06eafbe1bda68af2a434b56996312edf1f1c0b6297"}, {file = "databases-0.9.0.tar.gz", hash = "sha256:d2f259677609bf187737644c95fa41701072e995dfeb8d2882f335795c5b61b0"}, @@ -884,6 +1014,7 @@ version = "1.3.0" description = "Native Delta Lake Python binding based on delta-rs with Pandas integration" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "deltalake-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7381ed01f5968c4befdb6bc8706d99b39f33722074d7ee3be08e488aca3f1681"}, {file = "deltalake-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:66a96bc03f57b5868817d185f4d1148f43162b0bec8bc748a2eb8f75a8a5fca8"}, @@ -907,6 +1038,7 @@ version = "1.3.1" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] files = [ {file = "deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f"}, {file = "deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223"}, @@ -916,7 +1048,7 @@ files = [ wrapt = ">=1.10,<3" [package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] [[package]] name = "docker" @@ -924,6 +1056,7 @@ version = "7.1.0" description = "A Python library for the Docker Engine API." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, @@ -946,6 +1079,7 @@ version = "0.19.1" description = "ECDSA cryptographic signature library (pure python)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.6" +groups = ["main"] files = [ {file = "ecdsa-0.19.1-py2.py3-none-any.whl", hash = "sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3"}, {file = "ecdsa-0.19.1.tar.gz", hash = "sha256:478cba7b62555866fcb3bb3fe985e06decbdb68ef55713c4e5ab98c57d508e61"}, @@ -964,6 +1098,7 @@ version = "2.0.0" description = "An implementation of lxml.xmlfile for the standard library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, @@ -975,6 +1110,8 @@ version = "1.3.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, @@ -992,6 +1129,7 @@ version = "23.1.0" description = "Faker is a Python package that generates fake data for you." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "Faker-23.1.0-py3-none-any.whl", hash = "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2"}, {file = "Faker-23.1.0.tar.gz", hash = "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b"}, @@ -1006,6 +1144,7 @@ version = "0.115.14" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fastapi-0.115.14-py3-none-any.whl", hash = "sha256:6c0c8bf9420bd58f565e585036d971872472b4f7d3f6c73b698e10cffdefb3ca"}, {file = "fastapi-0.115.14.tar.gz", hash = "sha256:b1de15cdc1c499a4da47914db35d0e4ef8f1ce62b624e94e0e5824421df99739"}, @@ -1026,6 +1165,7 @@ version = "0.12.1" description = "A fast excel file reader for Python, written in Rust" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fastexcel-0.12.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7c4c959a49329a53540ed86d4f92cd4af2d774b08c47ad190841e4daf042758a"}, {file = "fastexcel-0.12.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e889290b2d1437e57c4fb500657f0d2908de6aa5b291475b93e3b5d7e0cea938"}, @@ -1048,6 +1188,7 @@ version = "1.8.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011"}, {file = "frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565"}, @@ -1187,6 +1328,7 @@ version = "2025.12.0" description = "File-system specification" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b"}, {file = "fsspec-2025.12.0.tar.gz", hash = "sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973"}, @@ -1217,7 +1359,7 @@ smb = ["smbprotocol"] ssh = ["paramiko"] test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] -test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard ; python_version < \"3.14\""] tqdm = ["tqdm"] [[package]] @@ -1226,6 +1368,7 @@ version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, @@ -1243,6 +1386,8 @@ version = "3.3.0" description = "Lightweight in-process concurrent programming" optional = false python-versions = ">=3.10" +groups = ["main", "dev"] +markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\"" files = [ {file = "greenlet-3.3.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f8496d434d5cb2dce025773ba5597f71f5410ae499d5dd9533e0653258cdb3d"}, {file = "greenlet-3.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b96dc7eef78fd404e022e165ec55327f935b9b52ff355b067eb4a0267fc1cffb"}, @@ -1304,6 +1449,7 @@ version = "1.15.0" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "griffe-1.15.0-py3-none-any.whl", hash = "sha256:6f6762661949411031f5fcda9593f586e6ce8340f0ba88921a0f2ef7a81eb9a3"}, {file = "griffe-1.15.0.tar.gz", hash = "sha256:7726e3afd6f298fbc3696e67958803e7ac843c1cfe59734b6251a40cdbfb5eea"}, @@ -1321,6 +1467,7 @@ version = "1.1.8" description = "Griffe extension for Pydantic." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "griffe_pydantic-1.1.8-py3-none-any.whl", hash = "sha256:22212c94216e03bf43d30ff3bc79cd53fb973ae2fe81d8b7510242232a1e6764"}, {file = "griffe_pydantic-1.1.8.tar.gz", hash = "sha256:72cde69c74c70f3dc0385a7a5243c736cd6bf6fcf8a41cae497383defe107041"}, @@ -1335,6 +1482,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -1346,6 +1494,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -1367,6 +1516,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1379,7 +1529,7 @@ httpcore = "==1.*" idna = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] @@ -1391,6 +1541,7 @@ version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -1405,17 +1556,31 @@ version = "2.3.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, ] +[[package]] +name = "isodate" +version = "0.7.2" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15"}, + {file = "isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6"}, +] + [[package]] name = "jinja2" version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -1433,6 +1598,7 @@ version = "1.0.1" description = "JSON Matching Expressions" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -1444,6 +1610,7 @@ version = "3.4.1" description = "A robust implementation of concurrent.futures.ProcessPoolExecutor" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "loky-3.4.1-py3-none-any.whl", hash = "sha256:7132da80d1a057b5917ff32c7867b65ed164aae84c259a1dbc44375791280c87"}, {file = "loky-3.4.1.tar.gz", hash = "sha256:66db350de68c301299c882ace3b8f06ba5c4cb2c45f8fcffd498160ce8280753"}, @@ -1458,6 +1625,8 @@ version = "1.16.4" description = "Mach-O header analysis and editing" optional = false python-versions = "*" +groups = ["build"] +markers = "sys_platform == \"darwin\"" files = [ {file = "macholib-1.16.4-py2.py3-none-any.whl", hash = "sha256:da1a3fa8266e30f0ce7e97c6a54eefaae8edd1e5f86f3eb8b95457cae90265ea"}, {file = "macholib-1.16.4.tar.gz", hash = "sha256:f408c93ab2e995cd2c46e34fe328b130404be143469e41bc366c807448979362"}, @@ -1472,6 +1641,7 @@ version = "3.10" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c"}, {file = "markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e"}, @@ -1487,6 +1657,7 @@ version = "4.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, @@ -1510,6 +1681,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -1608,6 +1780,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1619,6 +1792,7 @@ version = "1.3.4" description = "A deep merge function for 🐍." optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, @@ -1630,6 +1804,7 @@ version = "0.4.7" description = "Expand standard functools to methods" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "methodtools-0.4.7-py2.py3-none-any.whl", hash = "sha256:5e188c780b236adc12e75b5f078c5afb419ef99eb648569fc6d7071f053a1f11"}, {file = "methodtools-0.4.7.tar.gz", hash = "sha256:e213439dd64cfe60213f7015da6efe5dd4003fd89376db3baa09fe13ec2bb0ba"}, @@ -1640,7 +1815,7 @@ wirerope = ">=0.4.7" [package.extras] doc = ["sphinx"] -test = ["functools32 (>=3.2.3-2)", "pytest (>=4.6.7)", "pytest-cov (>=2.6.1)"] +test = ["functools32 (>=3.2.3-2) ; python_version < \"3\"", "pytest (>=4.6.7)", "pytest-cov (>=2.6.1)"] [[package]] name = "mkdocs" @@ -1648,6 +1823,7 @@ version = "1.6.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, @@ -1670,7 +1846,7 @@ watchdog = ">=2.0" [package.extras] i18n = ["babel (>=2.9.0)"] -min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.4)", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform_system == \"Windows\"", "ghp-import (==1.0)", "importlib-metadata (==4.4) ; python_version < \"3.10\"", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] [[package]] name = "mkdocs-autorefs" @@ -1678,6 +1854,7 @@ version = "1.4.3" description = "Automatically link across pages in MkDocs." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "mkdocs_autorefs-1.4.3-py3-none-any.whl", hash = "sha256:469d85eb3114801d08e9cc55d102b3ba65917a869b893403b8987b601cf55dc9"}, {file = "mkdocs_autorefs-1.4.3.tar.gz", hash = "sha256:beee715b254455c4aa93b6ef3c67579c399ca092259cc41b7d9342573ff1fc75"}, @@ -1694,6 +1871,7 @@ version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"}, {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"}, @@ -1710,6 +1888,7 @@ version = "9.7.1" description = "Documentation that simply works" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_material-9.7.1-py3-none-any.whl", hash = "sha256:3f6100937d7d731f87f1e3e3b021c97f7239666b9ba1151ab476cabb96c60d5c"}, {file = "mkdocs_material-9.7.1.tar.gz", hash = "sha256:89601b8f2c3e6c6ee0a918cc3566cb201d40bf37c3cd3c2067e26fadb8cce2b8"}, @@ -1739,6 +1918,7 @@ version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, @@ -1750,6 +1930,7 @@ version = "0.30.1" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "mkdocstrings-0.30.1-py3-none-any.whl", hash = "sha256:41bd71f284ca4d44a668816193e4025c950b002252081e387433656ae9a70a82"}, {file = "mkdocstrings-0.30.1.tar.gz", hash = "sha256:84a007aae9b707fb0aebfc9da23db4b26fc9ab562eb56e335e9ec480cb19744f"}, @@ -1774,6 +1955,7 @@ version = "1.19.0" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "mkdocstrings_python-1.19.0-py3-none-any.whl", hash = "sha256:395c1032af8f005234170575cc0c5d4d20980846623b623b35594281be4a3059"}, {file = "mkdocstrings_python-1.19.0.tar.gz", hash = "sha256:917aac66cf121243c11db5b89f66b0ded6c53ec0de5318ff5e22424eb2f2e57c"}, @@ -1791,6 +1973,7 @@ version = "5.2.0" description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "mmh3-5.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:81c504ad11c588c8629536b032940f2a359dda3b6cbfd4ad8f74cb24dcd1b0bc"}, {file = "mmh3-5.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0b898cecff57442724a0f52bf42c2de42de63083a91008fb452887e372f9c328"}, @@ -1923,12 +2106,51 @@ plot = ["matplotlib (==3.10.3)", "pandas (==2.3.1)"] test = ["pytest (==8.4.1)", "pytest-sugar (==1.0.0)"] type = ["mypy (==1.17.0)"] +[[package]] +name = "msal" +version = "1.34.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "msal-1.34.0-py3-none-any.whl", hash = "sha256:f669b1644e4950115da7a176441b0e13ec2975c29528d8b9e81316023676d6e1"}, + {file = "msal-1.34.0.tar.gz", hash = "sha256:76ba83b716ea5a6d75b0279c0ac353a0e05b820ca1f6682c0eb7f45190c43c2f"}, +] + +[package.dependencies] +cryptography = ">=2.5,<49" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.14,<0.19) ; python_version >= \"3.6\" and platform_system == \"Windows\"", "pymsalruntime (>=0.17,<0.19) ; python_version >= \"3.8\" and platform_system == \"Darwin\"", "pymsalruntime (>=0.18,<0.19) ; python_version >= \"3.8\" and platform_system == \"Linux\""] + +[[package]] +name = "msal-extensions" +version = "1.3.1" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "msal_extensions-1.3.1-py3-none-any.whl", hash = "sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca"}, + {file = "msal_extensions-1.3.1.tar.gz", hash = "sha256:c5b0fd10f65ef62b5f1d62f4251d51cbcaf003fcedae8c91b040a488614be1a4"}, +] + +[package.dependencies] +msal = ">=1.29,<2" + +[package.extras] +portalocker = ["portalocker (>=1.4,<4)"] + [[package]] name = "multidict" version = "6.7.0" description = "multidict implementation" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "multidict-6.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9f474ad5acda359c8758c8accc22032c6abe6dc87a8be2440d097785e27a9349"}, {file = "multidict-6.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4b7a9db5a870f780220e931d0002bbfd88fb53aceb6293251e2c839415c1b20e"}, @@ -2087,6 +2309,7 @@ version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -2132,6 +2355,7 @@ version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, @@ -2146,6 +2370,7 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["build", "dev"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -2157,6 +2382,7 @@ version = "0.5.7" description = "Divides large result sets into pages for easier browsing" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, @@ -2172,6 +2398,7 @@ version = "2.3.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, @@ -2271,6 +2498,7 @@ version = "1.7.4" description = "comprehensive password hashing framework supporting over 30 schemes" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "passlib-1.7.4-py2.py3-none-any.whl", hash = "sha256:aa6bca462b8d8bda89c70b382f0c298a20b5560af6cbfa2dce410c0a2fb669f1"}, {file = "passlib-1.7.4.tar.gz", hash = "sha256:defd50f72b65c5402ab2c573830a6978e5f202ad0d984793c8dde2c4152ebe04"}, @@ -2288,6 +2516,7 @@ version = "1.0.2" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pathspec-1.0.2-py3-none-any.whl", hash = "sha256:62f8558917908d237d399b9b338ef455a814801a4688bc41074b25feefd93472"}, {file = "pathspec-1.0.2.tar.gz", hash = "sha256:fa32b1eb775ed9ba8d599b22c5f906dc098113989da2c00bf8b210078ca7fb92"}, @@ -2305,6 +2534,8 @@ version = "2024.8.26" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" +groups = ["build"] +markers = "sys_platform == \"win32\"" files = [ {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, @@ -2316,6 +2547,8 @@ version = "2.1.2" description = "Python datetimes made easy" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, @@ -2350,6 +2583,7 @@ version = "0.4.0" description = "Efficient fuzzy matching for Polars DataFrames with support for multiple string similarity algorithms" optional = false python-versions = "<4.0,>=3.10" +groups = ["main"] files = [ {file = "pl_fuzzy_frame_match-0.4.0-py3-none-any.whl", hash = "sha256:7d35b4661fca2bf20afa4141baf619473fd455d5c8d3a1c44030120ade091cd9"}, {file = "pl_fuzzy_frame_match-0.4.0.tar.gz", hash = "sha256:157631fc6e1e2cd3dd797a335af8177e5e22d0edf719a0aa84ff99edf792c1c9"}, @@ -2369,6 +2603,7 @@ version = "4.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31"}, {file = "platformdirs-4.5.1.tar.gz", hash = "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda"}, @@ -2385,6 +2620,7 @@ version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -2400,6 +2636,8 @@ version = "1.25.2" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "polars-1.25.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59f2a34520ea4307a22e18b832310f8045a8a348606ca99ae785499b31eb4170"}, {file = "polars-1.25.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:e9fe45bdc2327c2e2b64e8849a992b6d3bd4a7e7848b8a7a3a439cca9674dc87"}, @@ -2436,7 +2674,7 @@ pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] @@ -2446,6 +2684,8 @@ version = "1.31.0" description = "Blazingly fast DataFrame library" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "sys_platform != \"win32\"" files = [ {file = "polars-1.31.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ccc68cd6877deecd46b13cbd2663ca89ab2a2cb1fe49d5cfc66a9cef166566d9"}, {file = "polars-1.31.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:a94c5550df397ad3c2d6adc212e59fd93d9b044ec974dd3653e121e6487a7d21"}, @@ -2482,7 +2722,7 @@ pyarrow = ["pyarrow (>=7.0.0)"] pydantic = ["pydantic"] sqlalchemy = ["polars[pandas]", "sqlalchemy"] style = ["great-tables (>=0.8.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; platform_system == \"Windows\""] xlsx2csv = ["xlsx2csv (>=0.8.0)"] xlsxwriter = ["xlsxwriter"] @@ -2492,6 +2732,7 @@ version = "0.4.3" description = "Polars plugin for pairwise distance functions" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "polars_distance-0.4.3-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d9a388510cad81be5c7ba978720595369a530a52394322f74e2455b9591ea73f"}, {file = "polars_distance-0.4.3-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:1aa81c2738825844bac342a7de33b4d940140c964f0e587118aab4f87674b02d"}, @@ -2515,6 +2756,7 @@ version = "0.10.4" description = "" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "polars_ds-0.10.4-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6f6dde4e835129155d0386fcdd0aa6b603e305eaf60c78d7ee743c4897bb1547"}, {file = "polars_ds-0.10.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1715b9aaef3e8f119eaf8d37cdc921c96350237edd1f3263d2d6e699b27c3e48"}, @@ -2540,6 +2782,7 @@ version = "0.5.0" description = "Transform string-based expressions into Polars DataFrame operations" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "polars_expr_transformer-0.5.0-py3-none-any.whl", hash = "sha256:37885454081a79970197c7e670844c8616b4df7fb927edd2384f4e6213b4b124"}, {file = "polars_expr_transformer-0.5.0.tar.gz", hash = "sha256:11492ec968c4b2bafe0fcffd4829a89b1737910b5f3e8ddb4f68dd3f4822bde1"}, @@ -2556,6 +2799,7 @@ version = "0.3.0" description = "High-performance graph analysis and pattern mining extension for Polars" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "polars_grouper-0.3.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6a2c56eb4621502447268c2d40bfc7696fe291691fe777b257cdda869bfbdde2"}, {file = "polars_grouper-0.3.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:3701fea159f2104d78e8aaad65c2af698275a8b8aa036a8c1d98ef18de06a822"}, @@ -2575,6 +2819,7 @@ version = "0.3.4" description = "Fast similarity join for polars DataFrames. Fork by Edwardvaneechoud with fixes." optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "polars_simed-0.3.4-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:fd8adf37248e19397f2ac9772d8bda49b67e2e22b3209c6b2decba181addede3"}, {file = "polars_simed-0.3.4-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e6755e2d895efdaf65d1948cf22999e6d9afc488b2f31a5c47850d89266c4b23"}, @@ -2593,6 +2838,7 @@ version = "0.4.1" description = "Accelerated property cache" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db"}, {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8"}, @@ -2724,6 +2970,7 @@ version = "2.9.11" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c"}, {file = "psycopg2_binary-2.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6fe6b47d0b42ce1c9f1fa3e35bb365011ca22e39db37074458f27921dca40f2"}, @@ -2800,6 +3047,7 @@ version = "18.1.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c"}, {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4"}, @@ -2854,6 +3102,7 @@ version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, @@ -2865,6 +3114,8 @@ version = "2.23" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "implementation_name != \"PyPy\"" files = [ {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"}, {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, @@ -2876,6 +3127,7 @@ version = "2.9.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, @@ -2891,7 +3143,7 @@ typing-extensions = [ [package.extras] email = ["email-validator (>=2.0.0)"] -timezone = ["tzdata"] +timezone = ["tzdata ; python_version >= \"3.9\" and sys_platform == \"win32\""] [[package]] name = "pydantic-core" @@ -2899,6 +3151,7 @@ version = "2.23.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, @@ -3000,6 +3253,7 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -3014,6 +3268,7 @@ version = "0.9.1" description = "Apache Iceberg is an open table format for huge analytic datasets" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,!=3.8.*,>=3.9" +groups = ["main"] files = [ {file = "pyiceberg-0.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a183d9217eb82159c01b23c683057f96c8b2375f592b921721d1c157895e2df"}, {file = "pyiceberg-0.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:57030bb15c397b0379242907c5611f5b4338fb799e972353fd0edafde6cfd2ef"}, @@ -3072,7 +3327,7 @@ pandas = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)"] polars = ["polars (>=1.21.0,<2.0.0)"] pyarrow = ["pyarrow (>=17.0.0,<20.0.0)"] pyiceberg-core = ["pyiceberg-core (>=0.4.0,<0.5.0)"] -ray = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)", "ray (==2.10.0)", "ray (>=2.10.0,<3.0.0)"] +ray = ["pandas (>=1.0.0,<3.0.0)", "pyarrow (>=17.0.0,<20.0.0)", "ray (==2.10.0) ; python_version < \"3.9\"", "ray (>=2.10.0,<3.0.0) ; python_version >= \"3.9\""] rest-sigv4 = ["boto3 (>=1.24.59)"] s3fs = ["s3fs (>=2023.1.0)"] snappy = ["python-snappy (>=0.6.0,<1.0.0)"] @@ -3086,6 +3341,7 @@ version = "6.17.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false python-versions = "<3.15,>=3.8" +groups = ["build"] files = [ {file = "pyinstaller-6.17.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:4e446b8030c6e5a2f712e3f82011ecf6c7ead86008357b0d23a0ec4bcde31dac"}, {file = "pyinstaller-6.17.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:aa9fd87aaa28239c6f0d0210114029bd03f8cac316a90bab071a5092d7c85ad7"}, @@ -3120,6 +3376,7 @@ version = "2025.11" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" +groups = ["build"] files = [ {file = "pyinstaller_hooks_contrib-2025.11-py3-none-any.whl", hash = "sha256:777e163e2942474aa41a8e6d31ac1635292d63422c3646c176d584d04d971c34"}, {file = "pyinstaller_hooks_contrib-2025.11.tar.gz", hash = "sha256:dfe18632e06655fa88d218e0d768fd753e1886465c12a6d4bce04f1aaeec917d"}, @@ -3129,12 +3386,34 @@ files = [ packaging = ">=22.0" setuptools = ">=42.0.0" +[[package]] +name = "pyjwt" +version = "2.10.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pymdown-extensions" version = "10.20" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pymdown_extensions-10.20-py3-none-any.whl", hash = "sha256:ea9e62add865da80a271d00bfa1c0fa085b20d133fb3fc97afdc88e682f60b2f"}, {file = "pymdown_extensions-10.20.tar.gz", hash = "sha256:5c73566ab0cf38c6ba084cb7c5ea64a119ae0500cce754ccb682761dfea13a52"}, @@ -3153,6 +3432,7 @@ version = "3.3.1" description = "pyparsing - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pyparsing-3.3.1-py3-none-any.whl", hash = "sha256:023b5e7e5520ad96642e2c6db4cb683d3970bd640cdf7115049a6e9c3682df82"}, {file = "pyparsing-3.3.1.tar.gz", hash = "sha256:47fad0f17ac1e2cad3de3b458570fbc9b03560aa029ed5e16ee5554da9a2251c"}, @@ -3167,6 +3447,7 @@ version = "8.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"}, {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"}, @@ -3190,6 +3471,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -3204,6 +3486,7 @@ version = "1.2.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61"}, {file = "python_dotenv-1.2.1.tar.gz", hash = "sha256:42667e897e16ab0d66954af0e60a9caa94f0fd4ecf3aaf6d2d260eec1aa36ad6"}, @@ -3218,6 +3501,7 @@ version = "3.5.0" description = "JOSE implementation in Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "python_jose-3.5.0-py2.py3-none-any.whl", hash = "sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771"}, {file = "python_jose-3.5.0.tar.gz", hash = "sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b"}, @@ -3240,6 +3524,7 @@ version = "0.0.21" description = "A streaming multipart parser for Python" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "python_multipart-0.0.21-py3-none-any.whl", hash = "sha256:cf7a6713e01c87aa35387f4774e812c4361150938d20d232800f75ffcf266090"}, {file = "python_multipart-0.0.21.tar.gz", hash = "sha256:7137ebd4d3bbf70ea1622998f902b97a29434a9e8dc40eb203bbcf7c2a2cba92"}, @@ -3251,6 +3536,7 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -3262,6 +3548,8 @@ version = "2020.1" description = "The Olson timezone database for Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, @@ -3273,6 +3561,8 @@ version = "311" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["dev"] +markers = "sys_platform == \"win32\"" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -3302,6 +3592,8 @@ version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" +groups = ["build"] +markers = "sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, @@ -3313,6 +3605,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -3395,6 +3688,7 @@ version = "1.1" description = "A custom YAML tag for referencing environment variables in YAML files." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, @@ -3409,6 +3703,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -3430,6 +3725,7 @@ version = "13.9.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, @@ -3449,6 +3745,7 @@ version = "4.9.1" description = "Pure-Python RSA implementation" optional = false python-versions = "<4,>=3.6" +groups = ["main"] files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, @@ -3463,6 +3760,7 @@ version = "0.8.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"}, {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"}, @@ -3490,6 +3788,7 @@ version = "2025.12.0" description = "Convenient Filesystem interface over S3" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "s3fs-2025.12.0-py3-none-any.whl", hash = "sha256:89d51e0744256baad7ae5410304a368ca195affd93a07795bc8ba9c00c9effbb"}, {file = "s3fs-2025.12.0.tar.gz", hash = "sha256:8612885105ce14d609c5b807553f9f9956b45541576a17ff337d9435ed3eb01f"}, @@ -3506,6 +3805,7 @@ version = "0.13.1" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724"}, {file = "s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf"}, @@ -3523,19 +3823,20 @@ version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["build"] files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] -core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -3543,6 +3844,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -3554,6 +3856,7 @@ version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, @@ -3565,6 +3868,7 @@ version = "2.0.45" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" +groups = ["main", "dev"] files = [ {file = "sqlalchemy-2.0.45-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c64772786d9eee72d4d3784c28f0a636af5b0a29f3fe26ff11f55efe90c0bd85"}, {file = "sqlalchemy-2.0.45-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7ae64ebf7657395824a19bca98ab10eb9a3ecb026bf09524014f1bb81cb598d4"}, @@ -3655,6 +3959,7 @@ version = "0.46.2" description = "The little ASGI library that shines." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35"}, {file = "starlette-0.46.2.tar.gz", hash = "sha256:7f7361f34eed179294600af672f565727419830b54b7b084efe44bb82d2fccd5"}, @@ -3672,6 +3977,7 @@ version = "1.7.3" description = "Strict, typed YAML parser" optional = false python-versions = ">=3.7.0" +groups = ["main"] files = [ {file = "strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7"}, {file = "strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407"}, @@ -3686,6 +3992,7 @@ version = "9.1.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, @@ -3701,6 +4008,7 @@ version = "4.14.0" description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "testcontainers-4.14.0-py3-none-any.whl", hash = "sha256:64e79b6b1e6d2b9b9e125539d35056caab4be739f7b7158c816d717f3596fa59"}, {file = "testcontainers-4.14.0.tar.gz", hash = "sha256:3b2d4fa487af23024f00fcaa2d1cf4a5c6ad0c22e638a49799813cb49b3176c7"}, @@ -3719,12 +4027,12 @@ aws = ["boto3 (>=1,<2)", "httpx"] azurite = ["azure-storage-blob (>=12,<13)"] chroma = ["chromadb-client (>=1,<2)"] cosmosdb = ["azure-cosmos (>=4,<5)"] -db2 = ["ibm_db_sa", "sqlalchemy (>=2,<3)"] +db2 = ["ibm_db_sa ; platform_machine != \"aarch64\" and platform_machine != \"arm64\"", "sqlalchemy (>=2,<3)"] generic = ["httpx", "redis (>=7,<8)"] google = ["google-cloud-datastore (>=2,<3)", "google-cloud-pubsub (>=2,<3)"] influxdb = ["influxdb (>=5,<6)", "influxdb-client (>=1,<2)"] k3s = ["kubernetes", "pyyaml (>=6.0.3)"] -keycloak = ["python-keycloak (>=6,<7)"] +keycloak = ["python-keycloak (>=6,<7) ; python_version < \"4.0\""] localstack = ["boto3 (>=1,<2)"] mailpit = ["cryptography"] minio = ["minio (>=7,<8)"] @@ -3734,7 +4042,7 @@ mysql = ["pymysql[rsa] (>=1,<2)", "sqlalchemy (>=2,<3)"] nats = ["nats-py (>=2,<3)"] neo4j = ["neo4j (>=6,<7)"] openfga = ["openfga-sdk"] -opensearch = ["opensearch-py (>=3,<4)"] +opensearch = ["opensearch-py (>=3,<4) ; python_version < \"4.0\""] oracle = ["oracledb (>=3,<4)", "sqlalchemy (>=2,<3)"] oracle-free = ["oracledb (>=3,<4)", "sqlalchemy (>=2,<3)"] qdrant = ["qdrant-client (>=1,<2)"] @@ -3754,6 +4062,8 @@ version = "2.3.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, @@ -3805,6 +4115,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -3826,6 +4137,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -3837,6 +4149,7 @@ version = "2025.3" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["dev"] files = [ {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, @@ -3848,16 +4161,17 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] [package.extras] -brotli = ["brotli (>=1.2.0)", "brotlicffi (>=1.2.0.0)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "uvicorn" @@ -3865,6 +4179,7 @@ version = "0.32.1" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "uvicorn-0.32.1-py3-none-any.whl", hash = "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e"}, {file = "uvicorn-0.32.1.tar.gz", hash = "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175"}, @@ -3876,7 +4191,7 @@ h11 = ">=0.8" typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} [package.extras] -standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] [[package]] name = "watchdog" @@ -3884,6 +4199,7 @@ version = "6.0.0" description = "Filesystem events monitoring" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, @@ -3926,6 +4242,7 @@ version = "1.0.0" description = "'Turn functions and methods into fully controllable objects'" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "wirerope-1.0.0-py2.py3-none-any.whl", hash = "sha256:59346555c7b5dbd1c683a4e123f8bed30ca99df646f6867ea6439ceabf43c2f6"}, {file = "wirerope-1.0.0.tar.gz", hash = "sha256:7da8bb6feeff9dd939bd7141ef0dc392674e43ba662e20909d6729db81a7c8d0"}, @@ -3936,7 +4253,7 @@ six = ">=1.11.0" [package.extras] doc = ["sphinx"] -test = ["pytest (>=4.6.7)", "pytest-checkdocs (>=1.2.5)", "pytest-checkdocs (>=2.9.0)", "pytest-cov (>=2.6.1)"] +test = ["pytest (>=4.6.7)", "pytest-checkdocs (>=1.2.5) ; python_version < \"3\"", "pytest-checkdocs (>=2.9.0) ; python_version >= \"3\"", "pytest-cov (>=2.6.1)"] [[package]] name = "wrapt" @@ -3944,6 +4261,7 @@ version = "1.17.3" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04"}, {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2"}, @@ -4034,6 +4352,7 @@ version = "3.2.9" description = "A Python module for creating Excel XLSX files." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3"}, {file = "xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c"}, @@ -4045,6 +4364,7 @@ version = "1.22.0" description = "Yet another URL library" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "yarl-1.22.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c7bd6683587567e5a49ee6e336e0612bec8329be1b7d4c8af5687dcdeb67ee1e"}, {file = "yarl-1.22.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5cdac20da754f3a723cceea5b3448e1a2074866406adeb4ef35b469d089adb8f"}, @@ -4184,6 +4504,6 @@ multidict = ">=4.0" propcache = ">=0.2.1" [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "f569b6c3944957d57bdcf64ab60324535b94bc3b86b399f8591ee358a324eddc" +content-hash = "02527847110dff93e53e993da009892b55bacf928a65ad0cde22ac42d236dc58" diff --git a/pyproject.toml b/pyproject.toml index 39711e894..57af0b563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,9 @@ tqdm = "^4.67.1" s3fs = "^2025.7.0" pl-fuzzy-frame-match = ">=0.4.0" pyyaml = "^6.0.3" +azure-storage-blob = "^12.24.0" +azure-identity = "^1.19.0" +adlfs = "^2025.1.0" [tool.poetry.scripts] @@ -71,6 +74,8 @@ stop_postgres = "test_utils.postgres.commands:stop_postgres" flowfile = "flowfile.__main__:main" start_minio = "test_utils.s3.commands:start_minio" stop_minio = "test_utils.s3.commands:stop_minio" +start_azurite = "test_utils.adls.commands:start_azurite" +stop_azurite = "test_utils.adls.commands:stop_azurite" flowfile-migrate = "tools.migrate.__main__:main" [tool.poetry.group.build] diff --git a/test_utils/adls/README.md b/test_utils/adls/README.md new file mode 100644 index 000000000..7cb7578a9 --- /dev/null +++ b/test_utils/adls/README.md @@ -0,0 +1,263 @@ +# Azure Data Lake Storage (ADLS) Testing with Azurite + +This directory contains utilities for testing ADLS connections using Azurite, Microsoft's official Azure Storage emulator. + +## Quick Start + +### Starting Azurite + +```bash +# Start Azurite container with test data +poetry run start_azurite +``` + +This will: +- Start an Azurite Docker container +- Create test containers (test-container, flowfile-test, sample-data, etc.) +- Populate with sample data in Parquet, CSV, and JSON formats +- Print connection details + +### Stopping Azurite + +```bash +# Stop and clean up Azurite container +poetry run stop_azurite +``` + +## Connection Details + +When Azurite is running, use these connection details: + +- **Account Name**: `devstoreaccount1` +- **Account Key**: `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` +- **Blob Endpoint**: `http://localhost:10000/devstoreaccount1` + +## Authentication Methods + +### 1. Access Key Authentication + +The simplest method for testing: + +```python +from flowfile_core.schemas.cloud_storage_schemas import FullCloudStorageConnection + +connection = FullCloudStorageConnection( + storage_type="adls", + auth_method="access_key", + connection_name="azurite_test", + azure_account_name="devstoreaccount1", + azure_account_key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + verify_ssl=False, # Required for local testing +) +``` + +### 2. Service Principal Authentication + +For production Azure environments: + +```python +connection = FullCloudStorageConnection( + storage_type="adls", + auth_method="service_principal", + connection_name="prod_adls", + azure_account_name="mystorageaccount", + azure_tenant_id="12345678-1234-1234-1234-123456789012", + azure_client_id="87654321-4321-4321-4321-210987654321", + azure_client_secret="your-client-secret", + verify_ssl=True, +) +``` + +### 3. SAS Token Authentication + +For temporary, scoped access: + +```python +connection = FullCloudStorageConnection( + storage_type="adls", + auth_method="sas_token", + connection_name="sas_access", + azure_account_name="mystorageaccount", + azure_sas_token="sv=2021-06-08&ss=bfqt&srt=sco&sp=rwdlacupiytfx...", + verify_ssl=True, +) +``` + +## Reading from ADLS + +### Single File + +```python +from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader + +# Get storage options +storage_options = CloudStorageReader.get_storage_options(connection) + +# Read single file +df = pl.scan_parquet( + "az://test-container/data/test_data.parquet", + storage_options=storage_options +).collect() +``` + +### Directory with Wildcards + +```python +# Read all parquet files in a directory +df = pl.scan_parquet( + "az://test-container/data/partitioned/*.parquet", + storage_options=storage_options +).collect() +``` + +## Writing to ADLS + +```python +from flowfile_core.schemas.cloud_storage_schemas import ( + CloudStorageWriteSettings, + get_cloud_storage_write_settings_worker_interface +) + +# Create write settings +write_settings = CloudStorageWriteSettings( + auth_mode="access_key", + connection_name="azurite_test", + resource_path="az://test-container/output/result.parquet", + file_format="parquet", + parquet_compression="snappy", + write_mode="overwrite", +) + +# Write data +from flowfile_worker.external_sources.s3_source.main import write_df_to_cloud + +write_df_to_cloud(df.lazy(), write_settings, logger) +``` + +## Test Data Structure + +After running `start_azurite`, the following test data is available: + +``` +test-container/ +├── data/ +│ ├── test_data.parquet # Sample DataFrame (5 rows) +│ ├── test_data.csv # Same data in CSV format +│ ├── test_data.json # Same data in NDJSON format +│ └── partitioned/ # Partitioned data for directory reads +│ ├── part_0.parquet +│ ├── part_1.parquet +│ └── part_2.parquet +``` + +## Frontend Usage + +### Creating an ADLS Connection + +1. Navigate to **Cloud Connections** in the UI +2. Click **New Connection** +3. Select **Azure Data Lake Storage** as the storage type +4. Choose authentication method: + - **Access Key**: Account name + account key + - **Service Principal**: Tenant ID + Client ID + Client Secret + - **SAS Token**: Account name + SAS token +5. For local testing with Azurite: + - Account Name: `devstoreaccount1` + - Account Key: `Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==` + - Custom Endpoint URL: `http://localhost:10000` + - Uncheck **Verify SSL** + +### Using ADLS in Nodes + +**Cloud Storage Reader Node:** +1. Add Cloud Storage Reader node to your flow +2. Select your ADLS connection +3. Enter path: `az://test-container/data/test_data.parquet` +4. Choose file format (Parquet, CSV, JSON, Delta) +5. Run the flow + +**Cloud Storage Writer Node:** +1. Add Cloud Storage Writer node to your flow +2. Select your ADLS connection +3. Enter output path: `az://test-container/output/result.parquet` +4. Choose file format and compression +5. Run the flow + +## URI Formats + +ADLS supports two URI formats: + +### Simple Format +``` +az://container/path/to/file.parquet +``` + +### ABFS Format (with account) +``` +abfs://container@account.dfs.core.windows.net/path/to/file.parquet +``` + +Both formats are supported by Flowfile. + +## Supported File Formats + +- **Parquet**: Columnar format, excellent compression +- **CSV**: Text format with configurable delimiter +- **JSON**: NDJSON (newline-delimited JSON) +- **Delta Lake**: ACID transactions, time travel + +## Environment Variables + +You can customize Azurite settings: + +```bash +export TEST_AZURITE_HOST=localhost +export TEST_AZURITE_BLOB_PORT=10000 +export TEST_AZURITE_ACCOUNT_NAME=devstoreaccount1 +export KEEP_AZURITE_RUNNING=true # Keep container after tests +``` + +## Troubleshooting + +### Connection Refused + +If you get connection errors: +1. Verify Azurite is running: `docker ps | grep azurite` +2. Check port 10000 is available: `lsof -i :10000` +3. Restart Azurite: `poetry run stop_azurite && poetry run start_azurite` + +### SSL Verification Errors + +For local Azurite testing: +- Always set `verify_ssl=False` in connections +- Or uncheck "Verify SSL" in the UI + +### Container Not Found + +Azurite uses the account name in the endpoint: +- Correct: `http://localhost:10000/devstoreaccount1` +- Wrong: `http://localhost:10000` + +### Authentication Errors + +Double-check the account key has no line breaks or extra spaces. + +## Production ADLS Setup + +For real Azure Storage accounts: + +1. **Create Storage Account** in Azure Portal +2. **Get Credentials**: + - Access Key: Storage Account → Access Keys + - Service Principal: Azure AD → App Registrations + - SAS Token: Storage Account → Shared access signature +3. **Grant Permissions**: + - Storage Blob Data Contributor role for service principals +4. **Create Connection** in Flowfile with production credentials +5. **Enable SSL**: Always use `verify_ssl=True` for production + +## Additional Resources + +- [Azure Storage Documentation](https://docs.microsoft.com/en-us/azure/storage/) +- [Azurite Emulator](https://github.com/Azure/Azurite) +- [Polars Azure Support](https://docs.pola.rs/user-guide/io/cloud-storage/) diff --git a/test_utils/adls/__init__.py b/test_utils/adls/__init__.py new file mode 100644 index 000000000..6dd879df9 --- /dev/null +++ b/test_utils/adls/__init__.py @@ -0,0 +1 @@ +"""Test utilities for Azure Data Lake Storage (ADLS) using Azurite emulator.""" diff --git a/test_utils/adls/commands.py b/test_utils/adls/commands.py new file mode 100644 index 000000000..54832efc2 --- /dev/null +++ b/test_utils/adls/commands.py @@ -0,0 +1,51 @@ +import logging + +# Set up logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S" +) +logger = logging.getLogger("azurite_commands") + + +def start_azurite(): + """Start Azurite container for ADLS testing""" + from . import fixtures + + if not fixtures.is_docker_available(): + logger.warning("Docker is not available. Cannot start Azurite container.") + print("\n" + "=" * 50) + print("SKIPPING: Docker is not available on this system") + print("Tests requiring Docker will need to be skipped") + print("=" * 50 + "\n") + return 0 # Return success to allow pipeline to continue + + if fixtures.start_azurite_container(): + print(f"Azurite started at http://localhost:{fixtures.AZURITE_BLOB_PORT}") + print(f"Account Name: {fixtures.AZURITE_ACCOUNT_NAME}") + print(f"Account Key: {fixtures.AZURITE_ACCOUNT_KEY}") + print("\nTest containers created:") + print(" - test-container") + print(" - flowfile-test") + print(" - sample-data") + print(" - worker-test-container") + print(" - demo-container") + return 0 + return 1 + + +def stop_azurite(): + """Stop Azurite container""" + from . import fixtures + + if not fixtures.is_docker_available(): + logger.warning("Docker is not available. Cannot stop Azurite container.") + print("\n" + "=" * 50) + print("SKIPPING: Docker is not available on this system") + print("Tests requiring Docker will need to be skipped") + print("=" * 50 + "\n") + return 0 + + if fixtures.stop_azurite_container(): + print("Azurite stopped successfully") + return 0 + return 1 diff --git a/test_utils/adls/data_generator.py b/test_utils/adls/data_generator.py new file mode 100644 index 000000000..e6097d4e2 --- /dev/null +++ b/test_utils/adls/data_generator.py @@ -0,0 +1,79 @@ +"""Generate test data for Azurite ADLS testing.""" +import io +import logging + +import polars as pl +from azure.storage.blob import BlobServiceClient + +logger = logging.getLogger("adls_data_generator") + + +def populate_test_data( + account_name: str, account_key: str, blob_endpoint: str, container_name: str = "test-container" +): + """ + Populate Azurite with test data in various formats. + + Args: + account_name: Azure storage account name + account_key: Azure storage account key + blob_endpoint: Blob storage endpoint URL + container_name: Container to populate with data + """ + # Create connection string + connection_string = ( + f"DefaultEndpointsProtocol=http;" + f"AccountName={account_name};" + f"AccountKey={account_key};" + f"BlobEndpoint={blob_endpoint};" + ) + + blob_service_client = BlobServiceClient.from_connection_string(connection_string) + container_client = blob_service_client.get_container_client(container_name) + + # Create sample DataFrame + df = pl.DataFrame( + { + "id": [1, 2, 3, 4, 5], + "name": ["Alice", "Bob", "Charlie", "David", "Eve"], + "age": [25, 30, 35, 40, 45], + "city": ["New York", "London", "Tokyo", "Paris", "Berlin"], + "score": [85.5, 92.3, 78.9, 88.1, 95.7], + } + ) + + # Upload Parquet file + parquet_buffer = io.BytesIO() + df.write_parquet(parquet_buffer) + parquet_buffer.seek(0) + blob_client = container_client.get_blob_client("data/test_data.parquet") + blob_client.upload_blob(parquet_buffer, overwrite=True) + logger.info(f"Uploaded: {container_name}/data/test_data.parquet") + + # Upload multiple Parquet files for directory testing + for i in range(3): + df_part = df.slice(i * 2, 2) + parquet_buffer = io.BytesIO() + df_part.write_parquet(parquet_buffer) + parquet_buffer.seek(0) + blob_client = container_client.get_blob_client(f"data/partitioned/part_{i}.parquet") + blob_client.upload_blob(parquet_buffer, overwrite=True) + logger.info(f"Uploaded: {container_name}/data/partitioned/part_{i}.parquet") + + # Upload CSV file + csv_buffer = io.BytesIO() + df.write_csv(csv_buffer) + csv_buffer.seek(0) + blob_client = container_client.get_blob_client("data/test_data.csv") + blob_client.upload_blob(csv_buffer, overwrite=True) + logger.info(f"Uploaded: {container_name}/data/test_data.csv") + + # Upload JSON file + json_buffer = io.BytesIO() + df.write_ndjson(json_buffer) + json_buffer.seek(0) + blob_client = container_client.get_blob_client("data/test_data.json") + blob_client.upload_blob(json_buffer, overwrite=True) + logger.info(f"Uploaded: {container_name}/data/test_data.json") + + logger.info(f"Successfully populated {container_name} with test data") diff --git a/test_utils/adls/fixtures.py b/test_utils/adls/fixtures.py new file mode 100644 index 000000000..bf6a56e7d --- /dev/null +++ b/test_utils/adls/fixtures.py @@ -0,0 +1,239 @@ +import logging +import os +import shutil +import subprocess +import time +from collections.abc import Generator +from contextlib import contextmanager + +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient + +from test_utils.adls.data_generator import populate_test_data + +logger = logging.getLogger("adls_fixture") + +AZURITE_HOST = os.environ.get("TEST_AZURITE_HOST", "localhost") +AZURITE_BLOB_PORT = int(os.environ.get("TEST_AZURITE_BLOB_PORT", 10000)) +AZURITE_QUEUE_PORT = int(os.environ.get("TEST_AZURITE_QUEUE_PORT", 10001)) +AZURITE_TABLE_PORT = int(os.environ.get("TEST_AZURITE_TABLE_PORT", 10002)) +AZURITE_ACCOUNT_NAME = os.environ.get("TEST_AZURITE_ACCOUNT_NAME", "devstoreaccount1") +AZURITE_ACCOUNT_KEY = os.environ.get( + "TEST_AZURITE_ACCOUNT_KEY", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +) +AZURITE_CONTAINER_NAME = os.environ.get("TEST_AZURITE_CONTAINER", "test-azurite-adls") +AZURITE_BLOB_ENDPOINT = f"http://{AZURITE_HOST}:{AZURITE_BLOB_PORT}/{AZURITE_ACCOUNT_NAME}" + +# Operating system detection +IS_MACOS = os.uname().sysname == "Darwin" if hasattr(os, "uname") else False +IS_WINDOWS = os.name == "nt" + + +def get_blob_service_client(): + """Get Azure Blob Service Client for Azurite""" + connection_string = ( + f"DefaultEndpointsProtocol=http;" + f"AccountName={AZURITE_ACCOUNT_NAME};" + f"AccountKey={AZURITE_ACCOUNT_KEY};" + f"BlobEndpoint=http://{AZURITE_HOST}:{AZURITE_BLOB_PORT}/{AZURITE_ACCOUNT_NAME};" + ) + return BlobServiceClient.from_connection_string(connection_string) + + +def wait_for_azurite(max_retries=30, interval=1): + """Wait for Azurite to be ready""" + for i in range(max_retries): + try: + client = get_blob_service_client() + # Try to list containers to verify connection + list(client.list_containers()) + logger.info("Azurite is ready") + return True + except Exception as e: + if i < max_retries - 1: + logger.debug(f"Waiting for Azurite... ({i+1}/{max_retries})") + time.sleep(interval) + else: + logger.error(f"Failed to connect to Azurite after {max_retries} attempts: {e}") + continue + return False + + +def is_container_running(container_name: str) -> bool: + """Check if Azurite container is already running""" + try: + result = subprocess.run( + ["docker", "ps", "--filter", f"name={container_name}", "--format", "{{.Names}}"], + capture_output=True, + text=True, + check=True, + ) + return container_name in result.stdout.strip() + except subprocess.CalledProcessError: + return False + + +def stop_azurite_container() -> bool: + """Stop the Azurite container and remove its data volume for a clean shutdown.""" + container_name = AZURITE_CONTAINER_NAME + volume_name = f"{container_name}-data" + + if not is_container_running(container_name): + logger.info(f"Container '{container_name}' is not running.") + # Attempt to remove the volume in case it was left orphaned + try: + subprocess.run(["docker", "volume", "rm", volume_name], check=False, capture_output=True) + except Exception: + pass # Ignore errors if volume doesn't exist + return True + + logger.info(f"Stopping and cleaning up container '{container_name}' and volume '{volume_name}'...") + try: + # Stop and remove the container + subprocess.run(["docker", "stop", container_name], check=True, capture_output=True) + subprocess.run(["docker", "rm", container_name], check=True, capture_output=True) + + # Remove the associated volume to clear all data + subprocess.run(["docker", "volume", "rm", volume_name], check=True, capture_output=True) + + logger.info("✅ Azurite container and data volume successfully removed.") + return True + except subprocess.CalledProcessError as e: + stderr = e.stderr.decode() if e.stderr else "" + if "no such volume" in stderr: + logger.info("Volume was already removed or never created.") + return True + logger.error(f"❌ Failed to clean up Azurite resources: {stderr}") + return False + + +def create_test_containers(): + """Create test containers and populate with sample data""" + client = get_blob_service_client() + + # Create test containers + containers = ["test-container", "flowfile-test", "sample-data", "worker-test-container", "demo-container"] + for container in containers: + try: + client.create_container(container) + logger.info(f"Created container: {container}") + except Exception as e: + if "ContainerAlreadyExists" in str(e): + logger.info(f"Container already exists: {container}") + else: + logger.warning(f"Error creating container {container}: {e}") + + +def is_docker_available() -> bool: + """ + Check if Docker is available on the system. + + Returns: + bool: True if Docker is available and working, False otherwise + """ + # Skip Docker on macOS and Windows in CI + if (IS_MACOS or IS_WINDOWS) and os.environ.get("CI", "").lower() in ("true", "1", "yes"): + logger.info("Skipping Docker on macOS/Windows in CI environment") + return False + + # If docker executable is not in PATH + if shutil.which("docker") is None: + logger.warning("Docker executable not found in PATH") + return False + + # Try a simple docker command + try: + result = subprocess.run( + ["docker", "info"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=5, + check=False, # Don't raise exception on non-zero return code + ) + + if result.returncode != 0: + logger.warning("Docker is not operational") + return False + + return True + except (subprocess.SubprocessError, OSError): + logger.warning("Error running Docker command") + return False + + +def start_azurite_container() -> bool: + """Start Azurite container with initialization""" + if is_container_running(AZURITE_CONTAINER_NAME): + logger.info(f"Container {AZURITE_CONTAINER_NAME} is already running") + return True + + try: + # Start Azurite with volume for persistence + subprocess.run( + [ + "docker", + "run", + "-d", + "--name", + AZURITE_CONTAINER_NAME, + "-p", + f"{AZURITE_BLOB_PORT}:10000", + "-p", + f"{AZURITE_QUEUE_PORT}:10001", + "-p", + f"{AZURITE_TABLE_PORT}:10002", + "-v", + f"{AZURITE_CONTAINER_NAME}-data:/data", + "mcr.microsoft.com/azure-storage/azurite", + "azurite-blob", + "--blobHost", + "0.0.0.0", + "--blobPort", + "10000", + "-l", + "/data", + ], + check=True, + ) + + # Wait for Azurite to be ready + if wait_for_azurite(): + create_test_containers() + populate_test_data( + account_name=AZURITE_ACCOUNT_NAME, + account_key=AZURITE_ACCOUNT_KEY, + blob_endpoint=AZURITE_BLOB_ENDPOINT, + container_name="test-container", + ) + return True + return False + + except Exception as e: + logger.error(f"Failed to start Azurite: {e}") + stop_azurite_container() + return False + + +@contextmanager +def managed_azurite() -> Generator[dict[str, any], None, None]: + """Context manager for Azurite container with full connection info""" + if not start_azurite_container(): + yield {} + return + + try: + connection_info = { + "account_name": AZURITE_ACCOUNT_NAME, + "account_key": AZURITE_ACCOUNT_KEY, + "blob_endpoint": AZURITE_BLOB_ENDPOINT, + "host": AZURITE_HOST, + "blob_port": AZURITE_BLOB_PORT, + "queue_port": AZURITE_QUEUE_PORT, + "table_port": AZURITE_TABLE_PORT, + } + yield connection_info + finally: + # Optionally keep container running for debugging + if os.environ.get("KEEP_AZURITE_RUNNING", "false").lower() != "true": + stop_azurite_container()