From d4b06249271f16e4001051df825b143002d7eee4 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Fri, 19 Jul 2024 12:58:35 +0200 Subject: [PATCH 1/4] OpenAPI: Standardize credentials in loadTable/loadView responses --- open-api/rest-catalog-open-api.py | 54 +++++++++++++++++ open-api/rest-catalog-open-api.yaml | 89 +++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index e890604c38cd..761810bb2ca4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1168,6 +1168,12 @@ class ViewUpdate(BaseModel): ] +class Credentials(BaseModel): + __root__: Union[ADLSCredentials, GCSCredentials, S3Credentials] = Field( + ..., discriminator='type' + ) + + class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -1195,6 +1201,11 @@ class LoadTableResult(BaseModel): - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + ## Credentials + + Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the + respective credentials exist in the `credentials` field before checking the `config` for credentials. + """ metadata_location: Optional[str] = Field( @@ -1203,6 +1214,7 @@ class LoadTableResult(BaseModel): description='May be null if the table is staged as part of a transaction', ) metadata: TableMetadata + credentials: Optional[Credentials] = None config: Optional[Dict[str, str]] = None @@ -1311,10 +1323,16 @@ class LoadViewResult(BaseModel): - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + ## Credentials + + Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the + respective credentials exist in the `credentials` field before checking the `config` for credentials. + """ metadata_location: str = Field(..., alias='metadata-location') metadata: ViewMetadata + credentials: Optional[Credentials] = None config: Optional[Dict[str, str]] = None @@ -1398,6 +1416,38 @@ class Schema(StructType): ) +class ADLSCredentials(BaseModel): + type: Literal['adls'] + sas_token: str = Field(..., alias='sas-token') + expires_at_ms: int = Field( + ..., + alias='expires-at-ms', + description='The epoch millis at which the given token expires', + ) + + +class GCSCredentials(BaseModel): + type: Literal['gcs'] + token: str + expires_at_ms: int = Field( + ..., + alias='expires-at-ms', + description='The epoch millis at which the given token expires', + ) + + +class S3Credentials(BaseModel): + type: Literal['s3'] + access_key_id: str = Field(..., alias='access-key-id') + secret_access_key: str = Field(..., alias='secret-access-key') + session_token: str = Field(..., alias='session-token') + expires_at_ms: int = Field( + ..., + alias='expires-at-ms', + description='The epoch millis at which the given token expires', + ) + + class CompletedPlanningResult(ScanTasks): """ Completed server-side planning result @@ -1430,12 +1480,16 @@ class CompletedPlanningWithIDResult(CompletedPlanningResult): TableMetadata.update_forward_refs() ViewMetadata.update_forward_refs() AddSchemaUpdate.update_forward_refs() +Credentials.update_forward_refs() ScanTasks.update_forward_refs() FetchPlanningResult.update_forward_refs() PlanTableScanResult.update_forward_refs() CreateTableRequest.update_forward_refs() CreateViewRequest.update_forward_refs() ReportMetricsRequest.update_forward_refs() +ADLSCredentials.update_forward_refs() +GCSCredentials.update_forward_refs() +S3Credentials.update_forward_refs() CompletedPlanningResult.update_forward_refs() FetchScanTasksResult.update_forward_refs() CompletedPlanningWithIDResult.update_forward_refs() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index cf4f7d3f9458..b377437f4910 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3103,6 +3103,82 @@ components: uuid: type: string + ADLSCredentials: + type: object + allOf: + - $ref: '#/components/schemas/Credentials' + required: + - type + - sas-token + - expires-at-ms + properties: + type: + type: string + enum: [ "adls" ] + sas-token: + type: string + expires-at-ms: + type: integer + format: int64 + description: The epoch millis at which the given token expires + + GCSCredentials: + type: object + allOf: + - $ref: '#/components/schemas/Credentials' + required: + - type + - token + - expires-at-ms + properties: + type: + type: string + enum: [ "gcs" ] + token: + type: string + expires-at-ms: + type: integer + format: int64 + description: The epoch millis at which the given token expires + + S3Credentials: + type: object + allOf: + - $ref: '#/components/schemas/Credentials' + required: + - type + - access-key-id + - secret-access-key + - session-token + - expires-at-ms + properties: + type: + type: string + enum: [ "s3" ] + access-key-id: + type: string + secret-access-key: + type: string + session-token: + type: string + expires-at-ms: + type: integer + format: int64 + description: The epoch millis at which the given token expires + + Credentials: + type: object + discriminator: + propertyName: type + mapping: + adls: '#/components/schemas/ADLSCredentials' + gcs: '#/components/schemas/GCSCredentials' + s3: '#/components/schemas/S3Credentials' + oneOf: + - $ref: '#/components/schemas/ADLSCredentials' + - $ref: '#/components/schemas/GCSCredentials' + - $ref: '#/components/schemas/S3Credentials' + LoadTableResult: description: | Result used when a table is successfully loaded. @@ -3129,6 +3205,11 @@ components: - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + + ## Credentials + + Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the + respective credentials exist in the `credentials` field before checking the `config` for credentials. type: object required: - metadata @@ -3138,6 +3219,8 @@ components: description: May be null if the table is staged as part of a transaction metadata: $ref: '#/components/schemas/TableMetadata' + credentials: + $ref: '#/components/schemas/Credentials' config: type: object additionalProperties: @@ -3395,6 +3478,10 @@ components: - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + ## Credentials + + Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the + respective credentials exist in the `credentials` field before checking the `config` for credentials. type: object required: - metadata-location @@ -3404,6 +3491,8 @@ components: type: string metadata: $ref: '#/components/schemas/ViewMetadata' + credentials: + $ref: '#/components/schemas/Credentials' config: type: object additionalProperties: From b8928e75f2cfdde6134a6369f6d2639d64448392 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Fri, 20 Sep 2024 14:55:50 +0200 Subject: [PATCH 2/4] Make storage_credentials a list --- open-api/rest-catalog-open-api.py | 49 ++++++++++++--------- open-api/rest-catalog-open-api.yaml | 68 ++++++++++++++++++----------- 2 files changed, 71 insertions(+), 46 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 761810bb2ca4..723045ea46a0 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1168,8 +1168,8 @@ class ViewUpdate(BaseModel): ] -class Credentials(BaseModel): - __root__: Union[ADLSCredentials, GCSCredentials, S3Credentials] = Field( +class Credential(BaseModel): + __root__: Union[ADLSCredential, GCSCredential, S3Credential] = Field( ..., discriminator='type' ) @@ -1201,10 +1201,11 @@ class LoadTableResult(BaseModel): - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - ## Credentials + ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the - respective credentials exist in the `credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. + In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. + Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. """ @@ -1214,7 +1215,9 @@ class LoadTableResult(BaseModel): description='May be null if the table is staged as part of a transaction', ) metadata: TableMetadata - credentials: Optional[Credentials] = None + storage_credentials: Optional[List[Credential]] = Field( + None, alias='storage-credentials' + ) config: Optional[Dict[str, str]] = None @@ -1323,16 +1326,19 @@ class LoadViewResult(BaseModel): - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled - ## Credentials + ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the - respective credentials exist in the `credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. + In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. + Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. """ metadata_location: str = Field(..., alias='metadata-location') metadata: ViewMetadata - credentials: Optional[Credentials] = None + storage_credentials: Optional[List[Credential]] = Field( + None, alias='storage-credentials' + ) config: Optional[Dict[str, str]] = None @@ -1416,35 +1422,38 @@ class Schema(StructType): ) -class ADLSCredentials(BaseModel): +class ADLSCredential(BaseModel): type: Literal['adls'] + scheme: str sas_token: str = Field(..., alias='sas-token') expires_at_ms: int = Field( ..., alias='expires-at-ms', - description='The epoch millis at which the given token expires', + description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', ) -class GCSCredentials(BaseModel): +class GCSCredential(BaseModel): type: Literal['gcs'] + scheme: str token: str expires_at_ms: int = Field( ..., alias='expires-at-ms', - description='The epoch millis at which the given token expires', + description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', ) -class S3Credentials(BaseModel): +class S3Credential(BaseModel): type: Literal['s3'] + scheme: str access_key_id: str = Field(..., alias='access-key-id') secret_access_key: str = Field(..., alias='secret-access-key') session_token: str = Field(..., alias='session-token') expires_at_ms: int = Field( ..., alias='expires-at-ms', - description='The epoch millis at which the given token expires', + description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', ) @@ -1480,16 +1489,16 @@ class CompletedPlanningWithIDResult(CompletedPlanningResult): TableMetadata.update_forward_refs() ViewMetadata.update_forward_refs() AddSchemaUpdate.update_forward_refs() -Credentials.update_forward_refs() +Credential.update_forward_refs() ScanTasks.update_forward_refs() FetchPlanningResult.update_forward_refs() PlanTableScanResult.update_forward_refs() CreateTableRequest.update_forward_refs() CreateViewRequest.update_forward_refs() ReportMetricsRequest.update_forward_refs() -ADLSCredentials.update_forward_refs() -GCSCredentials.update_forward_refs() -S3Credentials.update_forward_refs() +ADLSCredential.update_forward_refs() +GCSCredential.update_forward_refs() +S3Credential.update_forward_refs() CompletedPlanningResult.update_forward_refs() FetchScanTasksResult.update_forward_refs() CompletedPlanningWithIDResult.update_forward_refs() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b377437f4910..a75e2eea5e90 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3103,50 +3103,58 @@ components: uuid: type: string - ADLSCredentials: + ADLSCredential: type: object allOf: - - $ref: '#/components/schemas/Credentials' + - $ref: '#/components/schemas/Credential' required: - type + - scheme - sas-token - expires-at-ms properties: type: type: string enum: [ "adls" ] + scheme: + type: string sas-token: type: string expires-at-ms: type: integer format: int64 - description: The epoch millis at which the given token expires + description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires + - GCSCredentials: + GCSCredential: type: object allOf: - - $ref: '#/components/schemas/Credentials' + - $ref: '#/components/schemas/Credential' required: - type + - scheme - token - expires-at-ms properties: type: type: string enum: [ "gcs" ] + scheme: + type: string token: type: string expires-at-ms: type: integer format: int64 - description: The epoch millis at which the given token expires + description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires - S3Credentials: + S3Credential: type: object allOf: - - $ref: '#/components/schemas/Credentials' + - $ref: '#/components/schemas/Credential' required: - type + - scheme - access-key-id - secret-access-key - session-token @@ -3155,6 +3163,8 @@ components: type: type: string enum: [ "s3" ] + scheme: + type: string access-key-id: type: string secret-access-key: @@ -3164,20 +3174,20 @@ components: expires-at-ms: type: integer format: int64 - description: The epoch millis at which the given token expires + description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires - Credentials: + Credential: type: object discriminator: propertyName: type mapping: - adls: '#/components/schemas/ADLSCredentials' - gcs: '#/components/schemas/GCSCredentials' - s3: '#/components/schemas/S3Credentials' + adls: '#/components/schemas/ADLSCredential' + gcs: '#/components/schemas/GCSCredential' + s3: '#/components/schemas/S3Credential' oneOf: - - $ref: '#/components/schemas/ADLSCredentials' - - $ref: '#/components/schemas/GCSCredentials' - - $ref: '#/components/schemas/S3Credentials' + - $ref: '#/components/schemas/ADLSCredential' + - $ref: '#/components/schemas/GCSCredential' + - $ref: '#/components/schemas/S3Credential' LoadTableResult: description: | @@ -3206,10 +3216,11 @@ components: - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - ## Credentials + ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the - respective credentials exist in the `credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. + In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. + Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. type: object required: - metadata @@ -3219,8 +3230,10 @@ components: description: May be null if the table is staged as part of a transaction metadata: $ref: '#/components/schemas/TableMetadata' - credentials: - $ref: '#/components/schemas/Credentials' + storage-credentials: + type: array + items: + $ref: '#/components/schemas/Credential' config: type: object additionalProperties: @@ -3478,10 +3491,11 @@ components: - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled - ## Credentials + ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `credentials` field. Clients should first check whether the - respective credentials exist in the `credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. + In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. + Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. type: object required: - metadata-location @@ -3491,8 +3505,10 @@ components: type: string metadata: $ref: '#/components/schemas/ViewMetadata' - credentials: - $ref: '#/components/schemas/Credentials' + storage-credentials: + type: array + items: + $ref: '#/components/schemas/Credential' config: type: object additionalProperties: From 0313c120ac000f250156c4fbcd1a76abf6225fd8 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Tue, 24 Sep 2024 16:35:45 +0200 Subject: [PATCH 3/4] rename scheme to prefix --- open-api/rest-catalog-open-api.py | 15 ++++++++++++--- open-api/rest-catalog-open-api.yaml | 15 +++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 723045ea46a0..c2dd16f94c47 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1424,7 +1424,10 @@ class Schema(StructType): class ADLSCredential(BaseModel): type: Literal['adls'] - scheme: str + prefix: Optional[str] = Field( + None, + description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', + ) sas_token: str = Field(..., alias='sas-token') expires_at_ms: int = Field( ..., @@ -1435,7 +1438,10 @@ class ADLSCredential(BaseModel): class GCSCredential(BaseModel): type: Literal['gcs'] - scheme: str + prefix: Optional[str] = Field( + None, + description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', + ) token: str expires_at_ms: int = Field( ..., @@ -1446,7 +1452,10 @@ class GCSCredential(BaseModel): class S3Credential(BaseModel): type: Literal['s3'] - scheme: str + prefix: Optional[str] = Field( + None, + description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', + ) access_key_id: str = Field(..., alias='access-key-id') secret_access_key: str = Field(..., alias='secret-access-key') session_token: str = Field(..., alias='session-token') diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index a75e2eea5e90..63bdd0413732 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3109,15 +3109,16 @@ components: - $ref: '#/components/schemas/Credential' required: - type - - scheme - sas-token - expires-at-ms properties: type: type: string enum: [ "adls" ] - scheme: + prefix: type: string + description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most + specific prefix if several credentials of the same type are available. sas-token: type: string expires-at-ms: @@ -3132,15 +3133,16 @@ components: - $ref: '#/components/schemas/Credential' required: - type - - scheme - token - expires-at-ms properties: type: type: string enum: [ "gcs" ] - scheme: + prefix: type: string + description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most + specific prefix if several credentials of the same type are available. token: type: string expires-at-ms: @@ -3154,7 +3156,6 @@ components: - $ref: '#/components/schemas/Credential' required: - type - - scheme - access-key-id - secret-access-key - session-token @@ -3163,8 +3164,10 @@ components: type: type: string enum: [ "s3" ] - scheme: + prefix: type: string + description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most + specific prefix if several credentials of the same type are available. access-key-id: type: string secret-access-key: From 2d5820a04deb7153f953e3c335c05920317cbfcb Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Thu, 10 Oct 2024 08:42:21 +0200 Subject: [PATCH 4/4] Generalize how credentials are represented --- open-api/rest-catalog-open-api.py | 80 ++++--------------- open-api/rest-catalog-open-api.yaml | 116 +++++----------------------- 2 files changed, 36 insertions(+), 160 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index c2dd16f94c47..74ecdf2aae81 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -467,6 +467,14 @@ class AssertViewUUID(BaseModel): uuid: str +class StorageCredential(BaseModel): + prefix: str = Field( + ..., + description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix (by selecting the longest prefix) if several credentials of the same type are available.', + ) + config: Dict[str, str] + + class PlanStatus(BaseModel): __root__: Literal['completed', 'submitted', 'cancelled', 'failed'] = Field( ..., description='Status of a server-side planning operation' @@ -1168,12 +1176,6 @@ class ViewUpdate(BaseModel): ] -class Credential(BaseModel): - __root__: Union[ADLSCredential, GCSCredential, S3Credential] = Field( - ..., discriminator='type' - ) - - class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -1203,9 +1205,8 @@ class LoadTableResult(BaseModel): ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. - In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. - Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. """ @@ -1215,10 +1216,10 @@ class LoadTableResult(BaseModel): description='May be null if the table is staged as part of a transaction', ) metadata: TableMetadata - storage_credentials: Optional[List[Credential]] = Field( + config: Optional[Dict[str, str]] = None + storage_credentials: Optional[List[StorageCredential]] = Field( None, alias='storage-credentials' ) - config: Optional[Dict[str, str]] = None class ScanTasks(BaseModel): @@ -1328,18 +1329,17 @@ class LoadViewResult(BaseModel): ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. - In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. - Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. """ metadata_location: str = Field(..., alias='metadata-location') metadata: ViewMetadata - storage_credentials: Optional[List[Credential]] = Field( + config: Optional[Dict[str, str]] = None + storage_credentials: Optional[List[StorageCredential]] = Field( None, alias='storage-credentials' ) - config: Optional[Dict[str, str]] = None class ReportMetricsRequest(BaseModel): @@ -1422,50 +1422,6 @@ class Schema(StructType): ) -class ADLSCredential(BaseModel): - type: Literal['adls'] - prefix: Optional[str] = Field( - None, - description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', - ) - sas_token: str = Field(..., alias='sas-token') - expires_at_ms: int = Field( - ..., - alias='expires-at-ms', - description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', - ) - - -class GCSCredential(BaseModel): - type: Literal['gcs'] - prefix: Optional[str] = Field( - None, - description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', - ) - token: str - expires_at_ms: int = Field( - ..., - alias='expires-at-ms', - description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', - ) - - -class S3Credential(BaseModel): - type: Literal['s3'] - prefix: Optional[str] = Field( - None, - description='Indicates a storage location prefix where the credential is relevant. Clients should choose the most specific prefix if several credentials of the same type are available.', - ) - access_key_id: str = Field(..., alias='access-key-id') - secret_access_key: str = Field(..., alias='secret-access-key') - session_token: str = Field(..., alias='session-token') - expires_at_ms: int = Field( - ..., - alias='expires-at-ms', - description='The epoch millis since 1970-01-01T00:00:00Z at which the given token expires', - ) - - class CompletedPlanningResult(ScanTasks): """ Completed server-side planning result @@ -1498,16 +1454,12 @@ class CompletedPlanningWithIDResult(CompletedPlanningResult): TableMetadata.update_forward_refs() ViewMetadata.update_forward_refs() AddSchemaUpdate.update_forward_refs() -Credential.update_forward_refs() ScanTasks.update_forward_refs() FetchPlanningResult.update_forward_refs() PlanTableScanResult.update_forward_refs() CreateTableRequest.update_forward_refs() CreateViewRequest.update_forward_refs() ReportMetricsRequest.update_forward_refs() -ADLSCredential.update_forward_refs() -GCSCredential.update_forward_refs() -S3Credential.update_forward_refs() CompletedPlanningResult.update_forward_refs() FetchScanTasksResult.update_forward_refs() CompletedPlanningWithIDResult.update_forward_refs() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 63bdd0413732..5d669dc5fecf 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3103,94 +3103,20 @@ components: uuid: type: string - ADLSCredential: + StorageCredential: type: object - allOf: - - $ref: '#/components/schemas/Credential' - required: - - type - - sas-token - - expires-at-ms - properties: - type: - type: string - enum: [ "adls" ] - prefix: - type: string - description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most - specific prefix if several credentials of the same type are available. - sas-token: - type: string - expires-at-ms: - type: integer - format: int64 - description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires - - - GCSCredential: - type: object - allOf: - - $ref: '#/components/schemas/Credential' - required: - - type - - token - - expires-at-ms - properties: - type: - type: string - enum: [ "gcs" ] - prefix: - type: string - description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most - specific prefix if several credentials of the same type are available. - token: - type: string - expires-at-ms: - type: integer - format: int64 - description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires - - S3Credential: - type: object - allOf: - - $ref: '#/components/schemas/Credential' required: - - type - - access-key-id - - secret-access-key - - session-token - - expires-at-ms + - prefix + - config properties: - type: - type: string - enum: [ "s3" ] prefix: type: string description: Indicates a storage location prefix where the credential is relevant. Clients should choose the most - specific prefix if several credentials of the same type are available. - access-key-id: - type: string - secret-access-key: - type: string - session-token: - type: string - expires-at-ms: - type: integer - format: int64 - description: The epoch millis since 1970-01-01T00:00:00Z at which the given token expires - - Credential: - type: object - discriminator: - propertyName: type - mapping: - adls: '#/components/schemas/ADLSCredential' - gcs: '#/components/schemas/GCSCredential' - s3: '#/components/schemas/S3Credential' - oneOf: - - $ref: '#/components/schemas/ADLSCredential' - - $ref: '#/components/schemas/GCSCredential' - - $ref: '#/components/schemas/S3Credential' + specific prefix (by selecting the longest prefix) if several credentials of the same type are available. + config: + type: object + additionalProperties: + type: string LoadTableResult: description: | @@ -3221,9 +3147,8 @@ components: ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. - In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. - Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. type: object required: - metadata @@ -3233,14 +3158,14 @@ components: description: May be null if the table is staged as part of a transaction metadata: $ref: '#/components/schemas/TableMetadata' - storage-credentials: - type: array - items: - $ref: '#/components/schemas/Credential' config: type: object additionalProperties: type: string + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' ScanTasks: type: object @@ -3496,9 +3421,8 @@ components: ## Storage Credentials - Credentials for ADLS / GCS / S3 are provided through the `storage-credentials` field. - In order to avoid leaking non-expiring credentials, all credentials are required to have an expiration. - Clients should first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. + Credentials for ADLS / GCS / S3 / ... are provided through the `storage-credentials` field. + Clients must first check whether the respective credentials exist in the `storage-credentials` field before checking the `config` for credentials. type: object required: - metadata-location @@ -3508,14 +3432,14 @@ components: type: string metadata: $ref: '#/components/schemas/ViewMetadata' - storage-credentials: - type: array - items: - $ref: '#/components/schemas/Credential' config: type: object additionalProperties: type: string + storage-credentials: + type: array + items: + $ref: '#/components/schemas/StorageCredential' TokenType: type: string