diff --git a/README.md b/README.md index 14d61b2..75f4849 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ map.on('load', () => { | uniforms | object | - | Shader uniform values (requires `customFrag`) | | onLoadingStateChange | function | - | Loading state callback | | throttleMs | number | `100` | Throttle interval (ms) for data fetching during rapid selector changes. Set to `0` to disable. | +| transformRequest | function | - | Transform request URLs and add headers/credentials (see [authentication](#authentication)) | ## methods @@ -210,6 +211,23 @@ const result = await layer.queryData({ **Note:** Query results match rendered values (`scale_factor`/`add_offset` applied, `fillValue`/NaN filtered). For datasets rendered via `proj4` reprojection, queries sample the underlying source grid; because reprojection/resampling occurs for display, a visual pixel click may not align perfectly with the nearest source pixel. +## authentication + +Use `transformRequest` to add headers or credentials to requests. The function receives the fully resolved URL for each request, enabling per-path authentication like presigned S3 URLs. Supports any [fetch options](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options). + +```ts +// Static auth (same headers for all requests) +transformRequest: (url) => ({ + url, + headers: { Authorization: `Bearer ${token}` }, +}) + +// Presigned URLs (path-specific signatures) +transformRequest: async (url) => ({ + url: await getPresignedUrl(url), +}) +``` + ## thanks This experiment is only possible following in the footsteps of other work in this space. [zarr-gl](https://github.com/carderne/zarr-gl) showed that custom layers are a viable rendering option and [zarr-cesium](https://github.com/NOC-OI/zarr-cesium) showed how flexible web rendering can be. We borrow code and concepts from both. This library also leans on our prior work on [@carbonplan/maps](https://github.com/carbonplan/maps) for many of its patterns. LLMs of several makes aided in the coding and debugging of this library. diff --git a/src/index.ts b/src/index.ts index 9ad5ede..2b2c71b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,8 @@ export type { LoadingState, LoadingStateCallback, Selector, + TransformRequest, + RequestParameters, } from './types' // Query interface exports diff --git a/src/types.ts b/src/types.ts index 3b667fd..4e10bce 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,6 +3,15 @@ import * as zarr from 'zarrita' /** Bounds tuple: [xMin, yMin, xMax, yMax] */ export type Bounds = [number, number, number, number] +export interface RequestParameters extends Omit { + url: string + headers?: { [key: string]: string } +} + +export type TransformRequest = ( + url: string +) => RequestParameters | Promise + export type ColormapArray = number[][] | string[] export type SelectorValue = number | number[] | string | string[] @@ -81,6 +90,12 @@ export interface ZarrLayerOptions { * Example: "+proj=lcc +lat_1=38.5 +lat_2=38.5 +lat_0=38.5 +lon_0=-97.5 +x_0=0 +y_0=0 +R=6371229 +units=m +no_defs" */ proj4?: string + /** + * Function to transform request URLs and add custom headers/credentials. + * Useful for authentication, proxy routing, or request customization. + * When provided, the store cache is bypassed to prevent credential sharing between layers. + */ + transformRequest?: TransformRequest } export type CRS = 'EPSG:4326' | 'EPSG:3857' diff --git a/src/zarr-layer.ts b/src/zarr-layer.ts index 103d3d7..f298592 100644 --- a/src/zarr-layer.ts +++ b/src/zarr-layer.ts @@ -26,6 +26,7 @@ import type { Selector, NormalizedSelector, ZarrLayerOptions, + TransformRequest, } from './types' import type { ZarrMode, RenderContext } from './zarr-mode' import { TiledMode } from './tiled-mode' @@ -114,6 +115,7 @@ export class ZarrLayer { private initError: Error | null = null private throttleMs: number private proj4: string | undefined + private transformRequest: TransformRequest | undefined get fillValue(): number | null { return this._fillValue @@ -146,6 +148,7 @@ export class ZarrLayer { onLoadingStateChange, throttleMs = 100, proj4, + transformRequest, }: ZarrLayerOptions) { if (!id) { throw new Error('[ZarrLayer] id is required') @@ -206,6 +209,7 @@ export class ZarrLayer { this.onLoadingStateChange = onLoadingStateChange this.throttleMs = throttleMs this.proj4 = proj4 + this.transformRequest = transformRequest } private emitLoadingState(): void { @@ -443,6 +447,7 @@ export class ZarrLayer { latIsAscending: this.latIsAscending, coordinateKeys: Object.keys(this.selector), proj4: this.proj4, + transformRequest: this.transformRequest, }) await this.zarrStore.initialized diff --git a/src/zarr-store.ts b/src/zarr-store.ts index 3e0fd4e..aa7846a 100644 --- a/src/zarr-store.ts +++ b/src/zarr-store.ts @@ -1,11 +1,12 @@ import * as zarr from 'zarrita' -import type { Readable } from '@zarrita/storage' +import type { Readable, AsyncReadable } from '@zarrita/storage' import type { Bounds, SpatialDimensions, DimIndicesProps, CRS, UntiledLevel, + TransformRequest, } from './types' import type { XYLimits } from './map-utils' import { identifyDimensionIndices } from './zarr-utils' @@ -111,8 +112,150 @@ interface ZarrV3ArrayMetadata { attributes?: Record } +type AbsolutePath = `/${string}` +type RangeQuery = { offset: number; length: number } | { suffixLength: number } + +/** + * Merge RequestInit objects, properly combining headers instead of replacing. + * Request overrides take precedence over store overrides. + */ +const mergeInit = ( + storeOverrides: RequestInit, + requestOverrides?: RequestInit +): RequestInit => { + if (!requestOverrides) return storeOverrides + return { + ...storeOverrides, + ...requestOverrides, + headers: { + ...(storeOverrides.headers as Record), + ...(requestOverrides.headers as Record), + }, + } +} + +/** + * Handle fetch response, returning bytes or undefined for 404. + */ +const handleResponse = async ( + response: Response +): Promise => { + if (response.status === 404) return undefined + if (response.status === 200 || response.status === 206) { + return new Uint8Array(await response.arrayBuffer()) + } + throw new Error( + `Unexpected response status ${response.status} ${response.statusText}` + ) +} + +/** + * Fetch a byte range from a URL. + */ +const fetchRange = ( + url: string | URL, + offset: number, + length: number, + opts: RequestInit = {} +): Promise => { + return fetch(url, { + ...opts, + headers: { + ...(opts.headers as Record), + Range: `bytes=${offset}-${offset + length - 1}`, + }, + }) +} + +/** + * Fetch suffix bytes from a URL. + * Uses HEAD + range fallback for backends that don't support suffix ranges. + */ +const fetchSuffix = async ( + url: string | URL, + suffixLength: number, + init: RequestInit +): Promise => { + // Use HEAD + range fallback (matches zarrita's default behavior) + // Some backends (like S3) don't support bytes=-n suffix requests well + const headResponse = await fetch(url, { ...init, method: 'HEAD' }) + if (!headResponse.ok) { + return headResponse // will be picked up by handleResponse + } + const contentLength = headResponse.headers.get('Content-Length') + const length = Number(contentLength) + return fetchRange(url, length - suffixLength, suffixLength, init) +} + +/** + * Custom store that calls transformRequest for each request with the fully resolved URL. + * This enables per-path authentication like presigned S3 URLs. + */ +class TransformingFetchStore implements AsyncReadable { + private baseUrl: URL + private transformRequest: TransformRequest + + constructor(url: string, transformRequest: TransformRequest) { + this.baseUrl = new URL(url) + if (!this.baseUrl.pathname.endsWith('/')) { + this.baseUrl.pathname += '/' + } + this.transformRequest = transformRequest + } + + private resolveUrl(key: AbsolutePath): string { + const resolved = new URL(key.slice(1), this.baseUrl) + resolved.search = this.baseUrl.search + return resolved.href + } + + async get( + key: AbsolutePath, + opts?: RequestInit + ): Promise { + const resolvedUrl = this.resolveUrl(key) + const { url: transformedUrl, ...overrides } = await this.transformRequest( + resolvedUrl + ) + + const merged = mergeInit(overrides, opts) + const response = await fetch(transformedUrl, merged) + return handleResponse(response) + } + + async getRange( + key: AbsolutePath, + range: RangeQuery, + opts?: RequestInit + ): Promise { + const resolvedUrl = this.resolveUrl(key) + const { url: transformedUrl, ...overrides } = await this.transformRequest( + resolvedUrl + ) + + const merged = mergeInit(overrides, opts) + let response: Response + + if ('suffixLength' in range) { + response = await fetchSuffix(transformedUrl, range.suffixLength, merged) + } else { + response = await fetchRange( + transformedUrl, + range.offset, + range.length, + merged + ) + } + + return handleResponse(response) + } +} + type ConsolidatedStore = zarr.Listable -type ZarrStoreType = zarr.FetchStore | ConsolidatedStore +type ZarrStoreType = + | zarr.FetchStore + | TransformingFetchStore + | ConsolidatedStore interface ZarrStoreOptions { source: string @@ -123,6 +266,7 @@ interface ZarrStoreOptions { coordinateKeys?: string[] latIsAscending?: boolean | null proj4?: string + transformRequest?: TransformRequest } interface StoreDescription { @@ -147,6 +291,22 @@ interface StoreDescription { proj4: string | null } +/** + * Factory function to create a store with optional request transformation. + * When transformRequest is provided, returns a TransformingFetchStore that + * calls the transform function for each request with the fully resolved URL. + * This enables per-path authentication like presigned S3 URLs. + */ +const createFetchStore = ( + url: string, + transformRequest?: TransformRequest +): zarr.FetchStore | TransformingFetchStore => { + if (!transformRequest) { + return new zarr.FetchStore(url) + } + return new TransformingFetchStore(url, transformRequest) +} + export class ZarrStore { private static _cache = new Map< string, @@ -160,6 +320,7 @@ export class ZarrStore { spatialDimensions: SpatialDimensions private explicitBounds: Bounds | null coordinateKeys: string[] + private transformRequest?: TransformRequest metadata: ZarrV2ConsolidatedMetadata | ZarrV3GroupMetadata | null = null arrayMetadata: ZarrV3ArrayMetadata | null = null @@ -213,6 +374,7 @@ export class ZarrStore { coordinateKeys = [], latIsAscending = null, proj4, + transformRequest, }: ZarrStoreOptions) { if (!source) { throw new Error('source is a required parameter') @@ -228,22 +390,37 @@ export class ZarrStore { this.coordinateKeys = coordinateKeys this.latIsAscending = latIsAscending this.proj4 = proj4 ?? null + this.transformRequest = transformRequest this.initialized = this._initialize() } private async _initialize(): Promise { const storeCacheKey = `${this.source}:${this.version ?? 'auto'}` - let storeHandle = ZarrStore._storeCache.get(storeCacheKey) + let storeHandle: Promise | undefined - if (!storeHandle) { - const baseStore = new zarr.FetchStore(this.source) + if (this.transformRequest) { + // Bypass cache when transformRequest is provided (unique credentials per layer) + const baseStore = createFetchStore(this.source, this.transformRequest) if (this.version === 3) { storeHandle = Promise.resolve(baseStore) } else { storeHandle = zarr.tryWithConsolidated(baseStore).catch(() => baseStore) } - ZarrStore._storeCache.set(storeCacheKey, storeHandle) + } else { + // Use cached store for standard requests + storeHandle = ZarrStore._storeCache.get(storeCacheKey) + if (!storeHandle) { + const baseStore = new zarr.FetchStore(this.source) + if (this.version === 3) { + storeHandle = Promise.resolve(baseStore) + } else { + storeHandle = zarr + .tryWithConsolidated(baseStore) + .catch(() => baseStore) + } + ZarrStore._storeCache.set(storeCacheKey, storeHandle) + } } this.store = await storeHandle @@ -407,21 +584,24 @@ export class ZarrStore { private async _loadV2() { const cacheKey = `v2:${this.source}` - let zmetadata = ZarrStore._cache.get(cacheKey) as - | ZarrV2ConsolidatedMetadata - | undefined + // Bypass cache when transformRequest is provided (unique credentials per layer) + let zmetadata = this.transformRequest + ? undefined + : (ZarrStore._cache.get(cacheKey) as + | ZarrV2ConsolidatedMetadata + | undefined) if (!zmetadata) { if (this.isConsolidatedStore(this.store)) { const rootZattrsBytes = await this.store.get('/.zattrs') const rootZattrs = rootZattrsBytes ? decodeJSON(rootZattrsBytes) : {} zmetadata = { metadata: { '.zattrs': rootZattrs } } - ZarrStore._cache.set(cacheKey, zmetadata) + if (!this.transformRequest) ZarrStore._cache.set(cacheKey, zmetadata) } else { try { zmetadata = (await this._getJSON( '/.zmetadata' )) as ZarrV2ConsolidatedMetadata - ZarrStore._cache.set(cacheKey, zmetadata) + if (!this.transformRequest) ZarrStore._cache.set(cacheKey, zmetadata) } catch { const zattrs = await this._getJSON('/.zattrs') zmetadata = { metadata: { '.zattrs': zattrs } } @@ -484,19 +664,24 @@ export class ZarrStore { private async _loadV3() { const metadataCacheKey = `v3:${this.source}` - let metadata = ZarrStore._cache.get(metadataCacheKey) as - | ZarrV3GroupMetadata - | undefined + // Bypass cache when transformRequest is provided (unique credentials per layer) + let metadata = this.transformRequest + ? undefined + : (ZarrStore._cache.get(metadataCacheKey) as + | ZarrV3GroupMetadata + | undefined) if (!metadata) { metadata = (await this._getJSON('/zarr.json')) as ZarrV3GroupMetadata - ZarrStore._cache.set(metadataCacheKey, metadata) - - if (metadata.consolidated_metadata?.metadata) { - for (const [key, arrayMeta] of Object.entries( - metadata.consolidated_metadata.metadata - )) { - const arrayCacheKey = `v3:${this.source}/${key}` - ZarrStore._cache.set(arrayCacheKey, arrayMeta) + if (!this.transformRequest) { + ZarrStore._cache.set(metadataCacheKey, metadata) + + if (metadata.consolidated_metadata?.metadata) { + for (const [key, arrayMeta] of Object.entries( + metadata.consolidated_metadata.metadata + )) { + const arrayCacheKey = `v3:${this.source}/${key}` + ZarrStore._cache.set(arrayCacheKey, arrayMeta) + } } } } @@ -516,14 +701,15 @@ export class ZarrStore { ? `${this.levels[0]}/${this.variable}` : this.variable const arrayCacheKey = `v3:${this.source}/${arrayKey}` - let arrayMetadata = ZarrStore._cache.get(arrayCacheKey) as - | ZarrV3ArrayMetadata - | undefined + let arrayMetadata = this.transformRequest + ? undefined + : (ZarrStore._cache.get(arrayCacheKey) as ZarrV3ArrayMetadata | undefined) if (!arrayMetadata) { arrayMetadata = (await this._getJSON( `/${arrayKey}/zarr.json` )) as ZarrV3ArrayMetadata - ZarrStore._cache.set(arrayCacheKey, arrayMetadata) + if (!this.transformRequest) + ZarrStore._cache.set(arrayCacheKey, arrayMetadata) } this.arrayMetadata = arrayMetadata