From 91bde085b98a539f370478e6f9005985a7471b7c Mon Sep 17 00:00:00 2001 From: Grant Robinson Date: Thu, 9 Oct 2025 18:02:01 -0600 Subject: [PATCH 1/2] ES-2579 - improve the types - Allow more than 5 transform stream steps - Fix type mismatch between the `process` function and what the `ProcessFunction` type expects. --- lib/lib.d.ts | 4 +- lib/streams.d.ts | 183 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/lib/lib.d.ts b/lib/lib.d.ts index f554593..80fc73e 100644 --- a/lib/lib.d.ts +++ b/lib/lib.d.ts @@ -1330,8 +1330,8 @@ export declare namespace StreamUtil { * @todo question don't we already have other ways to do this? do we need this? * @todo unclear This is a transform stream which means it can't be the sink and yet it takes an outQueue as though it's sending to another queue. Don't get it. */ - function process(id: string, func: ProcessFunction, outQueue: string, onFlush?: any, opts?: any): TransformStream - function process(id: string, func: ProcessFunctionAsync, outQueue: string, onFlush?: any, opts?: any): TransformStream + function process(id: string, func: ProcessFunction, outQueue: string, onFlush?: any, opts?: any): TransformStream, U> + function process(id: string, func: ProcessFunctionAsync, outQueue: string, onFlush?: any, opts?: any): TransformStream, U> /** * todo document: what this functon does. Creates Correlation form read events diff --git a/lib/streams.d.ts b/lib/streams.d.ts index 3b24f0a..0923807 100644 --- a/lib/streams.d.ts +++ b/lib/streams.d.ts @@ -287,6 +287,99 @@ export function pipe(read: ReadableStream | stream.Strea */ export function pipe(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream | stream.Stream; +/** + * A callback-based version of [[`pipeAsync`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 8: The sink that is the last step of the pipe + * @param errorCallback Called if something goes wrong + * @returns The pipeline itself + */ +export function pipe(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream | stream.Stream; + +/** + * A callback-based version of [[`pipeAsync`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the next step of the pipe + * @typeParam T8 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t7 Pipeline step 8: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 9: The sink that is the last step of the pipe + * @param errorCallback Called if something goes wrong + * @returns The pipeline itself + */ +export function pipe(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, t7: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream | stream.Stream; + +/** + * A callback-based version of [[`pipeAsync`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the next step of the pipe + * @typeParam T8 The type of data generated and that moves to the next step of the pipe + * @typeParam T9 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t7 Pipeline step 8: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t8 Pipeline step 9: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 10: The sink that is the last step of the pipe + * @param errorCallback Called if something goes wrong + * @returns The pipeline itself + */ +export function pipe(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, t7: TransformStream, t8: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream | stream.Stream; + +/** + * A callback-based version of [[`pipeAsync`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * This catch-all overload handles pipelines with more than 8 transform streams. For such cases, TypeScript cannot + * maintain full type safety across all transformations, so the types are relaxed. + * + * @param streams Any number of streams (source, transforms, and sink) followed by an optional error callback + * @returns The pipeline itself + */ +export function pipe(...streams: Array): stream.Stream; + /** * An async/await-friendly version of [[`pipe`]]. Creates a pipeline of steps where the first step produces the data and then * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. @@ -387,6 +480,96 @@ export function pipeAsync(read: ReadableStream | stream. */ export function pipeAsync(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, write: WritableStream): Promise; +/** + * An async/await-friendly version of [[`pipe`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 8: The sink that is the last step of the pipe + * @returns A promise so it can play nice with async/await + */ +export function pipeAsync(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, write: WritableStream): Promise; + +/** + * An async/await-friendly version of [[`pipe`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the next step of the pipe + * @typeParam T8 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t7 Pipeline step 8: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 9: The sink that is the last step of the pipe + * @returns A promise so it can play nice with async/await + */ +export function pipeAsync(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, t7: TransformStream, write: WritableStream): Promise; + +/** + * An async/await-friendly version of [[`pipe`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * @typeParam T1 The type of data that is produced by the source + * @typeParam T2 The type of data generated and that moves to the next step of the pipe + * @typeParam T3 The type of data generated and that moves to the next step of the pipe + * @typeParam T4 The type of data generated and that moves to the next step of the pipe + * @typeParam T5 The type of data generated and that moves to the next step of the pipe + * @typeParam T6 The type of data generated and that moves to the next step of the pipe + * @typeParam T7 The type of data generated and that moves to the next step of the pipe + * @typeParam T8 The type of data generated and that moves to the next step of the pipe + * @typeParam T9 The type of data generated and that moves to the final step of the pipe + * + * @param read Pipeline step 1: The source that produces the data, the first step of the pipe + * @param t1 Pipeline step 2: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t2 Pipeline step 3: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t3 Pipeline step 4: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t4 Pipeline step 5: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t5 Pipeline step 6: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t6 Pipeline step 7: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t7 Pipeline step 8: A transformation step that takes data from the previous step and pushes the result to the next step + * @param t8 Pipeline step 9: A transformation step that takes data from the previous step and pushes the result to the next step + * @param write Pipeline step 10: The sink that is the last step of the pipe + * @returns A promise so it can play nice with async/await + */ +export function pipeAsync(read: ReadableStream | stream.Stream, t1: TransformStream, t2: TransformStream, t3: TransformStream, t4: TransformStream, t5: TransformStream, t6: TransformStream, t7: TransformStream, t8: TransformStream, write: WritableStream): Promise; + +/** + * An async/await-friendly version of [[`pipe`]]. Creates a pipeline of steps where the first step produces the data and then + * it flows to the next step and so on. The first step is the source, producing the content, the final step is the sink. + * + * This catch-all overload handles pipelines with more than 8 transform streams. For such cases, TypeScript cannot + * maintain full type safety across all transformations, so the types are relaxed. + * + * @param streams Any number of streams (source, transforms, and sink) + * @returns A promise so it can play nice with async/await + */ +export function pipeAsync(...streams: stream.Stream[]): Promise; + // export function pipe(read: ReadableStream | stream.Stream | stream.Readable, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream; // export function pipe(read: ReadableStream | stream.Stream | stream.Readable, t1: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream; // export function pipe(read: ReadableStream | stream.Stream | stream.Readable, t1: TransformStream, t2: TransformStream, write: WritableStream | stream.Stream, errorCallback?: ErrorCallback): WritableStream; From 06b80eb018ce6f9eea56a7812b31dc28d049e304 Mon Sep 17 00:00:00 2001 From: Grant Robinson Date: Thu, 9 Oct 2025 18:19:12 -0600 Subject: [PATCH 2/2] Bumped version up --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 91208be..491b732 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "leo-sdk", - "version": "7.1.13", + "version": "7.1.14", "description": "Load data onto the LEO Platform", "homepage": "https://leoplatform.io", "main": "index.js",