diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65ecb88..29c84a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,11 @@ VITE_TEST_INTEGRATION=1 VITE_TEST_DDN_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx VITE_TEST_DDN_API_SECRET=yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy VITE_TEST_SEAFOWL_SECRET=zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz +VITE_TEST_GITHUB_PAT=uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu +VITE_TEST_SEAFOWL_EXPORT_DEST_URL=https://demo.seafowl.cloud +# should match the username associated with the API_KEY for Splitgraph +VITE_TEST_SEAFOWL_EXPORT_DEST_DBNAME=miles +VITE_TEST_SEAFOWL_EXPORT_DEST_SECRET=tttttttttttttttttttttttttttttttt ``` Then simply append `--mode integration` flag to any variant of `yarn test` that @@ -68,6 +73,24 @@ where a separate process like `mitmproxy` can intercept outbound requests: yarn test-mitm --mode integration ``` +### Run tests in VSCode "JavaScript Debug Terminal" + +If using VSCode, the easiest debugging method is to open a "JavaScript Debug +Terminal" (which you can do via the command palette +"`Debug: JavaScript Debug Terminal`"). Put a `debugger;` statement where you +want to break, and then run vitest in single-threaded mode: + +```bash +yarn test --single-thread +``` + +Or, if you also want to use mitmproxy (assumed to be listening on port `7979`), +then: + +```bash +yarn test-mitm --single-thread +``` + ### Typecheck We use `tsc` for typechecking, with the default solution file `tsconfig.json` @@ -160,6 +183,9 @@ defined, in topological order. Therefore, to indicate a workspace is publishable, make sure the `package.json` includes `scripts.version` and `scripts.publish`. +If you want to publish a prerelease ("tag"), then add `--tag` to the +`publish-all` command, e.g. `yarn publish-all --tag canary --otp ` + Create deferred patch (0.0.x) changes (if necessary) in topological order (NOTE: To update all versions immediately, change `-d` to `-i`, and then there diff --git a/packages/base-client/package.json b/packages/base-client/package.json index 4b8d7cf..69a1682 100644 --- a/packages/base-client/package.json +++ b/packages/base-client/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/base-client", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/base-db/base-db.ts b/packages/base-db/base-db.ts index 39ba45b..a7c6031 100644 --- a/packages/base-db/base-db.ts +++ b/packages/base-db/base-db.ts @@ -21,22 +21,69 @@ import { import type { HTTPStrategies } from "@madatdata/client-http"; -export interface ImportPlugin extends Plugin { +export interface ImportPlugin< + PluginName extends string, + ConcreteSourceOptions extends object = any, + ConcreteDestOptions extends object = any +> extends Plugin { + __name: PluginName; importData: ( - sourceOptions: any, - destOptions: any - ) => Promise<{ response: any | null; error: any | null; info?: any | null }>; + sourceOptions: ConcreteSourceOptions, + destOptions: ConcreteDestOptions, + importOptions?: { defer: boolean } + ) => Promise<{ + taskId?: string | null; + response: any | null; + error: any | null; + info?: any | null; + }>; } // interface ImportPluginWithOptions extends ImportPlugin { // withOptions: WithOptions; // } -export interface ExportPlugin extends Plugin { +export interface ExportPlugin< + PluginName extends string, + ConcreteSourceOptions extends object = any, + ConcreteDestOptions extends object = any +> extends Plugin { + __name: PluginName; exportData: ( - sourceOptions: any, - destOptions: any - ) => Promise<{ response: any | null; error: any | null; info?: any | null }>; + sourceOptions: ConcreteSourceOptions, + destOptions: ConcreteDestOptions, + exportOptions?: { defer: boolean } + ) => Promise<{ + response: any | null; + error: any | null; + info?: any | null; + taskId?: string | null; + taskIds?: any; + }>; +} + +export interface DeferredTaskPlugin< + PluginName extends string, + // TODO: maybe should extend { __name: PluginName } to ensure finding + // correct task plugin by serialized deferred task + + DeferredTaskResponse extends { + completed: boolean; + response: any | null; + error: any | null; + info: any | null; + } = { + completed: boolean; + response: Record; + error: any; + info: any; + }, + MemoizedDeferredTask extends object = any +> extends Plugin { + __name: PluginName; + pollDeferredTask: ( + memoizedDeferredTask: MemoizedDeferredTask + ) => Promise; } // interface ExportPluginWithOptions extends ExportPlugin { @@ -51,6 +98,13 @@ export interface DbPluggableInterface exportData: >( ...exportDataArgsForPlugin: Parameters ) => Promise; + pollDeferredTask: < + MatchingPlugin extends DeferredTaskPluginFromList + >( + ...pollDeferredTaskArgsForPlugin: Parameters< + MatchingPlugin["pollDeferredTask"] + > + ) => Promise; } export interface Db { @@ -62,6 +116,12 @@ export interface Db { pluginName: MatchingPlugin["__name"], ...rest: Parameters ) => Promise; + pollDeferredTask: < + MatchingPlugin extends DeferredTaskPluginFromList + >( + pluginName: MatchingPlugin["__name"], + ...rest: Parameters + ) => Promise; makeClient: ( makeClientForProtocol: ( wrappedOptions: ImplementationSpecificClientOptions @@ -74,17 +134,25 @@ export type ImportPluginFromList< ConcretePluginList extends PluginList, PluginName extends ExtractPlugin< ConcretePluginList, - ImportPlugin + ImportPlugin >["__name"] = string -> = ExtractPlugin; +> = ExtractPlugin>; export type ExportPluginFromList< ConcretePluginList extends PluginList, PluginName extends ExtractPlugin< ConcretePluginList, - ExportPlugin + ExportPlugin >["__name"] = string -> = ExtractPlugin; +> = ExtractPlugin>; + +export type DeferredTaskPluginFromList< + ConcretePluginList extends PluginList, + PluginName extends ExtractPlugin< + ConcretePluginList, + DeferredTaskPlugin + >["__name"] = string +> = ExtractPlugin>; export interface DbOptions { plugins: ConcretePluginList; @@ -173,6 +241,20 @@ export abstract class BaseDb< ...rest: Parameters ): Promise; + abstract pollDeferredTask< + PluginName extends DeferredTaskPluginFromList< + ConcretePluginList, + string + >["__name"], + MatchingPlugin extends DeferredTaskPluginFromList< + ConcretePluginList, + PluginName + > + >( + pluginName: PluginName, + ...rest: Parameters + ): Promise; + /** * Return a fingerprint and normalized query (used as input to the fingerprint) * for a given SQL string. Default to SHA-256 and normalizing for HTTP headers. @@ -187,7 +269,7 @@ export abstract class BaseDb< // In vitest, really JSDOM, it's a bit of a mix between the two (window is available?) // NOTE: Need to test how this will work in a browser bundle which we don't even have yet const subtle = await (async () => { - if (!window?.crypto?.subtle) { + if (typeof window === "undefined" || !window?.crypto?.subtle) { const { webcrypto } = await import("crypto"); if (webcrypto.subtle) { diff --git a/packages/base-db/package.json b/packages/base-db/package.json index 3485128..3cdb458 100644 --- a/packages/base-db/package.json +++ b/packages/base-db/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/base-db", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/client-http/package.json b/packages/client-http/package.json index 3479328..068205f 100644 --- a/packages/client-http/package.json +++ b/packages/client-http/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/client-http", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/client-postgres/package.json b/packages/client-postgres/package.json index 610da07..fa391bd 100644 --- a/packages/client-postgres/package.json +++ b/packages/client-postgres/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/client-postgres", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/core/package.json b/packages/core/package.json index 8ae168d..50fcf3e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/core", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/core/splitgraph-seafowl-sync.test.ts b/packages/core/splitgraph-seafowl-sync.test.ts index 9519ebb..7bfa0ae 100644 --- a/packages/core/splitgraph-seafowl-sync.test.ts +++ b/packages/core/splitgraph-seafowl-sync.test.ts @@ -16,7 +16,7 @@ describe.skipIf(shouldSkipSeafowlTests() || shouldSkipIntegrationTests())( const seafowl = createRealSeafowlDataContext(); const { response } = await splitgraph.db.exportData( - "exportQuery", + "export-query-to-file", { query: `SELECT a as int_val, string_agg(random()::text, '') as text_val FROM generate_series(1, 5) a, generate_series(1, 50) b diff --git a/packages/core/splitgraph.test.ts b/packages/core/splitgraph.test.ts index 2fc32fc..24ad89d 100644 --- a/packages/core/splitgraph.test.ts +++ b/packages/core/splitgraph.test.ts @@ -291,7 +291,7 @@ describe("makeSplitgraphHTTPContext", () => { }, }, "plugins": [ - _SplitgraphImportCSVPlugin { + SplitgraphImportCSVPlugin { "__name": "csv", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { @@ -310,8 +310,46 @@ describe("makeSplitgraphHTTPContext", () => { }, "transformRequestHeaders": [Function], }, - _ExportQueryPlugin { - "__name": "exportQuery", + SplitgraphGeneratedImportPlugin { + "__name": "airbyte-github", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportQueryToFilePlugin { + "__name": "export-query-to-file", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportToSeafowlPlugin { + "__name": "export-to-seafowl", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { "options": { @@ -334,7 +372,7 @@ describe("makeSplitgraphHTTPContext", () => { "plugins": PluginRegistry { "hostContext": {}, "plugins": [ - _SplitgraphImportCSVPlugin { + SplitgraphImportCSVPlugin { "__name": "csv", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { @@ -353,8 +391,46 @@ describe("makeSplitgraphHTTPContext", () => { }, "transformRequestHeaders": [Function], }, - _ExportQueryPlugin { - "__name": "exportQuery", + SplitgraphGeneratedImportPlugin { + "__name": "airbyte-github", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportQueryToFilePlugin { + "__name": "export-query-to-file", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportToSeafowlPlugin { + "__name": "export-to-seafowl", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { "options": { diff --git a/packages/db-seafowl/db-seafowl.ts b/packages/db-seafowl/db-seafowl.ts index c11fda7..dc52296 100644 --- a/packages/db-seafowl/db-seafowl.ts +++ b/packages/db-seafowl/db-seafowl.ts @@ -31,11 +31,12 @@ import { // FIXME: we _should_ only be depending on types from this pacakge - should // they be in a separate package from the actual http-client? import type { HTTPStrategies, HTTPClientOptions } from "@madatdata/client-http"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; export type DefaultSeafowlPluginList< - ConcretePluginList extends (ImportPlugin | ExportPlugin)[] = ( - | ImportPlugin - | ExportPlugin + ConcretePluginList extends (ImportPlugin | ExportPlugin)[] = ( + | ImportPlugin + | ExportPlugin )[] > = ConcretePluginList[number][]; @@ -265,10 +266,16 @@ export class DbSeafowl ); } - async exportData( - _pluginName: ExtractPlugin["__name"], + // TODO: atm, there are no Seafowl export plugins + async exportData< + PluginName extends ExtractPlugin< + SeafowlPluginList, + ExportPlugin + >["__name"] + >( + _pluginName: PluginName, ..._rest: Parameters< - ExtractPlugin["exportData"] + ExtractPlugin>["exportData"] > ): Promise { await Promise.resolve(); @@ -281,10 +288,37 @@ export class DbSeafowl }; } - async importData( - pluginName: ExtractPlugin["__name"], + // TODO: atm, there are no deferred tasks for seafowl + async pollDeferredTask< + PluginName extends ExtractPlugin< + SeafowlPluginList, + DeferredTaskPlugin + >["__name"] + >( + _pluginName: PluginName, + ..._rest: Parameters< + ExtractPlugin< + SeafowlPluginList, + DeferredTaskPlugin + >["pollDeferredTask"] + > + ): Promise { + await Promise.resolve(); + return { + completed: false, + result: null, + }; + } + + async importData< + PluginName extends ExtractPlugin< + SeafowlPluginList, + ImportPlugin + >["__name"] + >( + pluginName: PluginName, ...rest: Parameters< - ExtractPlugin["importData"] + ExtractPlugin>["importData"] > ) { const [sourceOpts, destOpts] = rest; @@ -295,10 +329,11 @@ export class DbSeafowl plugin ): plugin is ExtractPlugin< SeafowlPluginList, - ImportPlugin & { __name: typeof pluginName } & Partial< - WithOptionsInterface - > - > => "importData" in Object.getPrototypeOf(plugin) + ImportPlugin & + Partial>> + > => + "importData" in Object.getPrototypeOf(plugin) && + plugin.__name === pluginName ) .pop(); diff --git a/packages/db-seafowl/package.json b/packages/db-seafowl/package.json index b3a360f..ba7ffca 100644 --- a/packages/db-seafowl/package.json +++ b/packages/db-seafowl/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/db-seafowl", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/db-seafowl/plugins/importers/index.ts b/packages/db-seafowl/plugins/importers/index.ts index 567790d..139d885 100644 --- a/packages/db-seafowl/plugins/importers/index.ts +++ b/packages/db-seafowl/plugins/importers/index.ts @@ -9,7 +9,7 @@ type DefaultPluginMap = { | { [k in DEFAULT_IMPORT_PLUGINS]: k extends "csv" | "parquet" ? SeafowlImportFilePlugin - : ImportPlugin; + : ImportPlugin; }; exporters: {}; }; diff --git a/packages/db-seafowl/plugins/importers/seafowl-import-file-plugin.ts b/packages/db-seafowl/plugins/importers/seafowl-import-file-plugin.ts index 132d6f1..dde8e23 100644 --- a/packages/db-seafowl/plugins/importers/seafowl-import-file-plugin.ts +++ b/packages/db-seafowl/plugins/importers/seafowl-import-file-plugin.ts @@ -24,7 +24,7 @@ type ImportFileOptions = ImportFileFromURLOptions; type DbInjectedOptions = Partial; export class SeafowlImportFilePlugin - implements ImportPlugin, WithOptionsInterface + implements ImportPlugin<"csv">, WithOptionsInterface { public readonly opts: Partial; private readonly seafowlClient?: Client; diff --git a/packages/db-splitgraph/db-splitgraph.test.ts b/packages/db-splitgraph/db-splitgraph.test.ts index e37246f..d19c071 100644 --- a/packages/db-splitgraph/db-splitgraph.test.ts +++ b/packages/db-splitgraph/db-splitgraph.test.ts @@ -3,9 +3,13 @@ import { randSuffix } from "@madatdata/test-helpers/rand-suffix"; import type { Expect, Equal } from "@madatdata/test-helpers/type-test-utils"; import { makeDb } from "./db-splitgraph"; import { SplitgraphImportCSVPlugin } from "./plugins/importers/splitgraph-import-csv-plugin"; -import { ExportQueryPlugin } from "./plugins/exporters/export-query-plugin"; +import { SplitgraphExportQueryToFilePlugin } from "./plugins/exporters/splitgraph-export-query-to-file-plugin"; -import { shouldSkipIntegrationTests } from "@madatdata/test-helpers/env-config"; +import { + shouldSkipExportFromSplitgraphToSeafowlIntegrationTests, + shouldSkipIntegrationTests, + shouldSkipIntegrationTestsForGitHubExternalDataSource, +} from "@madatdata/test-helpers/env-config"; import { setupMswServerTestHooks } from "@madatdata/test-helpers/msw-server-hooks"; import { setupMemo } from "@madatdata/test-helpers/setup-memo"; import { compose, graphql, rest, type DefaultBodyType } from "msw"; @@ -13,6 +17,8 @@ import { compose, graphql, rest, type DefaultBodyType } from "msw"; import { defaultHost } from "@madatdata/base-client"; import { faker } from "@faker-js/faker"; +import { SplitgraphAirbyteGithubImportPlugin } from "./plugins/importers/generated/airbyte-github/plugin"; +import { SplitgraphExportToSeafowlPlugin } from "./plugins/exporters/splitgraph-export-to-seafowl-plugin"; describe("importData", () => { it("returns false for unknown plugin", async () => { @@ -70,12 +76,16 @@ const createDb = () => { graphqlEndpoint: defaultHost.baseUrls.gql, transformRequestHeaders, - // NOTE: exportQuery is not mocked yet + // NOTE: not all plugins are fully mocked plugins: [ new SplitgraphImportCSVPlugin({ graphqlEndpoint: defaultHost.baseUrls.gql, transformRequestHeaders, }), + new SplitgraphAirbyteGithubImportPlugin({ + graphqlEndpoint: defaultHost.baseUrls.gql, + transformRequestHeaders, + }), ], }); }; @@ -108,13 +118,71 @@ const createRealDb = () => { graphqlEndpoint: defaultHost.baseUrls.gql, }), - new ExportQueryPlugin({ + new SplitgraphAirbyteGithubImportPlugin({ + graphqlEndpoint: defaultHost.baseUrls.gql, + }), + + new SplitgraphExportQueryToFilePlugin({ + graphqlEndpoint: defaultHost.baseUrls.gql, + }), + + new SplitgraphExportToSeafowlPlugin({ graphqlEndpoint: defaultHost.baseUrls.gql, }), ], }); }; +// @ts-expect-error https://stackoverflow.com/a/70711231 +const GITHUB_PAT_SECRET = import.meta.env.VITE_TEST_GITHUB_PAT_SECRET; + +describe.skipIf(shouldSkipIntegrationTestsForGitHubExternalDataSource())( + "importData for AirbyeGitHubImportPlugin", + () => { + it("can use the plugin", async () => { + const db = createRealDb(); + + const { username: namespace } = await fetchToken(db); + + // NOTE: not actually asserting anything here atm, and these tests + // should usually be skipped until we have a better way of integration + // testing that doesn't require spam-ingesting GitHub repos into Splitgraph, + // or at least has the capability to delete them afterward + // SEE: packages/test-helpers/env-config.ts for hardcoded skip logic + + // For now this is just a way to manually check everything is working + await db.importData( + "airbyte-github", + { + credentials: { + credentials: { + personal_access_token: GITHUB_PAT_SECRET, + }, + }, + params: { + repository: "splitgraph/seafowl", + start_date: "2021-06-01T00:00:00Z", + }, + }, + { + namespace: namespace, + repository: "madatdata-test-github-ingestion", + tables: [ + { + name: "stargazers", + options: { + airbyte_cursor_field: ["starred_at"], + airbyte_primary_key_field: [], + }, + schema: [], + }, + ], + } + ); + }, 60_000); + } +); + // Useful when writing initial tests against real server (where anon is allowed) // const _makeAnonymousDb = () => { // return makeDb({ @@ -566,11 +634,55 @@ describe("importData for SplitgraphImportCSVPlugin", () => { // TODO: Make a mocked version of this test describe.skipIf(shouldSkipIntegrationTests())("real export query", () => { - it("exports a basic postgres query to parquet", async () => { + // NOTE: test assumes that the task hasn't completed by the time we send the first check + it("deferred exports basic postgres query to parquet returns a taskId", async () => { + const db = createRealDb(); + const { + taskId, + response, + error: _e, + info, + } = await db.exportData( + "export-query-to-file", + // NOTE: Use a fairly big query (10,000 rows) so that it takes long enough to complete + // that when we check it's status for the first time, we can expect it to still be pending + // (with a low number of rows, this test sometimes failed since the task was complete when checking it) + { + query: `SELECT a as int_val, string_agg(random()::text, '') as text_val +FROM generate_series(1, 10000) a, generate_series(1, 50) b +GROUP BY a ORDER BY a;`, + vdbId: "ddn", + }, + { + format: "parquet", + filename: "random-series", + }, + { defer: true } + ); + + expect(typeof taskId).toBe("string"); + expect(taskId?.length).toEqual(36); + + expect(taskId).toBeDefined(); + + expect(response).toBeDefined(); + expect(info).toBeDefined(); + + const startedTask = await db.pollDeferredTask("export-query-to-file", { + taskId: taskId as string, + }); + + expect(startedTask.completed).toBe(false); + expect(startedTask.error).toBeNull(); + expect(startedTask.response?.status).toBe("STARTED"); + expect(startedTask.response!.exportFormat).toBe("parquet"); + }); + + it("exports a basic postgres query to parquet (and pollDeferredTask returns completed task)", async () => { const db = createRealDb(); const { response, error, info } = await db.exportData( - "exportQuery", + "export-query-to-file", { query: `SELECT a as int_val, string_agg(random()::text, '') as text_val FROM generate_series(1, 5) a, generate_series(1, 50) b @@ -615,9 +727,273 @@ GROUP BY a ORDER BY a;`, `); expect(error).toMatchInlineSnapshot("null"); + + // PIGGYBACK on this test to also test pollDeferredTask + // This is kind of cheating: we didn't initialize it as a deferred task, but + // we know that with the taskId, we can get the tatus of a deferred task. And + // we want to test that pollDeferredTask returns completed: true when a task + // has completed, and it's convenient to check that here since we know for + // sure that this task has completed (since we didn't defer it and therefore waited for it) + const shouldBeCompletedTask = await db.pollDeferredTask( + "export-query-to-file", + { taskId: response.taskId } + ); + + expect(shouldBeCompletedTask.completed).toBe(true); + expect(shouldBeCompletedTask.error).toBeNull(); + expect(shouldBeCompletedTask.response?.exportFormat).toBe("parquet"); + + const taskOutput = shouldBeCompletedTask.response?.output; + + expect(typeof taskOutput).toBe("object"); + expect("url" in (taskOutput as object)).toBe(true); + expect(typeof (taskOutput as { url: string })["url"]).toBe("string"); }, 30_000); }); +// @ts-expect-error https://stackoverflow.com/a/70711231 +const SEAFOWL_DEST_SECRET = import.meta.env + .VITE_TEST_SEAFOWL_EXPORT_DEST_SECRET; + +// @ts-expect-error https://stackoverflow.com/a/70711231 +const SEAFOWL_DEST_URL = import.meta.env.VITE_TEST_SEAFOWL_EXPORT_DEST_URL; + +// @ts-expect-error https://stackoverflow.com/a/70711231 +const SEAFOWL_DEST_DBNAME = import.meta.env + .VITE_TEST_SEAFOWL_EXPORT_DEST_DBNAME; + +describe.skipIf(shouldSkipExportFromSplitgraphToSeafowlIntegrationTests())( + "export from splitgraph to seafowl", + () => { + it("defers an export to seafowl", async () => { + const db = createRealDb(); + + const { username: splitgraphUsername } = await fetchToken(db); + + expect(splitgraphUsername).toEqual(SEAFOWL_DEST_DBNAME); + + // Should be _only the base URL_, like: https://demo.seafowl.cloud + expect(SEAFOWL_DEST_URL.endsWith("/")).toEqual(false); + expect(SEAFOWL_DEST_URL.endsWith("/q")).toEqual(false); + + const destTableSuffix = randSuffix(); + const destTable = `random_series_${destTableSuffix}`; + + const res = await db.exportData( + "export-to-seafowl", + { + queries: [ + { + source: { + query: `SELECT a as int_val, string_agg(random()::text, '') as text_val + FROM generate_series(1, 5) a, generate_series(1, 50) b + GROUP BY a ORDER BY a;`, + }, + destination: { + schema: "madatdata_testing", + table: destTable, + }, + }, + ], + }, + { + seafowlInstance: { + selfHosted: { + url: SEAFOWL_DEST_URL, + dbname: SEAFOWL_DEST_DBNAME, + secret: SEAFOWL_DEST_SECRET, + }, + }, + }, + { defer: true } + ); + + expect({ + queries: res.taskIds.queries.map((_: string) => "some query id"), + tables: res.taskIds.tables, + vdbs: res.taskIds.vdbs, + }).toMatchInlineSnapshot(` + { + "queries": [ + "some query id", + ], + "tables": [], + "vdbs": [], + } + `); + + expect(res.taskIds.queries.length).toEqual(1); + + const taskId = res.taskIds.queries[0].jobId; + + const startedTask = await db.pollDeferredTask("export-to-seafowl", { + taskId: taskId as string, + }); + + expect(startedTask.completed).toBe(false); + expect(startedTask.error).toBeNull(); + expect(startedTask.info).not.toBeNull(); + expect(startedTask.info?.jobStatus).not.toBeNull(); + expect(startedTask.info?.jobStatus?.status).toBe(200); + expect(startedTask.response?.status).toBe("STARTED"); + expect(startedTask.response?.exportFormat).toBe("sync"); + expect(typeof startedTask.response?.started).toBe("string"); + expect(startedTask.response?.finished).toBeNull(); + }, 20_000); + + it("exports a query to seafowl", async () => { + const db = createRealDb(); + + const { username: splitgraphUsername } = await fetchToken(db); + + expect(splitgraphUsername).toEqual(SEAFOWL_DEST_DBNAME); + + // Should be _only the base URL_, like: https://demo.seafowl.cloud + expect(SEAFOWL_DEST_URL.endsWith("/")).toEqual(false); + expect(SEAFOWL_DEST_URL.endsWith("/q")).toEqual(false); + + const destTableSuffix = randSuffix(); + const destTable = `random_series_${destTableSuffix}`; + + const queryToExport = `SELECT a as int_val, string_agg(random()::text, '') as text_val + FROM generate_series(1, 5) a, generate_series(1, 50) b + GROUP BY a ORDER BY a;`; + + const res = await db.exportData( + "export-to-seafowl", + { + queries: [ + { + source: { + query: queryToExport, + }, + destination: { + schema: "madatdata_testing", + table: destTable, + }, + }, + ], + }, + { + seafowlInstance: { + selfHosted: { + url: SEAFOWL_DEST_URL, + dbname: SEAFOWL_DEST_DBNAME, + secret: SEAFOWL_DEST_SECRET, + }, + }, + } + ); + + expect( + (({ error, info }: typeof res) => ({ + error, + info: { + allFailed: info.allFailed, + allPassed: info.allPassed, + somePassed: info.somePassed, + totalFailed: info.totalFailed, + totalPassed: info.totalPassed, + }, + }))(res) + ).toMatchInlineSnapshot(` + { + "error": null, + "info": { + "allFailed": false, + "allPassed": true, + "somePassed": false, + "totalFailed": 0, + "totalPassed": 1, + }, + } + `); + + expect(res.response.passedJobs.queries.length).toEqual(1); + expect(res.response.passedJobs.tables.length).toEqual(0); + expect(res.response.passedJobs.vdbs.length).toEqual(0); + + expect( + res.response.passedJobs.queries[0].result.response.exportFormat + ).toEqual("sync"); + expect(res.response.passedJobs.queries[0].result.response.status).toEqual( + "SUCCESS" + ); + expect( + res.response.passedJobs.queries[0].result.response.output.tables.length + ).toEqual(1); + + const exportedTableTuple = + res.response.passedJobs.queries[0].result.response.output.tables[0]; + + const [ + exportedQuery, + destDbname, + destSchema, + destTableName, + ingestedParquetURL, + ] = exportedTableTuple; + + expect(destTableName).toEqual(destTable); + expect(destDbname).toEqual(SEAFOWL_DEST_DBNAME); + + expect({ + exportedQuery, + destSchema, + }).toMatchInlineSnapshot(` + { + "destSchema": "madatdata_testing", + "exportedQuery": "SELECT a as int_val, string_agg(random()::text, '') as text_val + FROM generate_series(1, 5) a, generate_series(1, 50) b + GROUP BY a ORDER BY a", + } + `); + + expect(ingestedParquetURL.includes(".parquet")).toEqual(true); + + // PIGGYBACK on this test to also test pollDeferredTask case of a completd task + const shouldBeCompletedTask = await db.pollDeferredTask( + "export-to-seafowl", + { taskId: res.response.passedJobs.queries[0].job.jobId as string } + ); + + expect(shouldBeCompletedTask.completed).toBe(true); + expect(shouldBeCompletedTask.error).toBeNull(); + expect(shouldBeCompletedTask.info).not.toBeNull(); + expect(shouldBeCompletedTask.info?.jobStatus?.status).toBe(200); + expect(shouldBeCompletedTask.response?.exportFormat).toBe("sync"); + expect(shouldBeCompletedTask.response?.status).toBe("SUCCESS"); + expect(typeof shouldBeCompletedTask.response?.started).toBe("string"); + expect(typeof shouldBeCompletedTask.response?.finished).toBe("string"); + + const outputTable = ( + shouldBeCompletedTask.response?.output! as { tables: string[][] } + )["tables"][0]; + + const [ + inputQuery, + outputDbName, + outputSchema, + outputTableName, + intermediateParquetUrl, + ] = outputTable; + + expect(inputQuery).toEqual(exportedQuery); + expect(inputQuery).toMatchInlineSnapshot(` + "SELECT a as int_val, string_agg(random()::text, '') as text_val + FROM generate_series(1, 5) a, generate_series(1, 50) b + GROUP BY a ORDER BY a" + `); + expect(outputDbName).toEqual(destDbname); + expect(outputSchema).toEqual(destSchema); + expect(outputTableName).toEqual(destTableName); + expect(typeof intermediateParquetUrl).toBe("string"); + expect(intermediateParquetUrl.startsWith("https://")).toBe(true); + expect(intermediateParquetUrl.includes(".parquet")).toBe(true); + }, 60_000); + } +); + describe.skipIf(shouldSkipIntegrationTests())("real DDN", () => { it("uploads with TableParamsSchema semicolon delimiter", async () => { const db = createRealDb(); @@ -641,6 +1017,84 @@ describe.skipIf(shouldSkipIntegrationTests())("real DDN", () => { expect(info?.jobStatus.status).toEqual("SUCCESS"); expect(info?.jobLog?.url.includes(info.jobStatus.taskId)).toBe(true); + + // PIGGYBACK on this test to also test pollDeferredTask (just like with export) + // We wouldn't normally do this since we didn't defer the task and have already + // awaited it, but since we know it's complete we can conveniently check the + // test case of pollDeferredTask returning a completed task + const shouldBeCompletedTask = await db.pollDeferredTask("csv", { + // This is the hacky part, note that this didn't come from the return value + taskId: info.jobStatus.taskId, + namespace, + repository: "dunno", + }); + + expect(shouldBeCompletedTask.completed).toBe(true); + expect(shouldBeCompletedTask.error).toBeNull(); + expect(shouldBeCompletedTask.info?.jobStatus).not.toBeNull(); + expect(shouldBeCompletedTask.info?.jobLog).not.toBeNull(); + + expect(shouldBeCompletedTask.response).not.toBeNull(); + expect(typeof shouldBeCompletedTask.response?.jobLog?.url).toEqual( + "string" + ); + expect(shouldBeCompletedTask.response?.jobStatus?.status).toEqual( + "SUCCESS" + ); + expect(shouldBeCompletedTask.response?.jobStatus?.taskId).toEqual( + info.jobStatus.taskId + ); + expect(shouldBeCompletedTask.response?.jobStatus?.isManual).toEqual(true); + expect(typeof shouldBeCompletedTask.response?.jobStatus?.finished).toEqual( + "string" + ); + expect(typeof shouldBeCompletedTask.response?.jobStatus?.started).toEqual( + "string" + ); + }, 20_000); + + // NOTE: test assumes that the task hasn't completed by the time we send the first check + it("upload starts a deferred task", async () => { + const db = createRealDb(); + const { username: namespace } = await fetchToken(db); + + const { response, info, taskId } = await db.importData( + "csv", + { data: Buffer.from(`name;candies\r\nBob;5\r\nAlice;10`) }, + { + tableName: `irrelevant-${randSuffix()}`, + namespace, + repository: "dunno", + tableParams: { + delimiter: ";", + }, + }, + { defer: true } + ); + + expect(typeof taskId).toBe("string"); + expect(taskId?.length).toEqual(36); + + expect(taskId).toBeDefined(); + + expect(response).toBeDefined(); + expect(info).toBeDefined(); + + const startedTask = await db.pollDeferredTask("csv", { + taskId: taskId as string, + namespace, + repository: "dunno", + }); + + expect(startedTask.completed).toBe(false); + expect(startedTask.error).toBeNull(); + expect(startedTask.info).toBeNull(); + expect(startedTask.response?.jobStatus?.status).toBe("STARTED"); + expect(startedTask.response?.jobStatus?.finished).toBeNull(); + expect(startedTask.response?.jobStatus?.taskId).toEqual(taskId); + expect(typeof startedTask.response?.jobStatus?.started).toEqual("string"); + expect(startedTask.response?.jobStatus?.finished).toBeNull(); + expect(startedTask.response?.jobStatus?.isManual).toEqual(true); }, 20_000); }); describe("makeFakeJwt and claimsFromJwt", () => { diff --git a/packages/db-splitgraph/db-splitgraph.ts b/packages/db-splitgraph/db-splitgraph.ts index c82d40e..458d85c 100644 --- a/packages/db-splitgraph/db-splitgraph.ts +++ b/packages/db-splitgraph/db-splitgraph.ts @@ -12,7 +12,7 @@ import { // TODO: These could be injected in the constructor as the actual plugin map import { SplitgraphImportCSVPlugin } from "./plugins/importers/splitgraph-import-csv-plugin"; -import { ExportQueryPlugin } from "./plugins/exporters/export-query-plugin"; +import { SplitgraphExportQueryToFilePlugin } from "./plugins/exporters/splitgraph-export-query-to-file-plugin"; // TODO: It's not ideal for db-splitgraph to depend on base-client import { @@ -30,6 +30,11 @@ import { skipTransformFetchOptions, } from "@madatdata/client-http"; import type { GraphQLClientOptions } from "./plugins"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; + +// TODO: dont hardcode this, fix initialization interface for plugins +import { SplitgraphAirbyteGithubImportPlugin } from "./plugins/importers/generated/airbyte-github/plugin"; +import { SplitgraphExportToSeafowlPlugin } from "./plugins/exporters/splitgraph-export-to-seafowl-plugin"; interface DbSplitgraphPluginHostContext extends GraphQLClientOptions {} @@ -50,7 +55,7 @@ const makeTransformRequestHeadersForAuthenticatedRequest = export type DefaultSplitgraphPluginList = ( | SplitgraphImportCSVPlugin - | ExportQueryPlugin + | SplitgraphExportQueryToFilePlugin )[]; export const makeDefaultPluginList = ( @@ -72,7 +77,9 @@ export const makeDefaultPluginList = ( return [ new SplitgraphImportCSVPlugin({ ...graphqlOptions }), - new ExportQueryPlugin({ ...graphqlOptions }), + new SplitgraphAirbyteGithubImportPlugin({ ...graphqlOptions }), + new SplitgraphExportQueryToFilePlugin({ ...graphqlOptions }), + new SplitgraphExportToSeafowlPlugin({ ...graphqlOptions }), ]; }; @@ -284,13 +291,21 @@ export class DbSplitgraph }; } - async exportData( - pluginName: ExtractPlugin["__name"], + async exportData< + PluginName extends ExtractPlugin< + SplitgraphPluginList, + ExportPlugin + >["__name"] + >( + pluginName: PluginName, ...rest: Parameters< - ExtractPlugin["exportData"] + ExtractPlugin< + SplitgraphPluginList, + ExportPlugin + >["exportData"] > ) { - const [sourceOpts, destOpts] = rest; + const [sourceOpts, destOpts, exportOpts] = rest; const plugin = this.plugins .selectMatchingPlugins( @@ -298,10 +313,12 @@ export class DbSplitgraph plugin ): plugin is ExtractPlugin< SplitgraphPluginList, - ExportPlugin & { __name: typeof pluginName } & Partial< - WithOptionsInterface - > - > => "exportData" in Object.getPrototypeOf(plugin) + ExportPlugin & { + __name: typeof pluginName; + } & Partial>> + > => + "exportData" in Object.getPrototypeOf(plugin) && + plugin.__name === pluginName ) .pop(); @@ -313,29 +330,39 @@ export class DbSplitgraph throw new Error("plugin does not implement withOptions"); } - return await plugin - .withOptions({ - ...this.pluginConfig, - ...plugin, - transformRequestHeaders: (headers: HeadersInit) => - ( - (plugin as PluginWithTransformRequestHeadersOption) - .transformRequestHeaders ?? IdentityFunc - )(this.pluginConfig.transformRequestHeaders(headers)), - }) - .exportData(sourceOpts, destOpts); + const instantiatedPlugin = plugin.withOptions({ + ...this.pluginConfig, + ...plugin, + transformRequestHeaders: (headers: HeadersInit) => + ( + (plugin as PluginWithTransformRequestHeadersOption) + .transformRequestHeaders ?? IdentityFunc + )(this.pluginConfig.transformRequestHeaders(headers)), + }); + + return exportOpts + ? await instantiatedPlugin.exportData(sourceOpts, destOpts, exportOpts) + : await instantiatedPlugin.exportData(sourceOpts, destOpts); } - async importData( - pluginName: ExtractPlugin["__name"], + async importData< + PluginName extends ExtractPlugin< + SplitgraphPluginList, + ImportPlugin + >["__name"] + >( + pluginName: PluginName, ...rest: Parameters< - ExtractPlugin["importData"] + ExtractPlugin< + SplitgraphPluginList, + ImportPlugin + >["importData"] > ) { // TODO: type error in ...rest // this.plugins.callFunction(pluginName, "importData", ...rest); - const [sourceOpts, destOpts] = rest; + const [sourceOpts, destOpts, importOpts] = rest; const plugin = this.plugins .selectMatchingPlugins( @@ -343,10 +370,12 @@ export class DbSplitgraph plugin ): plugin is ExtractPlugin< SplitgraphPluginList, - ImportPlugin & { + ImportPlugin & { __name: typeof pluginName; - } & Partial> - > => "importData" in Object.getPrototypeOf(plugin) + } & Partial>> + > => + "importData" in Object.getPrototypeOf(plugin) && + plugin.__name === pluginName ) .pop(); @@ -358,7 +387,73 @@ export class DbSplitgraph throw new Error("plugin does not implement withOptions"); } - return await plugin + const instantiatedPlugin = plugin.withOptions({ + ...this.pluginConfig, + ...plugin, + transformRequestHeaders: (headers: HeadersInit) => + ( + (plugin as PluginWithTransformRequestHeadersOption) + .transformRequestHeaders ?? IdentityFunc + )(this.pluginConfig.transformRequestHeaders(headers)), + }); + + return importOpts + ? await instantiatedPlugin.importData(sourceOpts, destOpts, importOpts) + : await instantiatedPlugin.importData(sourceOpts, destOpts); + } + + async pollDeferredTask< + PluginName extends ExtractPlugin< + SplitgraphPluginList, + DeferredTaskPlugin + >["__name"], + MatchingPlugin extends ExtractPlugin< + SplitgraphPluginList, + // discriminate on __name to avoid including return type of every plugin with pollDeferredTask + { __name: PluginName } & DeferredTaskPlugin + >, + DeferredTaskResponse extends Awaited< + ReturnType + > + >( + pluginName: PluginName, + ...rest: Parameters< + ExtractPlugin< + SplitgraphPluginList, + DeferredTaskPlugin + >["pollDeferredTask"] + > + ) { + const plugin = this.plugins + .selectMatchingPlugins( + ( + plugin + ): plugin is ExtractPlugin< + SplitgraphPluginList, + DeferredTaskPlugin & { + __name: typeof pluginName; + } & Partial< + WithOptionsInterface< + DeferredTaskPlugin + > + > + > => + "pollDeferredTask" in Object.getPrototypeOf(plugin) && + plugin.__name === pluginName + ) + .pop(); + + if (!plugin) { + throw new Error(`Plugin not found: ${pluginName}`); + } + + if (!plugin.withOptions) { + throw new Error("plugin does not implement withOptions"); + } + + const [memoizedDeferredTask] = rest; + + const deferredTask = await plugin .withOptions({ ...this.pluginConfig, ...plugin, @@ -368,7 +463,9 @@ export class DbSplitgraph .transformRequestHeaders ?? IdentityFunc )(this.pluginConfig.transformRequestHeaders(headers)), }) - .importData(sourceOpts, destOpts); + .pollDeferredTask(memoizedDeferredTask); + + return deferredTask; } } diff --git a/packages/db-splitgraph/package.json b/packages/db-splitgraph/package.json index 29c3e25..4b6c4e6 100644 --- a/packages/db-splitgraph/package.json +++ b/packages/db-splitgraph/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/db-splitgraph", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", @@ -14,16 +14,16 @@ }, "devDependencies": { "@faker-js/faker": "7.3.0", - "@graphql-codegen/cli": "2.8.0", - "@graphql-codegen/near-operation-file-preset": "2.3.1", - "@graphql-codegen/schema-ast": "2.5.0", - "@graphql-codegen/typescript": "2.7.1", - "@graphql-codegen/typescript-operations": "2.5.1", + "@graphql-codegen/cli": "3.3.1", + "@graphql-codegen/near-operation-file-preset": "2.5.0", + "@graphql-codegen/schema-ast": "3.0.1", + "@graphql-codegen/typescript": "3.0.4", + "@graphql-codegen/typescript-operations": "3.0.4", "@madatdata/test-helpers": "workspace:*", "@types/node": "18.7.13", "cross-fetch": "3.1.5", "esno": "0.16.3", - "json-schema-to-typescript": "10.1.5", + "json-schema-to-typescript": "13.0.1", "rimraf": "3.0.2" }, "exports": { diff --git a/packages/db-splitgraph/plugins/exporters/export-query-plugin.ts b/packages/db-splitgraph/plugins/exporters/export-query-plugin.ts deleted file mode 100644 index 9d8ecd1..0000000 --- a/packages/db-splitgraph/plugins/exporters/export-query-plugin.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { ExportPlugin, WithOptionsInterface } from "@madatdata/base-db"; -import { SplitgraphGraphQLClient } from "../../gql-client/splitgraph-graphql-client"; -import { - ExportFormat, - type ExportJobOutput, -} from "../../gql-client/unified-types"; -import { Retryable, BackOffPolicy } from "typescript-retry-decorator"; - -import { gql } from "graphql-request"; -import type { - StartExportJobMutation, - StartExportJobMutationVariables, - ExportJobStatusQuery, - ExportJobStatusQueryVariables, -} from "./export-query-plugin.generated"; - -type ExportQuerySourceOptions = { - /** - * The raw SQL query for which to export the result - */ - query: string; - /** - * Run export in the context of vdb with this vdbId (default: ddn) - */ - vdbId?: string; -}; - -type ExportQueryDestOptions = { - format: "csv" | "parquet"; - filename?: string; -}; - -// from unified spec -// type _ExportJobStatus = "status" | "finished" | "output" | "started"; - -interface ExportQueryPluginOptions { - graphqlEndpoint: string; - transformRequestHeaders?: (requestHeaders: HeadersInit) => HeadersInit; -} -type DbInjectedOptions = Partial; - -// 1 hour -const MAX_POLL_TIMEOUT = 1_000 * 60 * 60; -// const MAX_ATTEMPTS = MAX_POLL_TIMEOUT - (25.5 * 1000) / 10000; -const MAX_BACKOFF_INTERVAL = 10_000; -const MAX_ATTEMPTS = Math.ceil( - (MAX_POLL_TIMEOUT - 25.5 * 1_000) / MAX_BACKOFF_INTERVAL -); -const retryOptions = { - maxAttempts: MAX_ATTEMPTS, - backOff: 500, - backOffPolicy: BackOffPolicy.ExponentialBackOffPolicy, - exponentialOption: { maxInterval: MAX_BACKOFF_INTERVAL, multiplier: 2 }, -}; -export class ExportQueryPlugin - implements ExportPlugin, WithOptionsInterface -{ - private readonly opts: ExportQueryPluginOptions; - public readonly graphqlEndpoint: ExportQueryPluginOptions["graphqlEndpoint"]; - public readonly graphqlClient: SplitgraphGraphQLClient; - public readonly transformRequestHeaders: Required["transformRequestHeaders"]; - - public readonly __name = "exportQuery"; - public static readonly __name = "exportQuery"; - - constructor(opts: ExportQueryPluginOptions) { - this.opts = opts; - - this.graphqlEndpoint = opts.graphqlEndpoint; - this.transformRequestHeaders = opts.transformRequestHeaders ?? IdentityFunc; - - this.graphqlClient = new SplitgraphGraphQLClient({ - graphqlEndpoint: this.graphqlEndpoint, - transformRequestHeaders: this.transformRequestHeaders, - }); - } - - // TODO: DRY with other plugins - withOptions(injectOpts: DbInjectedOptions) { - return new ExportQueryPlugin({ - ...this.opts, - ...injectOpts, - // TODO: replace transformer with some kind of chainable "link" plugin - transformRequestHeaders: (reqHeaders) => { - const withOriginal = { - ...reqHeaders, - ...this.opts.transformRequestHeaders?.(reqHeaders), - }; - - const withNext = { - ...withOriginal, - ...injectOpts.transformRequestHeaders?.(withOriginal), - }; - - return { - ...withOriginal, - ...withNext, - }; - }, - }); - } - - async exportData( - sourceOptions: ExportQuerySourceOptions, - destOptions: ExportQueryDestOptions - ) { - const { - response: exportResponse, - error: exportError, - info: exportInfo, - } = await this.startExport(sourceOptions, destOptions); - - if (exportError || !exportResponse || !exportResponse.exportQuery.id) { - return { - response: null, - error: exportError, - info: { ...exportInfo }, - }; - } - const { id: taskId, filename } = exportResponse.exportQuery; - - const { - response: taskResponse, - error: taskError, - info: taskInfo, - } = await this.waitForTask(taskId); - - // TODO: use superstruct or something to verify JSON is as expected - const parsedOutput = taskResponse?.output as ExportJobOutput; - if (!parsedOutput.url) { - throw new Error("output did not include url field as suspected"); - } - - return { - response: { - success: true, - taskId, - filename, - // Let taskResponse fields take highest priority, except for output, - // where we prefer our parsed (and typed) version to the raw JSON - ...taskResponse, - output: parsedOutput, - }, - error: taskError, - info: taskInfo, - }; - } - - private async startExport( - sourceOptions: ExportQuerySourceOptions, - destOptions: ExportQueryDestOptions - ) { - return this.graphqlClient.send< - StartExportJobMutation, - StartExportJobMutationVariables - >( - gql` - mutation StartExportJob( - $query: String! - $format: ExportFormat! - $filename: String! - $vdbId: String = null - ) { - exportQuery( - query: $query - format: $format - filename: $filename - vdbId: $vdbId - ) { - filename - id - } - } - `, - { - query: sourceOptions.query, - format: (() => { - switch (destOptions.format) { - case "csv": - return ExportFormat.Csv; - case "parquet": - return ExportFormat.Parquet; - } - })(), - filename: destOptions.filename ?? "exported-query", - vdbId: sourceOptions.vdbId, - } - ); - } - - // TODO: DRY (with at least splitgraph-import-csv-plugin) - @Retryable({ - ...retryOptions, - doRetry: ({ type }) => type === "retry", - }) - private async waitForTask(taskId: string) { - const { - response: jobStatusResponse, - error: jobStatusError, - info: jobStatusInfo, - } = await this.fetchExportJobStatus(taskId); - - if (jobStatusError) { - return { - response: null, - error: jobStatusError, - info: { jobStatus: jobStatusInfo }, - }; - } else if (!jobStatusResponse) { - throw { type: "retry" }; - // FIXME(codegen): this shouldn't be nullable - } else if (taskUnresolved(jobStatusResponse.status as ExportTaskStatus)) { - throw { type: "retry" }; - } - - return { - response: jobStatusResponse, - error: jobStatusError, - info: jobStatusInfo, - }; - } - - private async fetchExportJobStatus(taskId: string) { - const { response, error, info } = await this.graphqlClient.send< - ExportJobStatusQuery, - ExportJobStatusQueryVariables - >( - gql` - query ExportJobStatus($taskId: UUID!) { - exportJobStatus(taskId: $taskId) { - status - started - finished - exportFormat - output - } - } - `, - { - taskId: taskId, - } - ); - - if (error || !response) { - return { response: null, error, info }; - } - - return { - response: response.exportJobStatus, - error: null, - info, - }; - } -} - -const IdentityFunc = (x: T) => x; - -enum ExportTaskStatus { - // Standard Celery statuses - Pending = "PENDING", - Started = "STARTED", - Success = "SUCCESS", - Failure = "FAILURE", - Revoked = "REVOKED", - - // Custom Splitgraph statuses - Lost = "LOST", - TimedOut = "TIMED_OUT", - - // Currently unused statuses - Retry = "RETRY", - Received = "RECEIVED", - Rejected = "REJECTED", - Ignored = "IGNORED", -} - -const standbyStatuses = [ExportTaskStatus.Pending, ExportTaskStatus.Started]; - -const taskUnresolved = (ts: ExportTaskStatus) => standbyStatuses.includes(ts); diff --git a/packages/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin.ts b/packages/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin.ts new file mode 100644 index 0000000..1a0301a --- /dev/null +++ b/packages/db-splitgraph/plugins/exporters/splitgraph-base-export-plugin.ts @@ -0,0 +1,279 @@ +import type { ExportPlugin, WithOptionsInterface } from "@madatdata/base-db"; +import { SplitgraphGraphQLClient } from "../../gql-client/splitgraph-graphql-client"; +import { Retryable, BackOffPolicy } from "typescript-retry-decorator"; + +import { gql } from "graphql-request"; +import type { + ExportJobStatusQuery, + ExportJobStatusQueryVariables, +} from "./splitgraph-base-export-plugin.generated"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; + +// from unified spec +// type _ExportJobStatus = "status" | "finished" | "output" | "started"; + +interface SplitgraphExportPluginOptions { + graphqlEndpoint: string; + transformRequestHeaders?: (requestHeaders: HeadersInit) => HeadersInit; +} +type DbInjectedOptions = Partial; + +// 1 hour +const MAX_POLL_TIMEOUT = 1_000 * 60 * 60; +// const MAX_ATTEMPTS = MAX_POLL_TIMEOUT - (25.5 * 1000) / 10000; +const MAX_BACKOFF_INTERVAL = 10_000; +const MAX_ATTEMPTS = Math.ceil( + (MAX_POLL_TIMEOUT - 25.5 * 1_000) / MAX_BACKOFF_INTERVAL +); +const retryOptions = { + maxAttempts: MAX_ATTEMPTS, + backOff: 500, + backOffPolicy: BackOffPolicy.ExponentialBackOffPolicy, + exponentialOption: { maxInterval: MAX_BACKOFF_INTERVAL, multiplier: 2 }, +}; + +export type DeferredSplitgraphExportTask = { + completed: boolean; + response: Extract | null; + error: "no response" | "failed status" | null | any; + info: { jobStatus: { status: number; headers: any } | null } | null; +}; + +export abstract class SplitgraphExportPlugin< + PluginName extends string, + /** Concrete type of the derived class, for annotating return value of builder methods like withOptions */ + DerivedSplitgraphExportPlugin extends SplitgraphExportPlugin< + PluginName, + DerivedSplitgraphExportPlugin, + ConcreteExportSourceOptions, + ConcreteExportDestOptions, + StartedExportJob, + CompletedExportJob + >, + ConcreteExportSourceOptions extends object, + ConcreteExportDestOptions extends object, + StartedExportJob extends object, + CompletedExportJob extends Awaited< + ReturnType["exportData"]> + > +> implements + ExportPlugin, + DeferredTaskPlugin, + WithOptionsInterface +{ + public abstract readonly __name: PluginName; + + private readonly opts: SplitgraphExportPluginOptions; + public readonly graphqlEndpoint: SplitgraphExportPluginOptions["graphqlEndpoint"]; + public readonly graphqlClient: SplitgraphGraphQLClient; + public readonly transformRequestHeaders: Required["transformRequestHeaders"]; + + // TODO: make sense? will be overridden? + // public static readonly __name: PluginName; + + constructor(opts: SplitgraphExportPluginOptions) { + this.opts = opts; + + this.graphqlEndpoint = opts.graphqlEndpoint; + this.transformRequestHeaders = opts.transformRequestHeaders ?? IdentityFunc; + + this.graphqlClient = new SplitgraphGraphQLClient({ + graphqlEndpoint: this.graphqlEndpoint, + transformRequestHeaders: this.transformRequestHeaders, + }); + } + + // TODO: DRY with other plugins + withOptions(injectOpts: DbInjectedOptions): DerivedSplitgraphExportPlugin { + const mergedInjectOpts: SplitgraphExportPluginOptions = { + ...this.opts, + ...injectOpts, + transformRequestHeaders: (reqHeaders) => { + const withOriginal = { + ...reqHeaders, + ...this.opts.transformRequestHeaders?.(reqHeaders), + }; + + const withNext = { + ...withOriginal, + ...injectOpts.transformRequestHeaders?.(withOriginal), + }; + + return { + ...withOriginal, + ...withNext, + }; + }, + }; + + return new (Object.getPrototypeOf(this).constructor)(mergedInjectOpts); + } + + public abstract exportData( + sourceOptions: ConcreteExportSourceOptions, + destOptions: ConcreteExportDestOptions + ): Promise; + + public abstract exportData( + sourceOptions: ConcreteExportSourceOptions, + destOptions: ConcreteExportDestOptions, + exportOptions: { defer: Deferred } + ): Deferred extends true + ? Promise<{ taskId: string | null } & StartedExportJob> + : Promise; + + protected abstract startExport( + sourceOptions: ConcreteExportSourceOptions, + destOptions: ConcreteExportDestOptions + ): Promise; + + public async pollDeferredTask({ + taskId, + }: { + taskId: string; + }): Promise { + const { + response: jobStatusResponse, + error: jobStatusError, + info: jobStatusInfo, + } = await this.fetchExportJobStatus(taskId); + + if (jobStatusError) { + return { + completed: true, + response: null, + error: jobStatusError, + info: { jobStatus: jobStatusInfo }, + }; + } else if (!jobStatusResponse) { + return { + completed: false, + response: null, + error: "no response", + info: { jobStatus: jobStatusInfo }, + }; + } else if (taskUnresolved(jobStatusResponse.status as ExportTaskStatus)) { + return { + completed: false, + response: jobStatusResponse, + error: null, + info: { jobStatus: jobStatusInfo }, + }; + } else if (taskFailed(jobStatusResponse.status as ExportTaskStatus)) { + return { + completed: true, + response: jobStatusResponse, + error: "failed status", + info: { jobStatus: jobStatusInfo }, + }; + } + return { + completed: true, + response: jobStatusResponse, + error: jobStatusError, + info: { jobStatus: jobStatusInfo }, + }; + } + + // TODO: DRY (with at least splitgraph-import-csv-plugin) + // TODO: use pollDeferredTask (is it equivalent?) + @Retryable({ + ...retryOptions, + doRetry: ({ type }) => type === "retry", + }) + protected async waitForTask(taskId: string) { + const { + response: jobStatusResponse, + error: jobStatusError, + info: jobStatusInfo, + } = await this.fetchExportJobStatus(taskId); + + if (jobStatusError) { + return { + response: null, + error: jobStatusError, + info: { jobStatus: jobStatusInfo }, + }; + } else if (!jobStatusResponse) { + throw { type: "retry" }; + // FIXME(codegen): this shouldn't be nullable + } else if (taskUnresolved(jobStatusResponse.status as ExportTaskStatus)) { + throw { type: "retry" }; + } + + return { + response: jobStatusResponse, + error: jobStatusError, + info: jobStatusInfo, + }; + } + + private async fetchExportJobStatus(taskId: string) { + const { response, error, info } = await this.graphqlClient.send< + ExportJobStatusQuery, + ExportJobStatusQueryVariables + >( + gql` + query ExportJobStatus($taskId: UUID!) { + exportJobStatus(taskId: $taskId) { + status + started + finished + exportFormat + output + } + } + `, + { + taskId: taskId, + } + ); + + if (error || !response) { + return { response: null, error, info }; + } + + return { + response: response.exportJobStatus, + error: null, + info, + }; + } +} + +const IdentityFunc = (x: T) => x; + +enum ExportTaskStatus { + // Standard Celery statuses + Pending = "PENDING", + Started = "STARTED", + Success = "SUCCESS", + Failure = "FAILURE", + Revoked = "REVOKED", + + // Custom Splitgraph statuses + Lost = "LOST", + TimedOut = "TIMED_OUT", + + // Currently unused statuses + Retry = "RETRY", + Received = "RECEIVED", + Rejected = "REJECTED", + Ignored = "IGNORED", +} + +const failedStatuses = [ + ExportTaskStatus.Failure, + ExportTaskStatus.Revoked, + ExportTaskStatus.Lost, + ExportTaskStatus.TimedOut, + ExportTaskStatus.Retry, + ExportTaskStatus.Rejected, + ExportTaskStatus.Ignored, +]; + +const taskFailed = (ts: ExportTaskStatus) => failedStatuses.includes(ts); + +const standbyStatuses = [ExportTaskStatus.Pending, ExportTaskStatus.Started]; + +const taskUnresolved = (ts: ExportTaskStatus) => standbyStatuses.includes(ts); diff --git a/packages/db-splitgraph/plugins/exporters/splitgraph-export-query-to-file-plugin.ts b/packages/db-splitgraph/plugins/exporters/splitgraph-export-query-to-file-plugin.ts new file mode 100644 index 0000000..16b8f6a --- /dev/null +++ b/packages/db-splitgraph/plugins/exporters/splitgraph-export-query-to-file-plugin.ts @@ -0,0 +1,148 @@ +import type { ExportPlugin, WithOptionsInterface } from "@madatdata/base-db"; +import { + ExportFormat, + type ExportJobOutput, +} from "../../gql-client/unified-types"; + +import { gql } from "graphql-request"; +import type { + StartExportJobMutation, + StartExportJobMutationVariables, +} from "./splitgraph-export-query-to-file-plugin.generated"; +import { + DeferredSplitgraphExportTask, + SplitgraphExportPlugin, +} from "./splitgraph-base-export-plugin"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; + +type ExportQuerySourceOptions = { + /** + * The raw SQL query for which to export the result + */ + query: string; + /** + * Run export in the context of vdb with this vdbId (default: ddn) + */ + vdbId?: string; +}; + +type ExportQueryDestOptions = { + format: "csv" | "parquet"; + filename?: string; +}; + +export class SplitgraphExportQueryToFilePlugin + extends SplitgraphExportPlugin< + "export-query-to-file", + SplitgraphExportQueryToFilePlugin, + ExportQuerySourceOptions, + ExportQueryDestOptions, + Record, + Awaited["exportData"]>> + > + implements + ExportPlugin<"export-query-to-file">, + DeferredTaskPlugin<"export-query-to-file", DeferredSplitgraphExportTask>, + WithOptionsInterface +{ + public readonly __name = "export-query-to-file"; + public static readonly __name = "export-query-to-file"; + + async exportData( + sourceOptions: ExportQuerySourceOptions, + destOptions: ExportQueryDestOptions, + exportOptions?: { defer: boolean } + ) { + const { + response: exportResponse, + error: exportError, + info: exportInfo, + } = await this.startExport(sourceOptions, destOptions); + + if (exportOptions?.defer) { + return { + taskId: exportResponse?.exportQuery.id ?? null, + response: exportResponse, + error: exportError, + info: exportInfo, + }; + } + + if (exportError || !exportResponse || !exportResponse.exportQuery.id) { + return { + response: null, + error: exportError, + info: { ...exportInfo }, + }; + } + const { id: taskId, filename } = exportResponse.exportQuery; + + const { + response: taskResponse, + error: taskError, + info: taskInfo, + } = await this.waitForTask(taskId); + + // TODO: use superstruct or something to verify JSON is as expected + const parsedOutput = taskResponse?.output as ExportJobOutput; + if (!parsedOutput.url) { + throw new Error("output did not include url field as suspected"); + } + + return { + response: { + success: true, + taskId, + filename, + // Let taskResponse fields take highest priority, except for output, + // where we prefer our parsed (and typed) version to the raw JSON + ...taskResponse, + output: parsedOutput, + }, + error: taskError, + info: taskInfo, + }; + } + + protected async startExport( + sourceOptions: ExportQuerySourceOptions, + destOptions: ExportQueryDestOptions + ) { + return this.graphqlClient.send< + StartExportJobMutation, + StartExportJobMutationVariables + >( + gql` + mutation StartExportJob( + $query: String! + $format: ExportFormat! + $filename: String! + $vdbId: String = null + ) { + exportQuery( + query: $query + format: $format + filename: $filename + vdbId: $vdbId + ) { + filename + id + } + } + `, + { + query: sourceOptions.query, + format: (() => { + switch (destOptions.format) { + case "csv": + return ExportFormat.Csv; + case "parquet": + return ExportFormat.Parquet; + } + })(), + filename: destOptions.filename ?? "exported-query", + vdbId: sourceOptions.vdbId, + } + ); + } +} diff --git a/packages/db-splitgraph/plugins/exporters/splitgraph-export-to-seafowl-plugin.ts b/packages/db-splitgraph/plugins/exporters/splitgraph-export-to-seafowl-plugin.ts new file mode 100644 index 0000000..badcbac --- /dev/null +++ b/packages/db-splitgraph/plugins/exporters/splitgraph-export-to-seafowl-plugin.ts @@ -0,0 +1,281 @@ +import { gql } from "graphql-request"; +import type { ExportPlugin } from "@madatdata/base-db/base-db"; +import { SplitgraphExportPlugin } from "./splitgraph-base-export-plugin"; +import type { WithOptionsInterface } from "@madatdata/base-db/plugin-bindings"; +import type { + SeafowlExportQuerySourceDestinationTupleInput, + SeafowlExportTableSourceDestinationTupleInput, + SeafowlExportVdbSourceDestinationTupleInput, + SeafowlInstanceExportInput, +} from "../../gql-client/unified-types"; +import type { + StartExportToSeafowlJobMutation, + StartExportToSeafowlJobMutationVariables, +} from "./splitgraph-export-to-seafowl-plugin.generated"; + +// TODO: manually expand these types so we can add more informative typedocs to them +type ExportToSeafowlSourceOptions = { + tables?: ExpandRecursively[]; + queries?: ExpandRecursively[]; + vdbs?: ExpandRecursively[]; +}; + +type ExportToSeafowlDestOptions = { + seafowlInstance?: ExpandRecursively; +}; + +type JobResult = Awaited< + ReturnType<(typeof SplitgraphExportPlugin)["prototype"]["waitForTask"]> +>; + +export class SplitgraphExportToSeafowlPlugin + extends SplitgraphExportPlugin< + "export-to-seafowl", + SplitgraphExportToSeafowlPlugin, + ExportToSeafowlSourceOptions, + ExportToSeafowlDestOptions, + Record, + Awaited["exportData"]>> + > + implements + ExportPlugin<"export-to-seafowl">, + WithOptionsInterface +{ + public readonly __name = "export-to-seafowl"; + public static readonly __name = "export-to-seafowl"; + + /** + * Start the export job, and wait for all tasks to complete. If exportOptions.defer + * is set to true, then the task IDs will be returned instead of waiting for the job, + * and each will need to be checked separately with pollDeferredTask. + * + * Note that the `taskId` return will always be undefined, and the `taskIds` property + * will be set instead, with `taskIds{tables,queries,vdbs}.map(job => job.id)` + */ + async exportData( + sourceOptions: ExportToSeafowlSourceOptions, + destOptions: ExportToSeafowlDestOptions, + exportOptions?: { defer: boolean } + ) { + const { + response: startExportResponse, + error: startExportError, + info: startExportInfo, + } = await this.startExport(sourceOptions, destOptions); + + // Bail out if error, or response missing, or any fields missing from response + // Note that the fields might be empty lists, but we still expect them to be set + if ( + startExportError || + !startExportResponse || + !startExportResponse.exportToSeafowl.queries || + // NOTE: .tables can be null, whereas all the others are always lists (even if empty) + typeof startExportResponse.exportToSeafowl.tables === "undefined" || + !startExportResponse.exportToSeafowl.vdbs + ) { + return { + response: startExportResponse ?? null, + error: startExportError ?? null, + info: { + ...startExportInfo, + }, + }; + } + + const { + queries: queryExportJobs, + tables: tableExportMultiJob, + vdbs: vdbExportJobs, + } = startExportResponse.exportToSeafowl; + + // NOTE: As an efficiency measure, Splitgraph combines table exports into a single job, + // so the returned shape is slightly different from queries and vdbs (which each have a job for each) + const { jobId: tableExportJobId, tables: tableExportTablesDetails } = + tableExportMultiJob ?? { jobId: undefined, tables: [] }; + + // FIXME: These types are returned as any. Need some generic param passing + if (exportOptions?.defer) { + return { + taskIds: { + queries: queryExportJobs, + // There can be maximum one table job, but put it in a list for consistency with vdbs and queries + tables: tableExportJobId + ? [ + { + jobId: tableExportJobId, + tables: tableExportTablesDetails, + }, + ] + : [], + vdbs: vdbExportJobs, + }, + response: startExportResponse, + error: startExportError, + info: startExportInfo, + }; + } + + const passedJobs: { + queries: { + job: (typeof queryExportJobs)[number]; + result: JobResult; + }[]; + tables: { + job: { + tables: typeof tableExportTablesDetails; + }; + result: JobResult; + }[]; + vdbs: { + job: (typeof vdbExportJobs)[number]; + result: JobResult; + }[]; + } = { + queries: [], + tables: [], + vdbs: [], + }; + + const failedJobs: typeof passedJobs = { + queries: [], + tables: [], + vdbs: [], + }; + + const queryExportPromises = queryExportJobs.map((job) => + this.waitForTask(job.jobId).then((result) => + (result.response?.status === "SUCCESS" + ? passedJobs + : failedJobs + ).queries.push({ job, result }) + ) + ); + + // Keep it as a list for consistency with queries and vdbs (even though max one item) + const tableExportPromises = [{ tableExportJobId, tableExportTablesDetails }] + .filter((v) => !!v.tableExportJobId) + // Note the variable names in the local scope are re-using the names from the outer scope + .map(({ tableExportJobId, tableExportTablesDetails }) => + this.waitForTask(tableExportJobId!).then((result) => + (result.response?.status === "SUCCESS" + ? passedJobs + : failedJobs + ).tables.push({ + job: { + tables: tableExportTablesDetails, + }, + result, + }) + ) + ); + + const vdbExportPromises = vdbExportJobs.map((job) => + this.waitForTask(job.jobId).then((result) => + (result.response?.status === "SUCCESS" + ? passedJobs + : failedJobs + ).vdbs.push({ job, result }) + ) + ); + + const allPromises = [ + ...queryExportPromises, + ...tableExportPromises, + ...vdbExportPromises, + ]; + + await Promise.all(allPromises); + + const totalPassed = + passedJobs.queries.length + + passedJobs.tables.length + + passedJobs.vdbs.length; + const totalFailed = + failedJobs.queries.length + + failedJobs.tables.length + + failedJobs.vdbs.length; + + const rval = { + response: { + success: true, // FIXME: seems unnecessary, just here for consistency with other (export) plugins + passedJobs, + }, + error: + totalFailed > 0 + ? { + error: `${totalFailed} / ${ + totalFailed + totalPassed + } jobs failed`, + failedJobs, + } + : null, + info: { + allPassed: totalFailed === 0, // note: true also if no jobs were run + somePassed: totalFailed > 0 && totalPassed > 0, + allFailed: totalPassed === 0 && totalFailed > 0, + totalPassed, + totalFailed, + startExportInfo, + }, + }; + + return rval; + } + + protected async startExport( + sourceOptions: ExportToSeafowlSourceOptions, + destOptions: ExportToSeafowlDestOptions + ) { + return this.graphqlClient.send< + StartExportToSeafowlJobMutation, + StartExportToSeafowlJobMutationVariables + >( + gql` + mutation StartExportToSeafowlJob( + $queries: [SeafowlExportQuerySourceDestinationTupleInput!] = null + $tables: [SeafowlExportTableSourceDestinationTupleInput!] = null + $vdbs: [SeafowlExportVDBSourceDestinationTupleInput!] = null + $seafowlInstance: SeafowlInstanceExportInput! = { selfHosted: null } + ) { + exportToSeafowl( + queries: $queries + tables: $tables + vdbs: $vdbs + seafowlInstance: $seafowlInstance + ) { + tables { + jobId + tables { + sourceVdbId + sourceNamespace + sourceRepository + sourceTable + } + } + queries { + jobId + sourceVdbId + sourceQuery + } + vdbs { + jobId + sourceVdbId + } + } + } + `, + { + queries: sourceOptions.queries ?? [], + tables: sourceOptions.tables ?? [], + vdbs: sourceOptions.vdbs ?? [], + seafowlInstance: destOptions.seafowlInstance ?? { selfHosted: null }, + } + ); + } +} + +export type ExpandRecursively = T extends object + ? T extends infer O + ? { [K in keyof O]: ExpandRecursively } + : never + : T; diff --git a/packages/db-splitgraph/plugins/generate-plugins.script.ts b/packages/db-splitgraph/plugins/generate-plugins.script.ts index 258ed0b..15ff8ef 100644 --- a/packages/db-splitgraph/plugins/generate-plugins.script.ts +++ b/packages/db-splitgraph/plugins/generate-plugins.script.ts @@ -1,4 +1,5 @@ import { compile } from "json-schema-to-typescript"; +import { generateName } from "json-schema-to-typescript/dist/src/utils"; import path from "path"; import { writeFile, mkdir } from "fs/promises"; import { request, gql } from "graphql-request"; @@ -13,6 +14,14 @@ const targetDir = path.join(thisSourceFileDir, "importers", "generated"); const generateTypes = async () => { const allPlugins = await fetchSchemas(); + // Store set of used interface names to ensure global uniqueness so that we + // can export * from generated files. However, note this may not work because + // the intermediate types are also exported, and those will have collisions + const usedInterfaceNames = new Set(); + + // Set of plugin class names (same comment as above applies, i.e. uniqueness is expected anyway) + const usedPluginClassNames = new Set(); + for (let plugin of allPlugins.externalPlugins) { log("generateTypes:", plugin.pluginName); let pluginTargetDir = path.join(targetDir, plugin.pluginName); @@ -20,24 +29,71 @@ const generateTypes = async () => { log("mkdir:", fdir(pluginTargetDir)); await mkdir(pluginTargetDir, { recursive: true }); - let schemas = [ + let schemas: [schema: any, schemaName: string, interfaceName: string][] = [ [plugin.credentialsSchema, "CredentialsSchema"], [plugin.paramsSchema, "ParamsSchema"], [plugin.tableParamsSchema, "TableParamsSchema"], - ]; - - for (let [schema, schemaName] of schemas) { + ].map(([schema, schemaName]) => [ + schema, + schemaName, + (() => { + // Generate a name that's safe for interface type name and hasn't been used yet + // We don't expect collisions, since pluginName is unique, but just in case + const interfaceName = generateName( + plugin.pluginName + schemaName, + usedInterfaceNames + ); + usedInterfaceNames.add(interfaceName); + + return interfaceName; + })(), + ]); + + for (let [schema, schemaName, interfaceName] of schemas) { let schemaOutFile = path.join(pluginTargetDir, `${schemaName}.ts`); + log("interfaceName:", interfaceName); - let generatedTypescript = await compile( - // @ts-ignore-error Some jsonschema issues, apparently with array items - schema, - plugin.pluginName - ); + let generatedTypescript = await compile(schema, interfaceName, { + strictIndexSignatures: true, + }); log("write schema:", fdir(schemaOutFile)); await writeFile(schemaOutFile, generatedTypescript); } + + const [ + [_cs, _csn, credentialsSchemaInterfaceName], + [_ps, _psn, paramsSchemaInterfaceName], + [_tps, _tpsn, tableParamsSchemaInterfaceName], + ] = schemas; + + // NOTE: Concatenate "Splitgraph" outside of generated name to ensure PascalCasing + // (otherwise we get "SplitgraphairbyteGitHubImportPlugin" instead of "SplitgraphAirbyteGitHubImportPlugin") + let pluginClassName = + "Splitgraph" + + generateName(plugin.pluginName + "ImportPlugin", usedPluginClassNames); + let pluginOutFile = path.join(pluginTargetDir, "plugin.ts"); + + log("create plugin: ", pluginClassName, "in", pluginOutFile); + + await writeFile( + pluginOutFile, + `/** Auto-generated plugin **/ + +import type { ${paramsSchemaInterfaceName} } from "./ParamsSchema"; +import type { ${tableParamsSchemaInterfaceName} } from "./TableParamsSchema"; +import type { ${credentialsSchemaInterfaceName} } from "./CredentialsSchema"; +import { makeGeneratedImportPlugin } from "../../splitgraph-generated-import-plugin"; + +export const ${pluginClassName} = makeGeneratedImportPlugin< + "${plugin.pluginName}", + ${paramsSchemaInterfaceName}, + ${tableParamsSchemaInterfaceName}, + ${credentialsSchemaInterfaceName} +>("${plugin.pluginName}"); + +` + ); } }; diff --git a/packages/db-splitgraph/plugins/importers/base-import-plugin.ts b/packages/db-splitgraph/plugins/importers/base-import-plugin.ts deleted file mode 100644 index 705b735..0000000 --- a/packages/db-splitgraph/plugins/importers/base-import-plugin.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type SplitgraphDestOptions = { - namespace: string; - repository: string; - tableName: string; -}; diff --git a/packages/db-splitgraph/plugins/importers/splitgraph-base-import-plugin.ts b/packages/db-splitgraph/plugins/importers/splitgraph-base-import-plugin.ts new file mode 100644 index 0000000..c6fd855 --- /dev/null +++ b/packages/db-splitgraph/plugins/importers/splitgraph-base-import-plugin.ts @@ -0,0 +1,603 @@ +import type { ImportPlugin, WithOptionsInterface } from "@madatdata/base-db"; + +import { gql } from "graphql-request"; + +import { Retryable, BackOffPolicy } from "typescript-retry-decorator"; + +import { SplitgraphGraphQLClient } from "../../gql-client/splitgraph-graphql-client"; + +import type { + RepositoryIngestionJobStatusQuery, + RepositoryIngestionJobStatusQueryVariables, + StartExternalRepositoryLoadMutation, + StartExternalRepositoryLoadMutationVariables, +} from "./splitgraph-base-import-plugin.generated"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; + +export type SplitgraphDestOptions = { + namespace: string; + repository: string; + /* default is public */ + initialPermissions?: StartExternalRepositoryLoadMutationVariables["initialPermissions"]; +}; + +export interface SplitgraphImportPluginOptions { + graphqlEndpoint: string; + transformRequestHeaders?: (requestHeaders: HeadersInit) => HeadersInit; +} + +type DbInjectedOptions = Partial; + +// 1 hour +const MAX_POLL_TIMEOUT = 1_000 * 60 * 60; +// const MAX_ATTEMPTS = MAX_POLL_TIMEOUT - (25.5 * 1000) / 10000; +const MAX_BACKOFF_INTERVAL = 10_000; +const MAX_ATTEMPTS = Math.ceil( + (MAX_POLL_TIMEOUT - 25.5 * 1_000) / MAX_BACKOFF_INTERVAL +); +const retryOptions = { + maxAttempts: MAX_ATTEMPTS, + backOff: 500, + backOffPolicy: BackOffPolicy.ExponentialBackOffPolicy, + exponentialOption: { maxInterval: MAX_BACKOFF_INTERVAL, multiplier: 2 }, +}; + +type ProvidedExternalLoadMutationVariables = Pick< + StartExternalRepositoryLoadMutationVariables, + "tables" +> & + Partial< + Omit + >; + +// We query for multple nods and then filter client-side for the node matching namespace and repository +type JobStatusNode = Exclude< + RepositoryIngestionJobStatusQuery["repositoryIngestionJobStatus"], + null +>["nodes"][number]; + +export type DeferredSplitgraphImportTask = { + completed: boolean; + response: { + jobStatus: JobStatusNode | null; + jobLog?: { url: string }; + } | null; + error: "no response" | "failed status" | null | any; + info: { + jobStatus: { status: number; headers: any } | null; + jobLog?: { status: number; headers: any } | null; + } | null; +}; + +export abstract class SplitgraphImportPlugin< + PluginName extends string, + /** The "params" schema for the plugin, i.e. provided by auto-generated type */ + PluginParamsSchema extends object, + /** The "table params" schema for the plugin, i.e. provided by auto-generated type */ + PluginTableParamsSchema extends object, + /** The "credentials" schema for the plugin, i.e. provided by auto-generated type */ + PluginCredentialsSchema extends object, + /** Concrete type of the derived class, for annotating return value of builder methods like withOptions */ + DerivedSplitgraphImportPlugin extends SplitgraphImportPlugin< + PluginName, + PluginParamsSchema, + PluginTableParamsSchema, + PluginCredentialsSchema, + DerivedSplitgraphImportPlugin, + StartedImportJob, + CompletedImportJob, + ConcreteImportDestOptions, + ConcreteImportSourceOptions + >, + StartedImportJob extends object, + CompletedImportJob extends Awaited< + ReturnType["importData"]> + >, + ConcreteImportDestOptions extends SplitgraphDestOptions = SplitgraphDestOptions, + ConcreteImportSourceOptions extends object = Record +> implements + ImportPlugin, + DeferredTaskPlugin, + WithOptionsInterface +{ + public abstract readonly __name: PluginName; + + // TODO: make sense? will be overridden? + // public static readonly __name: PluginName; + + // TODO: deleted because static property doesn't make sense on abstract class? cannot have static property + // public static readonly __name = "csv"; + public readonly opts: SplitgraphImportPluginOptions; + public readonly graphqlEndpoint: SplitgraphImportPluginOptions["graphqlEndpoint"]; + public readonly graphqlClient: SplitgraphGraphQLClient; + public readonly transformRequestHeaders: Required["transformRequestHeaders"]; + + constructor(opts: SplitgraphImportPluginOptions) { + this.opts = opts; + + this.graphqlEndpoint = opts.graphqlEndpoint; + this.transformRequestHeaders = opts.transformRequestHeaders ?? IdentityFunc; + + this.graphqlClient = new SplitgraphGraphQLClient({ + graphqlEndpoint: this.graphqlEndpoint, + transformRequestHeaders: this.transformRequestHeaders, + }); + } + + /** + * Builder method to return a new instance of the derived class with the given + * options merged with any existng options in the current instance. The returned + * object will be an instance of the derived class (DerivedSplitgraphImportPlugin), + * as returned by `Object.getPrototypeOf(this).constructor()` + */ + withOptions(injectOpts: DbInjectedOptions): DerivedSplitgraphImportPlugin { + const mergedInjectOpts: SplitgraphImportPluginOptions = { + ...this.opts, + ...injectOpts, + transformRequestHeaders: (reqHeaders) => { + const withOriginal = { + ...reqHeaders, + ...this.opts.transformRequestHeaders?.(reqHeaders), + }; + + const withNext = { + ...withOriginal, + ...injectOpts.transformRequestHeaders?.(withOriginal), + }; + + return { + ...withOriginal, + ...withNext, + }; + }, + }; + + return new (Object.getPrototypeOf(this).constructor)(mergedInjectOpts); + } + + public async importData( + rawSourceOptions: ConcreteImportSourceOptions, + rawDestOptions: ConcreteImportDestOptions, + importOptions?: { defer: boolean } + ) { + const { + sourceOptions = rawSourceOptions, + destOptions = rawDestOptions, + ...importCtx + } = await this.beforeImport(rawSourceOptions, rawDestOptions); + + const { + response: loadResponse, + error: loadError, + info: loadInfo, + } = await this.startLoad(sourceOptions, destOptions); + + if (loadError || !loadResponse) { + return { + ...(importOptions?.defer ? { taskId: null } : {}), + response: null, + error: loadError, + info: { ...importCtx.info, ...loadInfo }, + }; + } + + const { taskId } = loadResponse.startExternalRepositoryLoad; + + if (importOptions?.defer) { + return { + taskId, + response: loadResponse, + error: loadError ?? null, + info: { ...importCtx.info, ...loadInfo }, + }; + } + + const { response: statusResponse, error: statusError } = + await this.waitForTask(taskId, destOptions); + + const lastKnownJobStatus = statusResponse?.jobStatus.status; + + const info = { + ...importCtx.info, + ...statusResponse, + }; + + if (lastKnownJobStatus != TaskStatus.Success) { + return { + response: { + success: false, + }, + error: { + success: false, + pending: lastKnownJobStatus && taskUnresolved(lastKnownJobStatus), + ...statusError, + }, + info, + }; + } + + return { + response: { + success: true, + }, + error: null, + info, + }; + } + + /** + * Return the params and tables variable for the load + */ + protected abstract makeLoadMutationVariables( + sourceOptions: ConcreteImportSourceOptions, + destOptions: ConcreteImportDestOptions + ): ProvidedExternalLoadMutationVariables; + + // TODO: Clean this up to use an intermediate private member inside the derived class + // instead of passing some magical object through some magical pipeline + + /** + * Derived classes should implement this method to perform any pre-import steps, + * such as uploading a CSV file to object storage. It should return sourceOptions + * and destOptions if they are mutated in the process. + */ + protected async beforeImport( + sourceOptions: ConcreteImportSourceOptions, + destOptions: ConcreteImportDestOptions + ): Promise<{ + response: null | Response; + error: unknown; + info: object; + sourceOptions: ConcreteImportSourceOptions; + destOptions: ConcreteImportDestOptions; + }> { + return Promise.resolve({ + sourceOptions, + destOptions, + info: {}, + response: null, + error: null, + }); + } + + // TODO: preview step should return available table names + + private async startLoad( + sourceOptions: ConcreteImportSourceOptions, + destOptions: ConcreteImportDestOptions + ) { + const { tables, ...optionalVariables } = this.makeLoadMutationVariables( + sourceOptions, + destOptions + ); + + return this.graphqlClient.send< + StartExternalRepositoryLoadMutation, + StartExternalRepositoryLoadMutationVariables + >( + gql` + mutation StartExternalRepositoryLoad( + $namespace: String! + $repository: String! + $tables: [ExternalTableInput!]! + $initialPermissions: InitialPermissions + $pluginName: String + $params: JSON + $credentialId: String + $credentialData: JSON + $sync: Boolean + ) { + startExternalRepositoryLoad( + namespace: $namespace + repository: $repository + pluginName: $pluginName + params: $params + initialPermissions: $initialPermissions + tables: $tables + credentialId: $credentialId + credentialData: $credentialData + sync: $sync + ) { + taskId + } + } + `, + { + // Only required variable, but can be empty list to indicate loading all tables + tables, + // Not required or passable, because it must be same as this.__name + pluginName: this.__name, + // These are required, but we default to destOptions.{namespace,repository} + namespace: optionalVariables.namespace ?? destOptions.namespace, + repository: optionalVariables.repository ?? destOptions.repository, + // Truly optional variables + params: optionalVariables.params ?? {}, + credentialData: optionalVariables.credentialData ?? undefined, + initialPermissions: destOptions.initialPermissions ?? undefined, + credentialId: optionalVariables.credentialId ?? undefined, + sync: optionalVariables.sync ?? undefined, + } + ); + } + + private async fetchJobLog( + taskId: string, + { + namespace, + repository, + }: Pick + ) { + const { response, error, info } = await this.graphqlClient.send< + { + jobLogs: { + url: string; + }; + }, + { namespace: string; repository: string; taskId: string } + >( + gql` + query JobLogsByTaskId( + $namespace: String! + $repository: String! + $taskId: String! + ) { + jobLogs( + namespace: $namespace + repository: $repository + taskId: $taskId + ) { + url + } + } + `, + { + namespace, + repository, + taskId, + } + ); + + if (error || !response?.jobLogs) { + return { + response: null, + error, + info, + }; + } + + return { + response: { + url: response.jobLogs.url, + }, + error: null, + info, + }; + } + + private async fetchIngestionJobStatus( + taskId: string, + { + namespace, + repository, + }: Pick + ) { + const { response, error, info } = await this.graphqlClient.send< + RepositoryIngestionJobStatusQuery, + RepositoryIngestionJobStatusQueryVariables + >( + // NOTE: Splitgraph GQL API has no resolver for ingestion job by taskId, + // so we fetch last 10 jobLogs to safely find the one matching taskId, + // although it should almost always be the first (most recent) job. + gql` + query RepositoryIngestionJobStatus( + $namespace: String! + $repository: String! + ) { + repositoryIngestionJobStatus( + first: 10 + namespace: $namespace + repository: $repository + ) { + nodes { + taskId + started + finished + isManual + status + } + } + } + `, + { + namespace, + repository, + } + ); + + if (error || !response) { + return { response: null, error, info }; + } + + // FIXME(codegen): This ? shouldn't be necessary + const matchingJob = response.repositoryIngestionJobStatus?.nodes.find( + (node) => node.taskId === taskId + ); + + if (!matchingJob) { + return { response: null, error: null, info }; + } + + return { + response: matchingJob, + error: null, + info, + }; + } + + public async pollDeferredTask({ + taskId, + namespace, + repository, + }: { + taskId: string; + namespace: string; + repository: string; + }): Promise { + try { + const taskStatus = await this.waitForTaskOnce(taskId, { + namespace, + repository, + }); + + return { + completed: true, + ...taskStatus, + }; + } catch (err) { + if ( + typeof err === "object" && + err !== null && + "type" in err && + (err as { type: "retry"; response: JobStatusNode }).type === "retry" + ) { + return { + completed: false, + error: null, // it's just a retry, so we don't include error + response: { + jobStatus: + "response" in err + ? (err as { response: JobStatusNode }).response + : null, + }, + info: null, + }; + } else { + // We got an unknown/unexpected error (basically, caught something that was not retry) + return { + completed: true, + error: err, + response: null, + info: null, + }; + } + } + } + + @Retryable({ + ...retryOptions, + doRetry: ({ type }) => type === "retry", + }) + private async waitForTask( + taskId: string, + { + namespace, + repository, + }: Pick + ) { + return await this.waitForTaskOnce(taskId, { namespace, repository }); + } + + private async waitForTaskOnce( + taskId: string, + { + namespace, + repository, + }: Pick + ) { + const { + response: jobStatusResponse, + error: jobStatusError, + info: jobStatusInfo, + } = await this.fetchIngestionJobStatus(taskId, { + namespace, + repository, + }); + + if (jobStatusError) { + return { + response: null, + error: jobStatusError, + info: { jobStatus: jobStatusInfo }, + }; + } else if (!jobStatusResponse) { + throw { type: "retry" }; + // FIXME(codegen): this shouldn't be nullable + } else if (taskUnresolved(jobStatusResponse.status!)) { + throw { type: "retry", response: jobStatusResponse }; + } + + const { + response: jobLogResponse, + error: jobLogError, + info: jobLogInfo, + } = await this.fetchJobLog(taskId, { namespace, repository }); + + if (jobLogError || !jobLogResponse) { + return { + response: null, + error: jobLogError, + info: { jobLog: jobLogInfo, jobStatus: jobStatusInfo }, + }; + } + + return { + response: { + jobStatus: jobStatusResponse, + jobLog: jobLogResponse, + }, + error: null, + info: { + jobStatus: jobStatusInfo, + jobLog: jobLogInfo, + }, + }; + } +} + +const IdentityFunc = (x: T) => x; + +enum TaskStatus { + // Standard Celery statuses + Pending = "PENDING", + Started = "STARTED", + Success = "SUCCESS", + Failure = "FAILURE", + Revoked = "REVOKED", + + // Custom Splitgraph statuses + Lost = "LOST", + TimedOut = "TIMED_OUT", + + // Currently unused statuses + Retry = "RETRY", + Received = "RECEIVED", + Rejected = "REJECTED", + Ignored = "IGNORED", +} + +const standbyStatuses = [TaskStatus.Pending, TaskStatus.Started]; + +// const unrecoverableStatuses = [ +// TaskStatus.Failure, +// TaskStatus.Revoked, +// TaskStatus.Lost, +// TaskStatus.TimedOut, +// ]; + +// const unexpectedStatuses = [ +// TaskStatus.Retry, +// TaskStatus.Received, +// TaskStatus.Rejected, +// TaskStatus.Ignored, +// ]; + +const taskUnresolved = (ts: TaskStatus) => standbyStatuses.includes(ts); + +// const taskNotRecoverable = (ts: TaskStatus) => +// unrecoverableStatuses.includes(ts); + +// const taskHasKnownButUnexpectedStatus = (ts: TaskStatus) => +// unexpectedStatuses.includes(ts); + +// type SomeOptional = Omit & +// Pick, OptionalKeys>; diff --git a/packages/db-splitgraph/plugins/importers/splitgraph-generated-import-plugin.ts b/packages/db-splitgraph/plugins/importers/splitgraph-generated-import-plugin.ts new file mode 100644 index 0000000..f1a3d61 --- /dev/null +++ b/packages/db-splitgraph/plugins/importers/splitgraph-generated-import-plugin.ts @@ -0,0 +1,123 @@ +import type { ImportPlugin, WithOptionsInterface } from "@madatdata/base-db"; + +import { + DeferredSplitgraphImportTask, + SplitgraphImportPlugin, +} from "./splitgraph-base-import-plugin"; +import type { SplitgraphDestOptions } from "./splitgraph-base-import-plugin"; +import type { SplitgraphImportPluginOptions } from "./splitgraph-base-import-plugin"; + +import type { ExternalTableColumnInput } from "../../gql-client/unified-types"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; + +export interface BaseGeneratedImportSourceOptions< + PluginParamsSchema extends object +> { + params: PluginParamsSchema; + sync?: boolean; +} + +export interface GeneratedImportSourceOptionsWithInlineCredentialData< + PluginParamsSchema extends object, + PluginCredentialsSchema extends object +> extends BaseGeneratedImportSourceOptions { + credentials: PluginCredentialsSchema; +} + +export interface GeneratedImportSourceOptionsWithSavedCredentialId< + PluginParamsSchema extends object +> extends BaseGeneratedImportSourceOptions { + credentialId: string; +} + +export type GeneratedImportSourceOptions< + PluginParamsSchema extends object, + PluginCredentialsSchema extends object +> = + | GeneratedImportSourceOptionsWithInlineCredentialData< + PluginParamsSchema, + PluginCredentialsSchema + > + | GeneratedImportSourceOptionsWithSavedCredentialId; + +export interface GeneratedImportDestOptions + extends SplitgraphDestOptions { + tables?: { + name: string; + options: TableParamsSchema; + /** + * Array containing which columns to include in the ingestion. Set to an + * empty array (`[]`) to default to including all columns. + * */ + schema: ExternalTableColumnInput[]; + }[]; +} + +export function makeGeneratedImportPlugin< + PluginName extends string, + ParamsSchema extends object, + TableParamsSchema extends object, + CredentialsSchema extends object, + ConcreteImportDestOptions extends GeneratedImportDestOptions = GeneratedImportDestOptions, + ConcreteImportSourceOptions extends GeneratedImportSourceOptions< + ParamsSchema, + CredentialsSchema + > = GeneratedImportSourceOptions +>( + pluginName: PluginName +): new (opts: SplitgraphImportPluginOptions) => ImportPlugin< + PluginName, + ConcreteImportSourceOptions, + ConcreteImportDestOptions +> & + WithOptionsInterface> { + class SplitgraphGeneratedImportPlugin + extends SplitgraphImportPlugin< + PluginName, + ParamsSchema, + TableParamsSchema, + CredentialsSchema, + SplitgraphGeneratedImportPlugin, + Record, + Awaited["importData"]>>, + ConcreteImportDestOptions, + ConcreteImportSourceOptions + > + implements + ImportPlugin< + PluginName, + ConcreteImportSourceOptions, + ConcreteImportDestOptions + >, + DeferredTaskPlugin, + WithOptionsInterface + { + public readonly __name = pluginName; + public static __name = pluginName; + + protected makeLoadMutationVariables( + sourceOptions: ConcreteImportSourceOptions, + destOptions: ConcreteImportDestOptions + ) { + // NOTE: only need to return variables that aren't already defaulted e.g. from destOptions + // which is why we can skip namespace, repository, and pluginName + return { + params: JSON.stringify({ + ...sourceOptions.params, + }), + tables: destOptions.tables ?? [], + credentialData: + "credentials" in sourceOptions + ? JSON.stringify(sourceOptions.credentials) + : undefined, + credentialId: + "credentialId" in sourceOptions + ? sourceOptions.credentialId + : undefined, + sync: "sync" in sourceOptions ? sourceOptions.sync : undefined, + }; + } + } + + return SplitgraphGeneratedImportPlugin; +} diff --git a/packages/db-splitgraph/plugins/importers/splitgraph-import-csv-plugin.ts b/packages/db-splitgraph/plugins/importers/splitgraph-import-csv-plugin.ts index cf7ae02..bd3a900 100644 --- a/packages/db-splitgraph/plugins/importers/splitgraph-import-csv-plugin.ts +++ b/packages/db-splitgraph/plugins/importers/splitgraph-import-csv-plugin.ts @@ -1,41 +1,29 @@ import type { ImportPlugin, WithOptionsInterface } from "@madatdata/base-db"; -import type { SplitgraphDestOptions } from "./base-import-plugin"; - import { gql } from "graphql-request"; -import { Retryable, BackOffPolicy } from "typescript-retry-decorator"; - -import { SplitgraphGraphQLClient } from "../../gql-client/splitgraph-graphql-client"; +import type { CsvParamsSchema } from "./generated/csv/ParamsSchema"; +import type { CsvTableParamsSchema } from "./generated/csv/TableParamsSchema"; +import type { CsvCredentialsSchema } from "./generated/csv/CredentialsSchema"; -import type { Csv as CsvTableParamsSchema } from "./generated/csv/TableParamsSchema"; -import type { Csv as CsvParamsSchema } from "./generated/csv/ParamsSchema"; -import type { Csv as CsvCredentialsSchema } from "./generated/csv/CredentialsSchema"; - -import type { - RepositoryIngestionJobStatusQuery, - RepositoryIngestionJobStatusQueryVariables, - StartExternalRepositoryLoadMutation, - StartExternalRepositoryLoadMutationVariables, -} from "./splitgraph-import-csv-plugin.generated"; +import { + DeferredSplitgraphImportTask, + SplitgraphDestOptions, + SplitgraphImportPlugin, +} from "./splitgraph-base-import-plugin"; +import type { DeferredTaskPlugin } from "@madatdata/base-db/base-db"; +// NOTE: CSV only supports loading one table at a time +// TODO: maybe go back kto importing this or using a generic from splitgraph-base-import-plugin +// to share the commonalities (e.g. for single-table: tableName; multi-table: tables, etc.) interface ImportCSVDestOptions extends SplitgraphDestOptions { - params?: CsvParamsSchema; - tableName: SplitgraphDestOptions["tableName"]; - // TODO: support > 1 table + tableName: string; tableParams?: CsvTableParamsSchema; credentials?: CsvCredentialsSchema; - /* default private */ - initialPermissions?: StartExternalRepositoryLoadMutationVariables["initialPermissions"]; -} - -interface ImportCSVPluginOptions { - graphqlEndpoint: string; - transformRequestHeaders?: (requestHeaders: HeadersInit) => HeadersInit; } +// TODO: should params always be required? and therefore part of an imported generic interface ImportCSVBaseOptions { - // _type: "import-csv-base"; - // importType: "csv"; + params?: CsvParamsSchema; } interface ImportCSVFromURLOptions extends ImportCSVBaseOptions { @@ -58,131 +46,26 @@ type ImportCSVSourceOptions = | ImportCSVFromURLOptions | ImportCSVFromDataOptions; -type DbInjectedOptions = Partial; - -// 1 hour -const MAX_POLL_TIMEOUT = 1_000 * 60 * 60; -// const MAX_ATTEMPTS = MAX_POLL_TIMEOUT - (25.5 * 1000) / 10000; -const MAX_BACKOFF_INTERVAL = 10_000; -const MAX_ATTEMPTS = Math.ceil( - (MAX_POLL_TIMEOUT - 25.5 * 1_000) / MAX_BACKOFF_INTERVAL -); -const retryOptions = { - maxAttempts: MAX_ATTEMPTS, - backOff: 500, - backOffPolicy: BackOffPolicy.ExponentialBackOffPolicy, - exponentialOption: { maxInterval: MAX_BACKOFF_INTERVAL, multiplier: 2 }, -}; - export class SplitgraphImportCSVPlugin - implements ImportPlugin, WithOptionsInterface + extends SplitgraphImportPlugin< + "csv", + CsvParamsSchema, + CsvTableParamsSchema, + CsvCredentialsSchema, + SplitgraphImportCSVPlugin, + Record, + Awaited["importData"]>>, + ImportCSVDestOptions, + ImportCSVSourceOptions + > + implements + ImportPlugin<"csv">, + DeferredTaskPlugin<"csv", DeferredSplitgraphImportTask>, + WithOptionsInterface { public readonly __name = "csv"; public static readonly __name = "csv"; - public readonly opts: ImportCSVPluginOptions; - public readonly graphqlEndpoint: ImportCSVPluginOptions["graphqlEndpoint"]; - public readonly graphqlClient: SplitgraphGraphQLClient; - public readonly transformRequestHeaders: Required["transformRequestHeaders"]; - - constructor(opts: ImportCSVPluginOptions) { - this.opts = opts; - - this.graphqlEndpoint = opts.graphqlEndpoint; - this.transformRequestHeaders = opts.transformRequestHeaders ?? IdentityFunc; - - this.graphqlClient = new SplitgraphGraphQLClient({ - graphqlEndpoint: this.graphqlEndpoint, - transformRequestHeaders: this.transformRequestHeaders, - }); - } - - // TODO: improve it (e.g. allow either mutation or copy), and/or generalize it - withOptions(injectOpts: DbInjectedOptions) { - return new SplitgraphImportCSVPlugin({ - ...this.opts, - ...injectOpts, - // TODO: replace transformer with some kind of chainable "link" plugin - transformRequestHeaders: (reqHeaders) => { - const withOriginal = { - ...reqHeaders, - ...this.opts.transformRequestHeaders?.(reqHeaders), - }; - - const withNext = { - ...withOriginal, - ...injectOpts.transformRequestHeaders?.(withOriginal), - }; - - return { - ...withOriginal, - ...withNext, - }; - }, - }); - } - - private async startLoad( - sourceOptions: ImportCSVFromURLOptions, - destOptions: ImportCSVDestOptions - ) { - return this.graphqlClient.send< - StartExternalRepositoryLoadMutation, - StartExternalRepositoryLoadMutationVariables - >( - gql` - mutation StartExternalRepositoryLoad( - $namespace: String! - $repository: String! - $tables: [ExternalTableInput!]! - $initialPermissions: InitialPermissions - $pluginName: String - $params: JSON - $credentialId: String - $sync: Boolean - ) { - startExternalRepositoryLoad( - namespace: $namespace - repository: $repository - pluginName: $pluginName - params: $params - initialPermissions: $initialPermissions - tables: $tables - credentialId: $credentialId - sync: $sync - ) { - taskId - } - } - `, - { - initialPermissions: destOptions.initialPermissions, - // NOTE: Optional params are required for typescript, ignored when sent - credentialId: undefined, - sync: undefined, - namespace: destOptions.namespace, - repository: destOptions.repository, - params: JSON.stringify({ - url: sourceOptions.url, - connection_type: "http", - ...destOptions.params, - }), - tables: [ - { - name: destOptions.tableName, - options: JSON.stringify({ - url: sourceOptions.url, - ...destOptions.tableParams, - }), - // TODO: allow user to specify schema in destOptions - schema: [], - }, - ], - pluginName: "csv", - } - ); - } - private async fetchPresignedURL() { const { response, error, info } = await this.graphqlClient.send<{ fileUploadDownloadUrls: { @@ -203,120 +86,6 @@ export class SplitgraphImportCSVPlugin return { response, error, info }; } - private async fetchJobLog( - taskId: string, - { - namespace, - repository, - }: Pick - ) { - const { response, error, info } = await this.graphqlClient.send< - { - jobLogs: { - url: string; - }; - }, - { namespace: string; repository: string; taskId: string } - >( - gql` - query JobLogsByTaskId( - $namespace: String! - $repository: String! - $taskId: String! - ) { - jobLogs( - namespace: $namespace - repository: $repository - taskId: $taskId - ) { - url - } - } - `, - { - namespace, - repository, - taskId, - } - ); - - if (error || !response?.jobLogs) { - return { - response: null, - error, - info, - }; - } - - return { - response: { - url: response.jobLogs.url, - }, - error: null, - info, - }; - } - - private async fetchIngestionJobStatus( - taskId: string, - { - namespace, - repository, - }: Pick - ) { - const { response, error, info } = await this.graphqlClient.send< - RepositoryIngestionJobStatusQuery, - RepositoryIngestionJobStatusQueryVariables - >( - // NOTE: Splitgraph GQL API has no resolver for ingestion job by taskId, - // so we fetch last 10 jobLogs to safely find the one matching taskId, - // although it should almost always be the first (most recent) job. - gql` - query RepositoryIngestionJobStatus( - $namespace: String! - $repository: String! - ) { - repositoryIngestionJobStatus( - first: 10 - namespace: $namespace - repository: $repository - ) { - nodes { - taskId - started - finished - isManual - status - } - } - } - `, - { - namespace, - repository, - } - ); - - if (error || !response) { - return { response: null, error, info }; - } - - // FIXME(codegen): This ? shouldn't be necessary - const matchingJob = response.repositoryIngestionJobStatus?.nodes.find( - (node) => node.taskId === taskId - ); - - if (!matchingJob) { - return { response: null, error: null, info }; - } - - return { - response: matchingJob, - error: null, - info, - }; - } - /** * NOTE: Splitgraph does not currently support multipart form data for CSV files, * because that requires fetching the presigned token with a multipart parameter, @@ -409,67 +178,32 @@ export class SplitgraphImportCSVPlugin }; } - @Retryable({ - ...retryOptions, - doRetry: ({ type }) => type === "retry", - }) - private async waitForTask( - taskId: string, - { - namespace, - repository, - }: Pick + protected makeLoadMutationVariables( + sourceOptions: ImportCSVSourceOptions, + destOptions: ImportCSVDestOptions ) { - const { - response: jobStatusResponse, - error: jobStatusError, - info: jobStatusInfo, - } = await this.fetchIngestionJobStatus(taskId, { - namespace, - repository, - }); - - if (jobStatusError) { - return { - response: null, - error: jobStatusError, - info: { jobStatus: jobStatusInfo }, - }; - } else if (!jobStatusResponse) { - throw { type: "retry" }; - // FIXME(codegen): this shouldn't be nullable - } else if (taskUnresolved(jobStatusResponse.status!)) { - throw { type: "retry" }; - } - - const { - response: jobLogResponse, - error: jobLogError, - info: jobLogInfo, - } = await this.fetchJobLog(taskId, { namespace, repository }); - - if (jobLogError || !jobLogResponse) { - return { - response: null, - error: jobLogError, - info: { jobLog: jobLogInfo, jobStatus: jobStatusInfo }, - }; - } - return { - response: { - jobStatus: jobStatusResponse, - jobLog: jobLogResponse, - }, - error: null, - info: { - jobStatus: jobStatusInfo, - jobLog: jobLogInfo, - }, + params: JSON.stringify({ + url: sourceOptions.url, + connection_type: "http", + ...sourceOptions.params, + }), + tables: [ + { + name: destOptions.tableName, + options: JSON.stringify({ + url: sourceOptions.url, + ...destOptions.tableParams, + }), + // TODO: allow user to specify schema in destOptions + schema: [], + }, + ], + pluginName: "csv", }; } - async importData( + protected async beforeImport( sourceOptions: ImportCSVSourceOptions, destOptions: ImportCSVDestOptions ) { @@ -489,7 +223,13 @@ export class SplitgraphImportCSVPlugin ); if (error || !response) { - return { response, error, info }; + return Promise.resolve({ + response, + error, + info, + sourceOptions, + destOptions, + }); } // Now that we uploaded data to url, remove the data property and add url property @@ -501,55 +241,13 @@ export class SplitgraphImportCSVPlugin importCtx.info = { ...info }; } - sourceOptions = sourceOptions as ImportCSVFromURLOptions; - - const { - response: loadResponse, - error: loadError, - info: loadInfo, - } = await this.startLoad(sourceOptions, destOptions); - - if (loadError || !loadResponse) { - return { - response: null, - error: loadError, - info: { ...importCtx.info, ...loadInfo }, - }; - } - - const { taskId } = loadResponse.startExternalRepositoryLoad; - - const { response: statusResponse, error: statusError } = - await this.waitForTask(taskId, destOptions); - - const lastKnownJobStatus = statusResponse?.jobStatus.status; - - const info = { - ...importCtx.info, - ...statusResponse, - }; - - if (lastKnownJobStatus != TaskStatus.Success) { - return { - response: { - success: false, - }, - error: { - success: false, - pending: lastKnownJobStatus && taskUnresolved(lastKnownJobStatus), - ...statusError, - }, - info, - }; - } - - return { - response: { - success: true, - }, + return Promise.resolve({ + response: null, error: null, - info, - }; + info: importCtx.info, + sourceOptions, + destOptions, + }); } } @@ -558,49 +256,3 @@ const IdentityFunc = (x: T) => x; // TODO: Consider adding `type-fest`: https://github.com/sindresorhus/type-fest // which has AsyncReturnValue to replace Unpromise> type Unpromise> = T extends Promise ? U : never; - -enum TaskStatus { - // Standard Celery statuses - Pending = "PENDING", - Started = "STARTED", - Success = "SUCCESS", - Failure = "FAILURE", - Revoked = "REVOKED", - - // Custom Splitgraph statuses - Lost = "LOST", - TimedOut = "TIMED_OUT", - - // Currently unused statuses - Retry = "RETRY", - Received = "RECEIVED", - Rejected = "REJECTED", - Ignored = "IGNORED", -} - -const standbyStatuses = [TaskStatus.Pending, TaskStatus.Started]; - -// const unrecoverableStatuses = [ -// TaskStatus.Failure, -// TaskStatus.Revoked, -// TaskStatus.Lost, -// TaskStatus.TimedOut, -// ]; - -// const unexpectedStatuses = [ -// TaskStatus.Retry, -// TaskStatus.Received, -// TaskStatus.Rejected, -// TaskStatus.Ignored, -// ]; - -const taskUnresolved = (ts: TaskStatus) => standbyStatuses.includes(ts); - -// const taskNotRecoverable = (ts: TaskStatus) => -// unrecoverableStatuses.includes(ts); - -// const taskHasKnownButUnexpectedStatus = (ts: TaskStatus) => -// unexpectedStatuses.includes(ts); - -// type SomeOptional = Omit & -// Pick, OptionalKeys>; diff --git a/packages/react/hooks.test.tsx b/packages/react/hooks.test.tsx index 480454d..53045ef 100644 --- a/packages/react/hooks.test.tsx +++ b/packages/react/hooks.test.tsx @@ -208,7 +208,7 @@ describe("makeDefaultAnonymousContext", () => { }, }, "plugins": [ - _SplitgraphImportCSVPlugin { + SplitgraphImportCSVPlugin { "__name": "csv", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { @@ -227,8 +227,46 @@ describe("makeDefaultAnonymousContext", () => { }, "transformRequestHeaders": [Function], }, - _ExportQueryPlugin { - "__name": "exportQuery", + SplitgraphGeneratedImportPlugin { + "__name": "airbyte-github", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportQueryToFilePlugin { + "__name": "export-query-to-file", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportToSeafowlPlugin { + "__name": "export-to-seafowl", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { "options": { @@ -251,7 +289,7 @@ describe("makeDefaultAnonymousContext", () => { "plugins": PluginRegistry { "hostContext": {}, "plugins": [ - _SplitgraphImportCSVPlugin { + SplitgraphImportCSVPlugin { "__name": "csv", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { @@ -270,8 +308,46 @@ describe("makeDefaultAnonymousContext", () => { }, "transformRequestHeaders": [Function], }, - _ExportQueryPlugin { - "__name": "exportQuery", + SplitgraphGeneratedImportPlugin { + "__name": "airbyte-github", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportQueryToFilePlugin { + "__name": "export-query-to-file", + "graphqlClient": SplitgraphGraphQLClient { + "graphqlClient": GraphQLClient { + "options": { + "headers": [Function], + }, + "url": "https://api.splitgraph.com/gql/cloud/unified/graphql", + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "opts": { + "graphqlEndpoint": "https://api.splitgraph.com/gql/cloud/unified/graphql", + "transformRequestHeaders": [Function], + }, + "transformRequestHeaders": [Function], + }, + SplitgraphExportToSeafowlPlugin { + "__name": "export-to-seafowl", "graphqlClient": SplitgraphGraphQLClient { "graphqlClient": GraphQLClient { "options": { diff --git a/packages/react/hooks.tsx b/packages/react/hooks.tsx index 2b7a77a..2f16098 100644 --- a/packages/react/hooks.tsx +++ b/packages/react/hooks.tsx @@ -45,9 +45,32 @@ export const SqlProvider = ({ const useSqlContext = () => useContext(SqlContext); +export interface UseSqlBaseOptions { + /** + * Either a boolean, or a function that returns a boolean given a query, + * which indicates whether or not the query should be executed. This can + * be helpful for dynamically built queries that may not be ready to execute + * immediately when the hook is rendered. + * + * For more precise control, you can also pass a function to the `query` parameter, + * and return `null` from it to indicate that it's not ready to execute yet. + */ + isReady?: boolean | ((query: string) => boolean); + + /** + * An optional AbortSignal to use to abort the query. If not provided, the + * hook will create its own AbortSignal to call on unmount, which is probably + * desirable to avoid sending a query multiple times during development. To opt + * out of this behavior, set it to `null`. + */ + abortSignal?: AbortSignal | null; +} + +export type UseSqlOptions = UseSqlBaseOptions & Overloads; + export function useSql( - query: string, - executeOptions: { rowMode: "array" } + query: string | null | (() => string | null), + executeOptions: UseSqlOptions<{ rowMode: "array" }> ): { loading: boolean; response: ExecutionResultWithArrayShapedRows | null; @@ -55,8 +78,8 @@ export function useSql( }; export function useSql( - query: string, - executeOptions?: { rowMode: "object" } + query: string | null | (() => string | null), + executeOptions?: UseSqlOptions<{ rowMode: "object" }> ): { loading: boolean; response: ExecutionResultWithObjectShapedRows | null; @@ -64,8 +87,14 @@ export function useSql( }; export function useSql( - query: string, - execOptions?: { rowMode?: "object" | "array" } + /** + * The query to execute, or a function that returns the query to execute. The + * query should be of type `string`, or `null` to indicate that the query is + * not ready to execute yet. This can be helpful for dynamically built queries, + * and for avoiding the need to use the `isReady` parameter. + */ + query: string | null | (() => string | null), + execOptions?: UseSqlOptions<{ rowMode?: "object" | "array" }> ) { const [state, setState] = useState<{ loading: boolean; @@ -94,10 +123,31 @@ export function useSql( return; } + const queryString = typeof query === "function" ? query() : query; + + if (!queryString) { + return; + } + + if (typeof execOptions?.isReady === "boolean" && !execOptions?.isReady) { + return; + } + + if ( + execOptions?.isReady && + typeof execOptions?.isReady === "function" && + !execOptions?.isReady(queryString) + ) { + return; + } + + const defaultAbortController = + execOptions?.abortSignal === null ? null : new AbortController(); + client - .execute(query, { + .execute(queryString, { rowMode: execOptions?.rowMode ?? "object", - ...execOptions, + abortSignal: execOptions?.abortSignal ?? defaultAbortController?.signal, }) .then((result) => setState({ @@ -106,7 +156,26 @@ export function useSql( error: result.error, }) ); - }, [query, execOptions?.rowMode]); + + return () => { + if (execOptions?.abortSignal === null) { + // User opted out of sending an abort signal + return; + } else if (execOptions?.abortSignal) { + // User provided an abort signal, so don't abort it here + return; + } else if (defaultAbortController) { + return defaultAbortController.abort(); + } else { + console.log("Unexpected state: no abort controller available to abort"); + } + }; + }, [ + query, + execOptions?.rowMode, + execOptions?.abortSignal, + execOptions?.isReady, + ]); return state; } diff --git a/packages/react/package.json b/packages/react/package.json index b274f71..045fba8 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@madatdata/react", - "version": "0.0.11", + "version": "0.0.12", "packageManager": "yarn@3.2.0", "main": "index.ts", "types": "./build/es2020/index.d.ts", diff --git a/packages/test-helpers/env-config.ts b/packages/test-helpers/env-config.ts index 8e3721c..20eae46 100644 --- a/packages/test-helpers/env-config.ts +++ b/packages/test-helpers/env-config.ts @@ -25,6 +25,26 @@ const shouldIncludeIntegrationTests = () => { ); }; +// True if there is a secret defined for testing Seafowl export destination, +// which might be different from the Seafowl instance used for Seafowl integration tests +const environmentHasSeafowlExportDestinationCredential = () => { + return ( + // @ts-expect-error https://stackoverflow.com/a/70711231 + !!import.meta.env.VITE_TEST_SEAFOWL_EXPORT_DEST_URL && + // @ts-expect-error https://stackoverflow.com/a/70711231 + !!import.meta.env.VITE_TEST_SEAFOWL_EXPORT_DEST_DBNAME && + // @ts-expect-error https://stackoverflow.com/a/70711231 + !!import.meta.env.VITE_TEST_SEAFOWL_EXPORT_DEST_SECRET + ); +}; + +export const shouldSkipExportFromSplitgraphToSeafowlIntegrationTests = () => { + return ( + shouldSkipIntegrationTests() || + !environmentHasSeafowlExportDestinationCredential() + ); +}; + /** * Inspect environment for configuration indicating whether to run integration * tests against the real production DDN at Splitgraph.com. @@ -72,6 +92,26 @@ export const shouldSkipIntegrationTests = () => { return !shouldIncludeIntegrationTests(); }; +/** + * Check whether tests that ingest from GitHub as an external data source (using + * the Splitgraph plugin `airbyte-github`) should be skipped. + * + * @returns true if missing GitHub PAT, or if integration tests should be skipped + */ +export const shouldSkipIntegrationTestsForGitHubExternalDataSource = () => { + // TODO: Temporarily hardcoded to avoid ingesting a bunch of data without deleting it + return true; + + // const environmentHasGitHubPATSecret = () => { + // return ( + // // @ts-expect-error https://stackoverflow.com/a/70711231 + // !!import.meta.env.VITE_TEST_GITHUB_PAT_SECRET + // ); + // }; + // return shouldSkipIntegrationTests() || !environmentHasGitHubPATSecret(); +}; + export const shouldSkipSeafowlTests = () => { + return true; return !environmentHasSeafowlCredential(); }; diff --git a/yarn.lock b/yarn.lock index 6afaaa8..94ddcf4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15,18 +15,6 @@ __metadata: languageName: node linkType: hard -"@apidevtools/json-schema-ref-parser@npm:9.0.9": - version: 9.0.9 - resolution: "@apidevtools/json-schema-ref-parser@npm:9.0.9" - dependencies: - "@jsdevtools/ono": ^7.1.3 - "@types/json-schema": ^7.0.6 - call-me-maybe: ^1.0.1 - js-yaml: ^4.1.0 - checksum: b21f6bdd37d2942c3967ee77569bc74fadd1b922f688daf5ef85057789a2c3a7f4afc473aa2f3a93ec950dabb6ef365f8bd9cf51e4e062a1ee1e59b989f8f9b4 - languageName: node - linkType: hard - "@ardatan/relay-compiler@npm:12.0.0": version: 12.0.0 resolution: "@ardatan/relay-compiler@npm:12.0.0" @@ -56,6 +44,15 @@ __metadata: languageName: node linkType: hard +"@ardatan/sync-fetch@npm:^0.0.1": + version: 0.0.1 + resolution: "@ardatan/sync-fetch@npm:0.0.1" + dependencies: + node-fetch: ^2.6.1 + checksum: af39bdfb4c2b35bd2c6acc540a5e302730dae17e73d3a18cd1a4aa50c1c741cb1869dffdef1379c491da5ad2e3cfa2bf3a8064e6046c12b46c6a97f54f100a8d + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.10.4, @babel/code-frame@npm:^7.18.6": version: 7.18.6 resolution: "@babel/code-frame@npm:7.18.6" @@ -317,6 +314,13 @@ __metadata: languageName: node linkType: hard +"@babel/helper-plugin-utils@npm:^7.19.0": + version: 7.21.5 + resolution: "@babel/helper-plugin-utils@npm:7.21.5" + checksum: 6f086e9a84a50ea7df0d5639c8f9f68505af510ea3258b3c8ac8b175efdfb7f664436cb48996f71791a1350ba68f47ad3424131e8e718c5e2ad45564484cbb36 + languageName: node + linkType: hard + "@babel/helper-replace-supers@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-replace-supers@npm:7.18.6" @@ -478,6 +482,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-syntax-import-assertions@npm:^7.20.0": + version: 7.20.0 + resolution: "@babel/plugin-syntax-import-assertions@npm:7.20.0" + dependencies: + "@babel/helper-plugin-utils": ^7.19.0 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 6a86220e0aae40164cd3ffaf80e7c076a1be02a8f3480455dddbae05fda8140f429290027604df7a11b3f3f124866e8a6d69dbfa1dda61ee7377b920ad144d5b + languageName: node + linkType: hard + "@babel/plugin-syntax-jsx@npm:^7.0.0, @babel/plugin-syntax-jsx@npm:^7.18.6": version: 7.18.6 resolution: "@babel/plugin-syntax-jsx@npm:7.18.6" @@ -875,6 +890,18 @@ __metadata: languageName: node linkType: hard +"@bcherny/json-schema-ref-parser@npm:10.0.5-fork": + version: 10.0.5-fork + resolution: "@bcherny/json-schema-ref-parser@npm:10.0.5-fork" + dependencies: + "@jsdevtools/ono": ^7.1.3 + "@types/json-schema": ^7.0.6 + call-me-maybe: ^1.0.1 + js-yaml: ^4.1.0 + checksum: e90eb3655c4e15f54ebc5138baac98471d159e3a253b484416c03c2d43f5c3bc80a4d6fe18acd71f77bf2f95f7fbc36730abb21cbd1f9d80a6af630c554e6d62 + languageName: node + linkType: hard + "@bcoe/v8-coverage@npm:^0.2.3": version: 0.2.3 resolution: "@bcoe/v8-coverage@npm:0.2.3" @@ -882,20 +909,6 @@ __metadata: languageName: node linkType: hard -"@endemolshinegroup/cosmiconfig-typescript-loader@npm:3.0.2": - version: 3.0.2 - resolution: "@endemolshinegroup/cosmiconfig-typescript-loader@npm:3.0.2" - dependencies: - lodash.get: ^4 - make-error: ^1 - ts-node: ^9 - tslib: ^2 - peerDependencies: - cosmiconfig: ">=6" - checksum: 7fe0198622b1063c40572034df7e8ba867865a1b4815afe230795929abcf785758b34d7806a8e2100ba8ab4e92c5a1c3e11a980c466c4406df6e7ec6e50df8b6 - languageName: node - linkType: hard - "@esbuild-kit/cjs-loader@npm:^2.0.1": version: 2.0.1 resolution: "@esbuild-kit/cjs-loader@npm:2.0.1" @@ -1248,50 +1261,55 @@ __metadata: languageName: node linkType: hard -"@graphql-codegen/add@npm:^3.2.0": - version: 3.2.0 - resolution: "@graphql-codegen/add@npm:3.2.0" +"@graphql-codegen/add@npm:^3.2.1": + version: 3.2.3 + resolution: "@graphql-codegen/add@npm:3.2.3" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 + "@graphql-codegen/plugin-helpers": ^3.1.1 tslib: ~2.4.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: 919b8ac18a96c8d9d7ed9acfe9c596cefadbdc75486b7f92b5252fc47c32f175f521074331b07db5f79780beed6cbf7d4408202e724763ff60287e637ab92a28 + checksum: 98b1b17104b7e2fa82e9ed30e21160b02cce530d0ff72ce7794478677168ac6381a8d814cdd25d60b41b91b6446ebd592ba4820bd5ac138016f9097fa6ebc483 languageName: node linkType: hard -"@graphql-codegen/cli@npm:2.8.0": - version: 2.8.0 - resolution: "@graphql-codegen/cli@npm:2.8.0" +"@graphql-codegen/cli@npm:3.3.1": + version: 3.3.1 + resolution: "@graphql-codegen/cli@npm:3.3.1" dependencies: - "@graphql-codegen/core": 2.6.0 - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-tools/apollo-engine-loader": ^7.3.1 - "@graphql-tools/code-file-loader": ^7.3.0 - "@graphql-tools/git-loader": ^7.2.0 - "@graphql-tools/github-loader": ^7.3.1 - "@graphql-tools/graphql-file-loader": ^7.4.0 - "@graphql-tools/json-file-loader": ^7.4.0 - "@graphql-tools/load": ^7.7.0 - "@graphql-tools/prisma-loader": ^7.2.2 - "@graphql-tools/url-loader": ^7.12.1 - "@graphql-tools/utils": ^8.8.0 - ansi-escapes: ^4.3.1 + "@babel/generator": ^7.18.13 + "@babel/template": ^7.18.10 + "@babel/types": ^7.18.13 + "@graphql-codegen/core": ^3.1.0 + "@graphql-codegen/plugin-helpers": ^4.2.0 + "@graphql-tools/apollo-engine-loader": ^7.3.6 + "@graphql-tools/code-file-loader": ^7.3.17 + "@graphql-tools/git-loader": ^7.2.13 + "@graphql-tools/github-loader": ^7.3.20 + "@graphql-tools/graphql-file-loader": ^7.5.0 + "@graphql-tools/json-file-loader": ^7.4.1 + "@graphql-tools/load": ^7.8.0 + "@graphql-tools/prisma-loader": ^7.2.49 + "@graphql-tools/url-loader": ^7.13.2 + "@graphql-tools/utils": ^9.0.0 + "@parcel/watcher": ^2.1.0 + "@whatwg-node/fetch": ^0.8.0 chalk: ^4.1.0 - chokidar: ^3.5.2 cosmiconfig: ^7.0.0 - cross-undici-fetch: ^0.4.11 debounce: ^1.2.0 detect-indent: ^6.0.0 - graphql-config: ^4.3.1 + graphql-config: ^4.5.0 inquirer: ^8.0.0 is-glob: ^4.0.1 + jiti: ^1.17.1 json-to-pretty-yaml: ^1.2.2 listr2: ^4.0.5 log-symbols: ^4.0.0 - mkdirp: ^1.0.4 + micromatch: ^4.0.5 + shell-quote: ^1.7.3 string-env-interpolation: ^1.0.1 ts-log: ^2.2.3 + tslib: ^2.4.0 yaml: ^1.10.0 yargs: ^17.0.0 peerDependencies: @@ -1301,43 +1319,43 @@ __metadata: graphql-code-generator: cjs/bin.js graphql-codegen: cjs/bin.js graphql-codegen-esm: esm/bin.js - checksum: d25aca360110709a2ce023e224d34e2c4466b87951d0b16ca2e73ed0d1fcafd2fc452e47f048fb511294d7811c9bf1c969b1fa2b0e290885573b20ecfa792895 + checksum: cc3c0b8f1fd8150591b35b549a04577e84ac9c30396ad100196f594f741dd34e69ec9b1335ec41c00afe4ecf1d7e3505c1859fa244bdef1d10753d2b441de962 languageName: node linkType: hard -"@graphql-codegen/core@npm:2.6.0": - version: 2.6.0 - resolution: "@graphql-codegen/core@npm:2.6.0" +"@graphql-codegen/core@npm:^3.1.0": + version: 3.1.0 + resolution: "@graphql-codegen/core@npm:3.1.0" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-tools/schema": ^8.5.0 - "@graphql-tools/utils": ^8.8.0 - tslib: ~2.4.0 + "@graphql-codegen/plugin-helpers": ^4.1.0 + "@graphql-tools/schema": ^9.0.0 + "@graphql-tools/utils": ^9.1.1 + tslib: ~2.5.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: b5baa2e1da5f151bc83322c1f4920a518401cf9dd9751b1d6d6ccf287298713db6a31ccfcc7525712ad3576f2ae35705839738fd43ebb3496bb172a265e018fe + checksum: 7ace2b185242d00575b877b0bbb884c66915d246ba17a98c0ab82c2ee3a93ff2c9ae9d7d75a1d346de02ca8f12eb02801ffe2da4846da6e164b43176cd02dd8c languageName: node linkType: hard -"@graphql-codegen/near-operation-file-preset@npm:2.3.1": - version: 2.3.1 - resolution: "@graphql-codegen/near-operation-file-preset@npm:2.3.1" +"@graphql-codegen/near-operation-file-preset@npm:2.5.0": + version: 2.5.0 + resolution: "@graphql-codegen/near-operation-file-preset@npm:2.5.0" dependencies: - "@graphql-codegen/add": ^3.2.0 - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-codegen/visitor-plugin-common": 2.11.1 + "@graphql-codegen/add": ^3.2.1 + "@graphql-codegen/plugin-helpers": ^2.7.2 + "@graphql-codegen/visitor-plugin-common": 2.13.1 "@graphql-tools/utils": ^8.8.0 parse-filepath: ^1.0.2 tslib: ~2.4.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: b824d130ab9a81453dc1779a82c8f770a71eb3f7cefac8a6f34c940aa77b96b7bf6a44ccf761542efd77a73cfeaad567e6bc14e8c65aeaad9d9ffefc1097c71f + checksum: 4d66e1e3df6530d561b5bc1eb2a9341f96c1c77b3247fb4bb03e2694e2da73d64ec97249a9c3c47416e1eaca2f15b189ec108e19a599076b27f2088fcbf4daf8 languageName: node linkType: hard -"@graphql-codegen/plugin-helpers@npm:^2.5.0": - version: 2.5.0 - resolution: "@graphql-codegen/plugin-helpers@npm:2.5.0" +"@graphql-codegen/plugin-helpers@npm:^2.7.2": + version: 2.7.2 + resolution: "@graphql-codegen/plugin-helpers@npm:2.7.2" dependencies: "@graphql-tools/utils": ^8.8.0 change-case-all: 1.0.14 @@ -1347,58 +1365,90 @@ __metadata: tslib: ~2.4.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: 18a07aebae2e21b8ff7abc28a8e40fc48e1880d187b2499a2da26d482446e0cfb059f41ce29ccedc1a19c04135c9db12e4fcf7f34f622f59b4f2acc255f21e7d + checksum: 66e0d507ad5db60b67092ebf7632d464d56ab446ac8fd87c293e00d9016944912d8cf9199e3e026b0a9247a50f50c4118a44f49e13675db64211652cd6259b05 languageName: node linkType: hard -"@graphql-codegen/schema-ast@npm:2.5.0, @graphql-codegen/schema-ast@npm:^2.5.0": - version: 2.5.0 - resolution: "@graphql-codegen/schema-ast@npm:2.5.0" +"@graphql-codegen/plugin-helpers@npm:^3.1.1": + version: 3.1.2 + resolution: "@graphql-codegen/plugin-helpers@npm:3.1.2" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-tools/utils": ^8.8.0 + "@graphql-tools/utils": ^9.0.0 + change-case-all: 1.0.15 + common-tags: 1.8.2 + import-from: 4.0.0 + lodash: ~4.17.0 tslib: ~2.4.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: cdab4f43885c9bdce7adf93784771a1568f49aa820fdf97c667fe7ddbd1fc7431b17866338464d3826d9ed086505df124fdafa55c412de0a7ee04b6f8a13e4b4 + checksum: 4d0c615738570681b5ffd3c07305a35d6aa3e5fd87c9199c0a670b95529ab865b1df978ce584d5b415107a567ac484e56a48db129a6d1d2eb8a254fbd3260e39 languageName: node linkType: hard -"@graphql-codegen/typescript-operations@npm:2.5.1": - version: 2.5.1 - resolution: "@graphql-codegen/typescript-operations@npm:2.5.1" +"@graphql-codegen/plugin-helpers@npm:^4.1.0, @graphql-codegen/plugin-helpers@npm:^4.2.0": + version: 4.2.0 + resolution: "@graphql-codegen/plugin-helpers@npm:4.2.0" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-codegen/typescript": ^2.7.1 - "@graphql-codegen/visitor-plugin-common": 2.11.1 + "@graphql-tools/utils": ^9.0.0 + change-case-all: 1.0.15 + common-tags: 1.8.2 + import-from: 4.0.0 + lodash: ~4.17.0 + tslib: ~2.5.0 + peerDependencies: + graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 + checksum: 5d26adc132026916db061d23b06fc2c329f372f19ecf56e39bd2b30082bff642f2030cd6dc0bad8d2d3ab781c1e145384e040310a479b83d85ec804b2983646d + languageName: node + linkType: hard + +"@graphql-codegen/schema-ast@npm:3.0.1, @graphql-codegen/schema-ast@npm:^3.0.1": + version: 3.0.1 + resolution: "@graphql-codegen/schema-ast@npm:3.0.1" + dependencies: + "@graphql-codegen/plugin-helpers": ^4.1.0 + "@graphql-tools/utils": ^9.0.0 + tslib: ~2.5.0 + peerDependencies: + graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 + checksum: 620aa67a4ae59ccb4609763b7347d05e2cec62bf1362be3e1e01fc00969cdbb858398542aa261128e5b5e3cb6808b77861bdcf82662e80326e72b418f25f465f + languageName: node + linkType: hard + +"@graphql-codegen/typescript-operations@npm:3.0.4": + version: 3.0.4 + resolution: "@graphql-codegen/typescript-operations@npm:3.0.4" + dependencies: + "@graphql-codegen/plugin-helpers": ^4.2.0 + "@graphql-codegen/typescript": ^3.0.4 + "@graphql-codegen/visitor-plugin-common": 3.1.1 auto-bind: ~4.0.0 - tslib: ~2.4.0 + tslib: ~2.5.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: 5ceadc52704bcdebcddd5f002b5d65463fecf7530c2527d23474d538fa7203fca37c6a1c7aa51ddccb384a723f1aed99b8e915f81bbf8f0fdf0c985263b3dc62 + checksum: 734ce9d77d377037b2ac8ae70c02b7d733dee1917bb0559daedc44afade1b7eea2b7d88f10404c2fca82719fd9588a5e482e609fae1169639b8b94a3e7577fcd languageName: node linkType: hard -"@graphql-codegen/typescript@npm:2.7.1, @graphql-codegen/typescript@npm:^2.7.1": - version: 2.7.1 - resolution: "@graphql-codegen/typescript@npm:2.7.1" +"@graphql-codegen/typescript@npm:3.0.4, @graphql-codegen/typescript@npm:^3.0.4": + version: 3.0.4 + resolution: "@graphql-codegen/typescript@npm:3.0.4" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 - "@graphql-codegen/schema-ast": ^2.5.0 - "@graphql-codegen/visitor-plugin-common": 2.11.1 + "@graphql-codegen/plugin-helpers": ^4.2.0 + "@graphql-codegen/schema-ast": ^3.0.1 + "@graphql-codegen/visitor-plugin-common": 3.1.1 auto-bind: ~4.0.0 - tslib: ~2.4.0 + tslib: ~2.5.0 peerDependencies: graphql: ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: 2755e204392388c54b62a087abe268cbfdd21c38ec4182e840307b4553d8cfac22a32936ff5076cd5c21b1592ecc22454764b7b9c50015c36dc0cd6390983ac0 + checksum: ede5311c5f620e16c3e98f51bdcedc7fd42f2c552a8ca5a946f757dc9709c384a12d32d088769d04041461d3318159ad0644f70056aea1877ee80a0bdc48de1f languageName: node linkType: hard -"@graphql-codegen/visitor-plugin-common@npm:2.11.1": - version: 2.11.1 - resolution: "@graphql-codegen/visitor-plugin-common@npm:2.11.1" +"@graphql-codegen/visitor-plugin-common@npm:2.13.1": + version: 2.13.1 + resolution: "@graphql-codegen/visitor-plugin-common@npm:2.13.1" dependencies: - "@graphql-codegen/plugin-helpers": ^2.5.0 + "@graphql-codegen/plugin-helpers": ^2.7.2 "@graphql-tools/optimize": ^1.3.0 "@graphql-tools/relay-operation-optimizer": ^6.5.0 "@graphql-tools/utils": ^8.8.0 @@ -1410,21 +1460,41 @@ __metadata: tslib: ~2.4.0 peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: ea425894d93ac89b1811c1fff4451a345695804decac08ce140fd306ad826ac21cc9936e89b52c71ba4a9b16ef7623cedfc04a0fd5c81dcaf2a8b47474bc365c + checksum: 0c329aa6e435602f2f6c1569ec2091b7850f58cc5dca7ac763c38c82588545ec1110c1de587f5f3949b11ff96f94401d1e63e329607d78424583b276fd08f1ae languageName: node linkType: hard -"@graphql-tools/apollo-engine-loader@npm:^7.3.1": - version: 7.3.1 - resolution: "@graphql-tools/apollo-engine-loader@npm:7.3.1" +"@graphql-codegen/visitor-plugin-common@npm:3.1.1": + version: 3.1.1 + resolution: "@graphql-codegen/visitor-plugin-common@npm:3.1.1" dependencies: - "@graphql-tools/utils": 8.8.0 - cross-undici-fetch: ^0.4.11 - sync-fetch: 0.4.1 + "@graphql-codegen/plugin-helpers": ^4.2.0 + "@graphql-tools/optimize": ^1.3.0 + "@graphql-tools/relay-operation-optimizer": ^6.5.0 + "@graphql-tools/utils": ^9.0.0 + auto-bind: ~4.0.0 + change-case-all: 1.0.15 + dependency-graph: ^0.11.0 + graphql-tag: ^2.11.0 + parse-filepath: ^1.0.2 + tslib: ~2.5.0 + peerDependencies: + graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 + checksum: 8dbef75ba0576a7de5e134b53778d64d818084df01e74554adc979f08db8446e9734889575c820449b6ef9febd6a9005fb89c7c9c101a9e398c195b5c3279353 + languageName: node + linkType: hard + +"@graphql-tools/apollo-engine-loader@npm:^7.3.6": + version: 7.3.26 + resolution: "@graphql-tools/apollo-engine-loader@npm:7.3.26" + dependencies: + "@ardatan/sync-fetch": ^0.0.1 + "@graphql-tools/utils": ^9.2.1 + "@whatwg-node/fetch": ^0.8.0 tslib: ^2.4.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 7161cd58984b51313abed8fc0f91959d49efd1c2dfd4f7aec746e1eb08b9b3baf92dab143e12596305a95e49752496961e413a94d100dd65ceb45f8179dc20b6 + checksum: 2dd0324cd677c0a399a3cda6f155e4367ac38c8f6ec8a36c50931e97ce93d70f716f95575bcfda33d5a5b3c75f0ba04c73b82d22613f1a89e3c31965f576ae22 languageName: node linkType: hard @@ -1442,18 +1512,32 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/code-file-loader@npm:^7.3.0": - version: 7.3.0 - resolution: "@graphql-tools/code-file-loader@npm:7.3.0" +"@graphql-tools/batch-execute@npm:^8.5.22": + version: 8.5.22 + resolution: "@graphql-tools/batch-execute@npm:8.5.22" dependencies: - "@graphql-tools/graphql-tag-pluck": 7.3.0 - "@graphql-tools/utils": 8.8.0 + "@graphql-tools/utils": ^9.2.1 + dataloader: ^2.2.2 + tslib: ^2.4.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 4f7f7ba104737a57d0bd5b2b38b8dbf24d6db9e3ce8f8d1b8c3109702305c212b54b7642853c1584b12ce3a9136286b600076e94861c0bd1d29a218f44401224 + languageName: node + linkType: hard + +"@graphql-tools/code-file-loader@npm:^7.3.17": + version: 7.3.23 + resolution: "@graphql-tools/code-file-loader@npm:7.3.23" + dependencies: + "@graphql-tools/graphql-tag-pluck": 7.5.2 + "@graphql-tools/utils": ^9.2.1 globby: ^11.0.3 tslib: ^2.4.0 unixify: ^1.0.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: ab184dd2bec2be9f1c39cc5cae493a539dd5314969af577cfa6f62ae0675b402367b7ebf0a5a88d22cc3ce74983e57fe2e64527f55543e6bc5123baddf619970 + checksum: fb1dfa807b9d5798936c7fe31cf5356412d9b5a25a08d5952b607921637afbe26555cb662cf97f82dfdf47ed8e7c2a42f527238fb2defd3be4505e15fb6027c3 languageName: node linkType: hard @@ -1473,38 +1557,122 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/git-loader@npm:^7.2.0": - version: 7.2.0 - resolution: "@graphql-tools/git-loader@npm:7.2.0" +"@graphql-tools/delegate@npm:^9.0.31": + version: 9.0.35 + resolution: "@graphql-tools/delegate@npm:9.0.35" dependencies: - "@graphql-tools/graphql-tag-pluck": 7.3.0 - "@graphql-tools/utils": 8.8.0 + "@graphql-tools/batch-execute": ^8.5.22 + "@graphql-tools/executor": ^0.0.20 + "@graphql-tools/schema": ^9.0.19 + "@graphql-tools/utils": ^9.2.1 + dataloader: ^2.2.2 + tslib: ^2.5.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 4edd827d1767dc33ea1e28f8ceb30f82e1cdb47a9390cecc0a45f51594c6df28d11b2c2eec083e27fa45a76451c8490f2f826effc2dff9977de3fe1b66b2aadb + languageName: node + linkType: hard + +"@graphql-tools/executor-graphql-ws@npm:^0.0.14": + version: 0.0.14 + resolution: "@graphql-tools/executor-graphql-ws@npm:0.0.14" + dependencies: + "@graphql-tools/utils": ^9.2.1 + "@repeaterjs/repeater": 3.0.4 + "@types/ws": ^8.0.0 + graphql-ws: 5.12.1 + isomorphic-ws: 5.0.0 + tslib: ^2.4.0 + ws: 8.13.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: c18f3ca3d70098017ff71045ae13de1d88c8dc0954af0d7a389aebdc831c82b678f9cf9b50ed065d5262d59a558b4f9be3b7b04e5002bae47a503493fc0b7542 + languageName: node + linkType: hard + +"@graphql-tools/executor-http@npm:^0.1.7, @graphql-tools/executor-http@npm:^0.1.9": + version: 0.1.10 + resolution: "@graphql-tools/executor-http@npm:0.1.10" + dependencies: + "@graphql-tools/utils": ^9.2.1 + "@repeaterjs/repeater": ^3.0.4 + "@whatwg-node/fetch": ^0.8.1 + dset: ^3.1.2 + extract-files: ^11.0.0 + meros: ^1.2.1 + tslib: ^2.4.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: d5cb0b9f8deb2335cac3b0e8fa5a63e827fafd35d1dae88f4ae201f3ce0531be95a8ec3b0b7fbe618a66ad5838e3c574cf8f965c3d71b49b7dbcd7ba2e67019d + languageName: node + linkType: hard + +"@graphql-tools/executor-legacy-ws@npm:^0.0.11": + version: 0.0.11 + resolution: "@graphql-tools/executor-legacy-ws@npm:0.0.11" + dependencies: + "@graphql-tools/utils": ^9.2.1 + "@types/ws": ^8.0.0 + isomorphic-ws: 5.0.0 + tslib: ^2.4.0 + ws: 8.13.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: f9dd5dc87537c6adb3e1fb8e083944cfd9b2a9b34016f705b7b99105e744f11290f23aee726bb05ae32411c7d07a1ebc7b5bd35445053fc44877979f0ce4cd2e + languageName: node + linkType: hard + +"@graphql-tools/executor@npm:^0.0.20": + version: 0.0.20 + resolution: "@graphql-tools/executor@npm:0.0.20" + dependencies: + "@graphql-tools/utils": ^9.2.1 + "@graphql-typed-document-node/core": 3.2.0 + "@repeaterjs/repeater": ^3.0.4 + tslib: ^2.4.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 5390800be4a346eb7d37d51458cd9cf4a24e01014fe1b392682ea4bce2b27fc1d5c7aebcb3dafaeefcb0a89e5b307fc2060816533b61d6796e0fa0e5e1f10959 + languageName: node + linkType: hard + +"@graphql-tools/git-loader@npm:^7.2.13": + version: 7.3.0 + resolution: "@graphql-tools/git-loader@npm:7.3.0" + dependencies: + "@graphql-tools/graphql-tag-pluck": 7.5.2 + "@graphql-tools/utils": ^9.2.1 is-glob: 4.0.3 micromatch: ^4.0.4 tslib: ^2.4.0 unixify: ^1.0.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 28db64d4ec5c59f4ebfd2ae06b9566ed0dfa47b75264b63c908b1a1da2b30a97ccf3a2e66f32d2625ef26d82940610371a949892c9f22c88fa95419898c3ef46 + checksum: 9d0ae74188d728daabdb22bf86beb22af9583e700e6ab732b3cbe09143f0d1c5b235c8dc941229707be796ef28373838c8354f222c8b0b181cc748bfa03fc3a3 languageName: node linkType: hard -"@graphql-tools/github-loader@npm:^7.3.1": - version: 7.3.1 - resolution: "@graphql-tools/github-loader@npm:7.3.1" +"@graphql-tools/github-loader@npm:^7.3.20": + version: 7.3.28 + resolution: "@graphql-tools/github-loader@npm:7.3.28" dependencies: - "@graphql-tools/graphql-tag-pluck": 7.3.0 - "@graphql-tools/utils": 8.8.0 - cross-undici-fetch: ^0.4.11 - sync-fetch: 0.4.1 + "@ardatan/sync-fetch": ^0.0.1 + "@graphql-tools/executor-http": ^0.1.9 + "@graphql-tools/graphql-tag-pluck": ^7.4.6 + "@graphql-tools/utils": ^9.2.1 + "@whatwg-node/fetch": ^0.8.0 tslib: ^2.4.0 + value-or-promise: ^1.0.12 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: e65a691e085e96d44cd5f68ef95f9a07f2b0307178d1cafac2044ca612a84de6d902aef02d50ddc7044e18b7e998841cd0330185078705e15cff0023a80cc9c2 + checksum: 1ef168d72b0615e5e05408794fef549e841c399a12b7074ae4764fee28d145aebdf50ba573f0695159edced626f5757b7825be2b246c437bbdf5457aeff13e5b languageName: node linkType: hard -"@graphql-tools/graphql-file-loader@npm:^7.3.7, @graphql-tools/graphql-file-loader@npm:^7.4.0": +"@graphql-tools/graphql-file-loader@npm:^7.3.7": version: 7.4.0 resolution: "@graphql-tools/graphql-file-loader@npm:7.4.0" dependencies: @@ -1519,18 +1687,34 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/graphql-tag-pluck@npm:7.3.0": - version: 7.3.0 - resolution: "@graphql-tools/graphql-tag-pluck@npm:7.3.0" +"@graphql-tools/graphql-file-loader@npm:^7.5.0": + version: 7.5.17 + resolution: "@graphql-tools/graphql-file-loader@npm:7.5.17" + dependencies: + "@graphql-tools/import": 6.7.18 + "@graphql-tools/utils": ^9.2.1 + globby: ^11.0.3 + tslib: ^2.4.0 + unixify: ^1.0.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: f0d6768fbb03fe6c5a0a2c1fe78e2fa8c009b13d6c40a7153c61e7266348192386310ace85dd46a96fa4317a4a37c02d1959fd2a0c6eaa521446234506147cdc + languageName: node + linkType: hard + +"@graphql-tools/graphql-tag-pluck@npm:7.5.2, @graphql-tools/graphql-tag-pluck@npm:^7.4.6": + version: 7.5.2 + resolution: "@graphql-tools/graphql-tag-pluck@npm:7.5.2" dependencies: "@babel/parser": ^7.16.8 + "@babel/plugin-syntax-import-assertions": ^7.20.0 "@babel/traverse": ^7.16.8 "@babel/types": ^7.16.8 - "@graphql-tools/utils": 8.8.0 + "@graphql-tools/utils": ^9.2.1 tslib: ^2.4.0 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 405fbc33dd976f564fa05a71f4a84acaf9c533ca2d5eb4f71297043225b71ee7cd2cc2f8ce7509b3e0f62201c6e22bdc26613c7505ad101a177e08c7a757d80f + checksum: fbe2419f97ca700bb5f3fa7ff7a4ecab2519732339c2f5807ff0fc33dcb50e3b6e921b6c0b285992b34e95cb812d514f0d62d82f9275a8c074bcaff64cbff7bb languageName: node linkType: hard @@ -1547,7 +1731,20 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/json-file-loader@npm:^7.3.7, @graphql-tools/json-file-loader@npm:^7.4.0": +"@graphql-tools/import@npm:6.7.18": + version: 6.7.18 + resolution: "@graphql-tools/import@npm:6.7.18" + dependencies: + "@graphql-tools/utils": ^9.2.1 + resolve-from: 5.0.0 + tslib: ^2.4.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 15c32c5937899a25f8c2b0dee98ca1e1245ba85a56a8a59d52c2c78693da2e95fb27f235ef95c3a576bd96843d53541b6d90931a0032c0011dea871d53b5027a + languageName: node + linkType: hard + +"@graphql-tools/json-file-loader@npm:^7.3.7": version: 7.4.0 resolution: "@graphql-tools/json-file-loader@npm:7.4.0" dependencies: @@ -1561,7 +1758,21 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/load@npm:^7.5.5, @graphql-tools/load@npm:^7.7.0": +"@graphql-tools/json-file-loader@npm:^7.4.1": + version: 7.4.18 + resolution: "@graphql-tools/json-file-loader@npm:7.4.18" + dependencies: + "@graphql-tools/utils": ^9.2.1 + globby: ^11.0.3 + tslib: ^2.4.0 + unixify: ^1.0.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: e6571fb10bdf29c4e5aabdc9c87d32be0d1e493a701886fc9c24efee2e0cef0df898a9a48c449f0465a89da816e9f2cf7a51e9476fbe8b7d0aefd3c18e934234 + languageName: node + linkType: hard + +"@graphql-tools/load@npm:^7.5.5": version: 7.7.0 resolution: "@graphql-tools/load@npm:7.7.0" dependencies: @@ -1575,6 +1786,20 @@ __metadata: languageName: node linkType: hard +"@graphql-tools/load@npm:^7.8.0": + version: 7.8.14 + resolution: "@graphql-tools/load@npm:7.8.14" + dependencies: + "@graphql-tools/schema": ^9.0.18 + "@graphql-tools/utils": ^9.2.1 + p-limit: 3.1.0 + tslib: ^2.4.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 12ffd6460da3d996d614faa3ced99f526247334bb671301b15ed1d2153314a8813f734d863086d154891ac4b35da090668f0ea7702508d19f8dd0f72413b585c + languageName: node + linkType: hard + "@graphql-tools/merge@npm:8.3.0, @graphql-tools/merge@npm:^8.2.6": version: 8.3.0 resolution: "@graphql-tools/merge@npm:8.3.0" @@ -1587,6 +1812,18 @@ __metadata: languageName: node linkType: hard +"@graphql-tools/merge@npm:^8.4.1": + version: 8.4.2 + resolution: "@graphql-tools/merge@npm:8.4.2" + dependencies: + "@graphql-tools/utils": ^9.2.1 + tslib: ^2.4.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 96d57a3e810055a2883bf9d3450e88082da207ffb1406222c9fa954e47bac4a328696785fdb7eec95a030d5f75504f7b4c6484c94f248cee13e6ad25aca70c75 + languageName: node + linkType: hard + "@graphql-tools/optimize@npm:^1.3.0": version: 1.3.0 resolution: "@graphql-tools/optimize@npm:1.3.0" @@ -1598,33 +1835,31 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/prisma-loader@npm:^7.2.2": - version: 7.2.2 - resolution: "@graphql-tools/prisma-loader@npm:7.2.2" +"@graphql-tools/prisma-loader@npm:^7.2.49": + version: 7.2.72 + resolution: "@graphql-tools/prisma-loader@npm:7.2.72" dependencies: - "@graphql-tools/url-loader": 7.12.1 - "@graphql-tools/utils": 8.8.0 + "@graphql-tools/url-loader": ^7.17.18 + "@graphql-tools/utils": ^9.2.1 "@types/js-yaml": ^4.0.0 "@types/json-stable-stringify": ^1.0.32 - "@types/jsonwebtoken": ^8.5.0 + "@whatwg-node/fetch": ^0.8.2 chalk: ^4.1.0 debug: ^4.3.1 dotenv: ^16.0.0 - graphql-request: ^4.0.0 - http-proxy-agent: ^5.0.0 - https-proxy-agent: ^5.0.0 - isomorphic-fetch: ^3.0.0 + graphql-request: ^6.0.0 + http-proxy-agent: ^6.0.0 + https-proxy-agent: ^6.0.0 + jose: ^4.11.4 js-yaml: ^4.0.0 json-stable-stringify: ^1.0.1 - jsonwebtoken: ^8.5.1 lodash: ^4.17.20 - replaceall: ^0.1.6 scuid: ^1.1.0 tslib: ^2.4.0 yaml-ast-parser: ^0.0.43 peerDependencies: graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - checksum: 44601d4ee4747a6d080ab2a0449bc203c4eb12251716127a12535abe8b889f3dd31d3363f0f1bd6ed72b31686a8240e3272d93827f6082d81d17140b4e4556ad + checksum: 949506d2306ef54a8c68152b93c574148ad03c9bf3f5042fbd6aff0e6fe77c8afa3bc3ffceea239afd4ebda5cc0bd3076b5dc939645b838c472c958c75f1deaf languageName: node linkType: hard @@ -1641,7 +1876,7 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/schema@npm:8.5.0, @graphql-tools/schema@npm:^8.5.0": +"@graphql-tools/schema@npm:8.5.0": version: 8.5.0 resolution: "@graphql-tools/schema@npm:8.5.0" dependencies: @@ -1655,7 +1890,44 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/url-loader@npm:7.12.1, @graphql-tools/url-loader@npm:^7.12.1, @graphql-tools/url-loader@npm:^7.9.7": +"@graphql-tools/schema@npm:^9.0.0, @graphql-tools/schema@npm:^9.0.18, @graphql-tools/schema@npm:^9.0.19": + version: 9.0.19 + resolution: "@graphql-tools/schema@npm:9.0.19" + dependencies: + "@graphql-tools/merge": ^8.4.1 + "@graphql-tools/utils": ^9.2.1 + tslib: ^2.4.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 1be91f61bf4be0c1c9aa640a6ad5b58328d5434d15e78ba73a47263420939db6741ad6723dece4611257e7e1e56518e116b76513a3014305d3f52d67aafb62fb + languageName: node + linkType: hard + +"@graphql-tools/url-loader@npm:^7.13.2, @graphql-tools/url-loader@npm:^7.17.18": + version: 7.17.18 + resolution: "@graphql-tools/url-loader@npm:7.17.18" + dependencies: + "@ardatan/sync-fetch": ^0.0.1 + "@graphql-tools/delegate": ^9.0.31 + "@graphql-tools/executor-graphql-ws": ^0.0.14 + "@graphql-tools/executor-http": ^0.1.7 + "@graphql-tools/executor-legacy-ws": ^0.0.11 + "@graphql-tools/utils": ^9.2.1 + "@graphql-tools/wrap": ^9.4.2 + "@types/ws": ^8.0.0 + "@whatwg-node/fetch": ^0.8.0 + isomorphic-ws: ^5.0.0 + tslib: ^2.4.0 + value-or-promise: ^1.0.11 + ws: ^8.12.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: e4deccaa4b333a91022e9a19594e6c696c4463c94f091893c8d056e4090b2c8c5e5036b0e7bcce79f0c4c0ad2f0e6f3c8d170a765f0d5a2ba29965bee096f355 + languageName: node + linkType: hard + +"@graphql-tools/url-loader@npm:^7.9.7": version: 7.12.1 resolution: "@graphql-tools/url-loader@npm:7.12.1" dependencies: @@ -1680,7 +1952,7 @@ __metadata: languageName: node linkType: hard -"@graphql-tools/utils@npm:8.8.0, @graphql-tools/utils@npm:^8.6.5, @graphql-tools/utils@npm:^8.8.0": +"@graphql-tools/utils@npm:8.8.0, @graphql-tools/utils@npm:^8.8.0": version: 8.8.0 resolution: "@graphql-tools/utils@npm:8.8.0" dependencies: @@ -1691,6 +1963,18 @@ __metadata: languageName: node linkType: hard +"@graphql-tools/utils@npm:^9.0.0, @graphql-tools/utils@npm:^9.1.1, @graphql-tools/utils@npm:^9.2.1": + version: 9.2.1 + resolution: "@graphql-tools/utils@npm:9.2.1" + dependencies: + "@graphql-typed-document-node/core": ^3.1.1 + tslib: ^2.4.0 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 94ed12df5f49e5c338322ffd931236a687a3d5c443bf499f9baab5d4fcd9792234111142be8aa506a01ca2e82732996c4e1d8f6159ff9cc7fdc5c97f63e55226 + languageName: node + linkType: hard + "@graphql-tools/wrap@npm:8.5.0": version: 8.5.0 resolution: "@graphql-tools/wrap@npm:8.5.0" @@ -1706,6 +1990,30 @@ __metadata: languageName: node linkType: hard +"@graphql-tools/wrap@npm:^9.4.2": + version: 9.4.2 + resolution: "@graphql-tools/wrap@npm:9.4.2" + dependencies: + "@graphql-tools/delegate": ^9.0.31 + "@graphql-tools/schema": ^9.0.18 + "@graphql-tools/utils": ^9.2.1 + tslib: ^2.4.0 + value-or-promise: ^1.0.12 + peerDependencies: + graphql: ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: 294d529a4b8e90cceaaa691e3b67d932626ad59a53260166e0281506a4e1a2b9d1c018984dffd0edcf518555baee23beaa8665167226da014d4d0b58c37cd744 + languageName: node + linkType: hard + +"@graphql-typed-document-node/core@npm:3.2.0, @graphql-typed-document-node/core@npm:^3.2.0": + version: 3.2.0 + resolution: "@graphql-typed-document-node/core@npm:3.2.0" + peerDependencies: + graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + checksum: fa44443accd28c8cf4cb96aaaf39d144a22e8b091b13366843f4e97d19c7bfeaf609ce3c7603a4aeffe385081eaf8ea245d078633a7324c11c5ec4b2011bb76d + languageName: node + linkType: hard + "@graphql-typed-document-node/core@npm:^3.1.1": version: 3.1.1 resolution: "@graphql-typed-document-node/core@npm:3.1.1" @@ -1715,13 +2023,6 @@ __metadata: languageName: node linkType: hard -"@iarna/toml@npm:^2.2.5": - version: 2.2.5 - resolution: "@iarna/toml@npm:2.2.5" - checksum: b63b2b2c4fd67969a6291543ada0303d45593801ee744b60f5390f183c03d9192bc67a217abb24be945158f1935f02840d9ffff40c0142aa171b5d3b6b6a3ea5 - languageName: node - linkType: hard - "@istanbuljs/schema@npm:^0.1.2, @istanbuljs/schema@npm:^0.1.3": version: 0.1.3 resolution: "@istanbuljs/schema@npm:0.1.3" @@ -1890,11 +2191,11 @@ __metadata: resolution: "@madatdata/db-splitgraph@workspace:packages/db-splitgraph" dependencies: "@faker-js/faker": 7.3.0 - "@graphql-codegen/cli": 2.8.0 - "@graphql-codegen/near-operation-file-preset": 2.3.1 - "@graphql-codegen/schema-ast": 2.5.0 - "@graphql-codegen/typescript": 2.7.1 - "@graphql-codegen/typescript-operations": 2.5.1 + "@graphql-codegen/cli": 3.3.1 + "@graphql-codegen/near-operation-file-preset": 2.5.0 + "@graphql-codegen/schema-ast": 3.0.1 + "@graphql-codegen/typescript": 3.0.4 + "@graphql-codegen/typescript-operations": 3.0.4 "@madatdata/base-client": "workspace:*" "@madatdata/base-db": "workspace:*" "@madatdata/client-http": "workspace:*" @@ -1904,7 +2205,7 @@ __metadata: esno: 0.16.3 graphql: 16.5.0 graphql-request: 5.1.0 - json-schema-to-typescript: 10.1.5 + json-schema-to-typescript: 13.0.1 rimraf: 3.0.2 typescript-retry-decorator: 2.0.5 languageName: unknown @@ -2084,10 +2385,63 @@ __metadata: languageName: node linkType: hard -"@open-draft/until@npm:^1.0.3": - version: 1.0.3 - resolution: "@open-draft/until@npm:1.0.3" - checksum: 323e92ebef0150ed0f8caedc7d219b68cdc50784fa4eba0377eef93533d3f46514eb2400ced83dda8c51bddc3d2c7b8e9cf95e5ec85ab7f62dfc015d174f62f2 +"@open-draft/until@npm:^1.0.3": + version: 1.0.3 + resolution: "@open-draft/until@npm:1.0.3" + checksum: 323e92ebef0150ed0f8caedc7d219b68cdc50784fa4eba0377eef93533d3f46514eb2400ced83dda8c51bddc3d2c7b8e9cf95e5ec85ab7f62dfc015d174f62f2 + languageName: node + linkType: hard + +"@parcel/watcher@npm:^2.1.0": + version: 2.1.0 + resolution: "@parcel/watcher@npm:2.1.0" + dependencies: + is-glob: ^4.0.3 + micromatch: ^4.0.5 + node-addon-api: ^3.2.1 + node-gyp: latest + node-gyp-build: ^4.3.0 + checksum: 17f512ad6d5dbb40053ceea7091f8af754afc63786b8f050b225b89a8ba24900468aad8bc4edb25c0349b4c0c8d061f50aa19242c0af52cbc30e6ebf50c7bf4c + languageName: node + linkType: hard + +"@peculiar/asn1-schema@npm:^2.3.6": + version: 2.3.6 + resolution: "@peculiar/asn1-schema@npm:2.3.6" + dependencies: + asn1js: ^3.0.5 + pvtsutils: ^1.3.2 + tslib: ^2.4.0 + checksum: fc09387c6e3dea07fca21b54ea8c71ce3ec0f8c92377237e51aef729f0c2df92781aa7a18a546a6fe809519faeaa222df576ec21a35c6095037a78677204a55b + languageName: node + linkType: hard + +"@peculiar/json-schema@npm:^1.1.12": + version: 1.1.12 + resolution: "@peculiar/json-schema@npm:1.1.12" + dependencies: + tslib: ^2.0.0 + checksum: b26ececdc23c5ef25837f8be8d1eb5e1c8bb6e9ae7227ac59ffea57fff56bd05137734e7685e9100595d3d88d906dff638ef8d1df54264c388d3eac1b05aa060 + languageName: node + linkType: hard + +"@peculiar/webcrypto@npm:^1.4.0": + version: 1.4.3 + resolution: "@peculiar/webcrypto@npm:1.4.3" + dependencies: + "@peculiar/asn1-schema": ^2.3.6 + "@peculiar/json-schema": ^1.1.12 + pvtsutils: ^1.3.2 + tslib: ^2.5.0 + webcrypto-core: ^1.7.7 + checksum: 5604c02b7e9a8cef61bb4430e733e939c7737533ba65ba5fac4beb3a6d613add478ab45455cb57506789b6d00704d83e4965a0f712de3e8f40706e0961670e5c + languageName: node + linkType: hard + +"@repeaterjs/repeater@npm:3.0.4, @repeaterjs/repeater@npm:^3.0.4": + version: 3.0.4 + resolution: "@repeaterjs/repeater@npm:3.0.4" + checksum: cca0db3e802bc26fcce0b4a574074d9956da53bf43094de03c0e4732d05e13441279a92f0b96e2a7a39da50933684947a138c1213406eaafe39cfd4683d6c0df languageName: node linkType: hard @@ -2203,7 +2557,7 @@ __metadata: languageName: node linkType: hard -"@types/glob@npm:*": +"@types/glob@npm:^7.1.3": version: 7.2.0 resolution: "@types/glob@npm:7.2.0" dependencies: @@ -2234,7 +2588,7 @@ __metadata: languageName: node linkType: hard -"@types/json-schema@npm:^7.0.6": +"@types/json-schema@npm:^7.0.11, @types/json-schema@npm:^7.0.6": version: 7.0.11 resolution: "@types/json-schema@npm:7.0.11" checksum: 527bddfe62db9012fccd7627794bd4c71beb77601861055d87e3ee464f2217c85fca7a4b56ae677478367bbd248dbde13553312b7d4dbc702a2f2bbf60c4018d @@ -2257,19 +2611,10 @@ __metadata: languageName: node linkType: hard -"@types/jsonwebtoken@npm:^8.5.0": - version: 8.5.8 - resolution: "@types/jsonwebtoken@npm:8.5.8" - dependencies: - "@types/node": "*" - checksum: 56738a918c543dba30786066959f801212e7fb5cd4ec53cf7b8d227711ed358834feb9e5141f7f88ec7c642bb39757330a5a8917e3b22e0ff9084940d35f0d70 - languageName: node - linkType: hard - -"@types/lodash@npm:^4.14.168": - version: 4.14.182 - resolution: "@types/lodash@npm:4.14.182" - checksum: 7dd137aa9dbabd632408bd37009d984655164fa1ecc3f2b6eb94afe35bf0a5852cbab6183148d883e9c73a958b7fec9a9bcf7c8e45d41195add6a18c34958209 +"@types/lodash@npm:^4.14.182": + version: 4.14.194 + resolution: "@types/lodash@npm:4.14.194" + checksum: 113f34831c461469d91feca2dde737f88487732898b4d25e9eb23b087bb193985f864d1e1e0f3b777edc5022e460443588b6000a3b2348c966f72d17eedc35ea languageName: node linkType: hard @@ -2322,10 +2667,10 @@ __metadata: languageName: node linkType: hard -"@types/prettier@npm:^2.1.5": - version: 2.6.3 - resolution: "@types/prettier@npm:2.6.3" - checksum: e1836699ca189fff6d2a73dc22e028b6a6f693ed1180d5998ac29fa197caf8f85aa92cb38db642e4a370e616b451cb5722ad2395dab11c78e025a1455f37d1f0 +"@types/prettier@npm:^2.6.1": + version: 2.7.2 + resolution: "@types/prettier@npm:2.7.2" + checksum: b47d76a5252265f8d25dd2fe2a5a61dc43ba0e6a96ffdd00c594cb4fd74c1982c2e346497e3472805d97915407a09423804cc2110a0b8e1b22cffcab246479b7 languageName: node linkType: hard @@ -2550,6 +2895,39 @@ __metadata: languageName: node linkType: hard +"@whatwg-node/events@npm:^0.0.3": + version: 0.0.3 + resolution: "@whatwg-node/events@npm:0.0.3" + checksum: af26f40d4d0a0f5f0ee45fc6124afb8d6b33988dae96ab0fb87aa5e66d1ff08a749491b9da533ea524bbaebd4a770736f254d574a91ab4455386aa098cee8c77 + languageName: node + linkType: hard + +"@whatwg-node/fetch@npm:^0.8.0, @whatwg-node/fetch@npm:^0.8.1, @whatwg-node/fetch@npm:^0.8.2": + version: 0.8.8 + resolution: "@whatwg-node/fetch@npm:0.8.8" + dependencies: + "@peculiar/webcrypto": ^1.4.0 + "@whatwg-node/node-fetch": ^0.3.6 + busboy: ^1.6.0 + urlpattern-polyfill: ^8.0.0 + web-streams-polyfill: ^3.2.1 + checksum: 891407ba57e32e5af70a3b0a86980c4466dcf2ba8581b6927475c85400280b163085519e98821dd94776da9aa1b0b1e221e718009e2abed9c8a0d4721025b2ab + languageName: node + linkType: hard + +"@whatwg-node/node-fetch@npm:^0.3.6": + version: 0.3.6 + resolution: "@whatwg-node/node-fetch@npm:0.3.6" + dependencies: + "@whatwg-node/events": ^0.0.3 + busboy: ^1.6.0 + fast-querystring: ^1.1.1 + fast-url-parser: ^1.1.3 + tslib: ^2.3.1 + checksum: d3d7b0a0242c0511c7b666de66d9096fb24ea251426ce76e3a26a8ca17408de5d4d4f81b5aaec840cc7025f0321fb97e06067c53f377c844a5a9473dd76491ae + languageName: node + linkType: hard + "@xmldom/xmldom@npm:^0.8.3": version: 0.8.7 resolution: "@xmldom/xmldom@npm:0.8.7" @@ -2695,6 +3073,15 @@ __metadata: languageName: node linkType: hard +"agent-base@npm:^7.0.1": + version: 7.0.1 + resolution: "agent-base@npm:7.0.1" + dependencies: + debug: ^4.3.4 + checksum: 38d4eed432734b8115baa2a034ddee618f7a27f273c1d31b374f4cc431ed5c56aae4b66a3e8ffe02747e40fd6b4322daf4b44f77ba4375c543cbdfb89bddebf1 + languageName: node + linkType: hard + "agentkeepalive@npm:^4.2.1": version: 4.2.1 resolution: "agentkeepalive@npm:4.2.1" @@ -2728,7 +3115,7 @@ __metadata: languageName: node linkType: hard -"ansi-escapes@npm:^4.2.1, ansi-escapes@npm:^4.3.0, ansi-escapes@npm:^4.3.1": +"ansi-escapes@npm:^4.2.1, ansi-escapes@npm:^4.3.0": version: 4.3.2 resolution: "ansi-escapes@npm:4.3.2" dependencies: @@ -2810,13 +3197,6 @@ __metadata: languageName: node linkType: hard -"arg@npm:^4.1.0": - version: 4.1.3 - resolution: "arg@npm:4.1.3" - checksum: 544af8dd3f60546d3e4aff084d451b96961d2267d668670199692f8d054f0415d86fc5497d0e641e91546f0aa920e7c29e5250e99fc89f5552a34b5d93b77f43 - languageName: node - linkType: hard - "argparse@npm:^2.0.1": version: 2.0.1 resolution: "argparse@npm:2.0.1" @@ -2877,6 +3257,17 @@ __metadata: languageName: node linkType: hard +"asn1js@npm:^3.0.1, asn1js@npm:^3.0.5": + version: 3.0.5 + resolution: "asn1js@npm:3.0.5" + dependencies: + pvtsutils: ^1.3.2 + pvutils: ^1.1.3 + tslib: ^2.4.0 + checksum: 3b6af1bbadd5762ef8ead5daf2f6bda1bc9e23bc825c4dcc996aa1f9521ad7390a64028565d95d98090d69c8431f004c71cccb866004759169d7c203cf9075eb + languageName: node + linkType: hard + "assert-plus@npm:1.0.0, assert-plus@npm:^1.0.0": version: 1.0.0 resolution: "assert-plus@npm:1.0.0" @@ -3147,13 +3538,6 @@ __metadata: languageName: node linkType: hard -"buffer-from@npm:^1.0.0": - version: 1.1.2 - resolution: "buffer-from@npm:1.1.2" - checksum: 0448524a562b37d4d7ed9efd91685a5b77a50672c556ea254ac9a6d30e3403a517d8981f10e565db24e8339413b43c97ca2951f10e399c6125a0d8911f5679bb - languageName: node - linkType: hard - "buffer@npm:^5.5.0, buffer@npm:^5.7.1": version: 5.7.1 resolution: "buffer@npm:5.7.1" @@ -3401,6 +3785,24 @@ __metadata: languageName: node linkType: hard +"change-case-all@npm:1.0.15": + version: 1.0.15 + resolution: "change-case-all@npm:1.0.15" + dependencies: + change-case: ^4.1.2 + is-lower-case: ^2.0.2 + is-upper-case: ^2.0.2 + lower-case: ^2.0.2 + lower-case-first: ^2.0.2 + sponge-case: ^1.0.1 + swap-case: ^2.0.2 + title-case: ^3.0.3 + upper-case: ^2.0.2 + upper-case-first: ^2.0.2 + checksum: e1dabdcd8447a3690f3faf15f92979dfbc113109b50916976e1d5e518e6cfdebee4f05f54d0ca24fb79a4bf835185b59ae25e967bb3dc10bd236a775b19ecc52 + languageName: node + linkType: hard + "change-case@npm:^4.1.2": version: 4.1.2 resolution: "change-case@npm:4.1.2" @@ -3435,7 +3837,7 @@ __metadata: languageName: node linkType: hard -"chokidar@npm:^3.4.2, chokidar@npm:^3.5.2, chokidar@npm:^3.5.3": +"chokidar@npm:^3.4.2, chokidar@npm:^3.5.3": version: 3.5.3 resolution: "chokidar@npm:3.5.3" dependencies: @@ -3468,16 +3870,16 @@ __metadata: languageName: node linkType: hard -"cli-color@npm:^2.0.0": - version: 2.0.2 - resolution: "cli-color@npm:2.0.2" +"cli-color@npm:^2.0.2": + version: 2.0.3 + resolution: "cli-color@npm:2.0.3" dependencies: d: ^1.0.1 - es5-ext: ^0.10.59 + es5-ext: ^0.10.61 es6-iterator: ^2.0.3 memoizee: ^0.4.15 timers-ext: ^0.1.7 - checksum: a1d8cb11128bb02fbffbccc92eb137a1fc280a04c50f125a884c03bfef19b8a0832636d023abd0a1c326618673ee2555785d10261daf65b840ea514da81cfe8e + checksum: b1c5f3d0ec29cbe22be7a01d90bd0cfa080ffed6f1c321ea20ae3f10c6041f0e411e28ee2b98025945bee3548931deed1ae849b53c21b523ba74efef855cd73d languageName: node linkType: hard @@ -3781,16 +4183,19 @@ __metadata: languageName: node linkType: hard -"cosmiconfig-toml-loader@npm:1.0.0": - version: 1.0.0 - resolution: "cosmiconfig-toml-loader@npm:1.0.0" +"cosmiconfig@npm:8.0.0": + version: 8.0.0 + resolution: "cosmiconfig@npm:8.0.0" dependencies: - "@iarna/toml": ^2.2.5 - checksum: 00836a57c3c029a0d23f4eeeafc59a0be45cdf2707c5a6859020f545d50f939bfb01bc047fa41118faa92e69e25001f34d7687b05a97a469ed59fc870528b875 + import-fresh: ^3.2.1 + js-yaml: ^4.1.0 + parse-json: ^5.0.0 + path-type: ^4.0.0 + checksum: ff4cdf89ac1ae52e7520816622c21a9e04380d04b82d653f5139ec581aa4f7f29e096d46770bc76c4a63c225367e88a1dfa233ea791669a35101f5f9b972c7d1 languageName: node linkType: hard -"cosmiconfig@npm:7.0.1, cosmiconfig@npm:^7.0.0": +"cosmiconfig@npm:^7.0.0": version: 7.0.1 resolution: "cosmiconfig@npm:7.0.1" dependencies: @@ -3803,13 +4208,6 @@ __metadata: languageName: node linkType: hard -"create-require@npm:^1.1.0": - version: 1.1.1 - resolution: "create-require@npm:1.1.1" - checksum: a9a1503d4390d8b59ad86f4607de7870b39cad43d929813599a23714831e81c520bddf61bcdd1f8e30f05fd3a2b71ae8538e946eb2786dc65c2bbc520f692eff - languageName: node - linkType: hard - "cross-fetch@npm:3.1.5, cross-fetch@npm:^3.1.5": version: 3.1.5 resolution: "cross-fetch@npm:3.1.5" @@ -3937,6 +4335,13 @@ __metadata: languageName: node linkType: hard +"dataloader@npm:^2.2.2": + version: 2.2.2 + resolution: "dataloader@npm:2.2.2" + checksum: 4dabd247089c29f194e94d5434d504f99156c5c214a03463c20f3f17f40398d7e179edee69a27c16e315519ac8739042a810090087ae26449a0e685156a02c65 + languageName: node + linkType: hard + "date-time@npm:^3.1.0": version: 3.1.0 resolution: "date-time@npm:3.1.0" @@ -4095,13 +4500,6 @@ __metadata: languageName: node linkType: hard -"diff@npm:^4.0.1": - version: 4.0.2 - resolution: "diff@npm:4.0.2" - checksum: f2c09b0ce4e6b301c221addd83bf3f454c0bc00caa3dd837cf6c127d6edf7223aa2bbe3b688feea110b7f262adbfc845b757c44c8a9f8c0c5b15d8fa9ce9d20d - languageName: node - linkType: hard - "dir-glob@npm:^3.0.1": version: 3.0.1 resolution: "dir-glob@npm:3.0.1" @@ -4283,7 +4681,7 @@ __metadata: languageName: node linkType: hard -"es5-ext@npm:^0.10.35, es5-ext@npm:^0.10.46, es5-ext@npm:^0.10.50, es5-ext@npm:^0.10.53, es5-ext@npm:^0.10.59, es5-ext@npm:~0.10.14, es5-ext@npm:~0.10.2, es5-ext@npm:~0.10.46": +"es5-ext@npm:^0.10.35, es5-ext@npm:^0.10.46, es5-ext@npm:^0.10.50, es5-ext@npm:^0.10.53, es5-ext@npm:~0.10.14, es5-ext@npm:~0.10.2, es5-ext@npm:~0.10.46": version: 0.10.61 resolution: "es5-ext@npm:0.10.61" dependencies: @@ -4294,6 +4692,17 @@ __metadata: languageName: node linkType: hard +"es5-ext@npm:^0.10.61": + version: 0.10.62 + resolution: "es5-ext@npm:0.10.62" + dependencies: + es6-iterator: ^2.0.3 + es6-symbol: ^3.1.3 + next-tick: ^1.1.0 + checksum: 25f42f6068cfc6e393cf670bc5bba249132c5f5ec2dd0ed6e200e6274aca2fed8e9aec8a31c76031744c78ca283c57f0b41c7e737804c6328c7b8d3fbcba7983 + languageName: node + linkType: hard + "es6-error@npm:^4.1.1": version: 4.1.1 resolution: "es6-error@npm:4.1.1" @@ -4964,6 +5373,13 @@ __metadata: languageName: node linkType: hard +"fast-decode-uri-component@npm:^1.0.1": + version: 1.0.1 + resolution: "fast-decode-uri-component@npm:1.0.1" + checksum: 427a48fe0907e76f0e9a2c228e253b4d8a8ab21d130ee9e4bb8339c5ba4086235cf9576831f7b20955a752eae4b525a177ff9d5825dd8d416e7726939194fbee + languageName: node + linkType: hard + "fast-deep-equal@npm:^3.1.1": version: 3.1.3 resolution: "fast-deep-equal@npm:3.1.3" @@ -5018,6 +5434,15 @@ __metadata: languageName: node linkType: hard +"fast-querystring@npm:^1.1.1": + version: 1.1.1 + resolution: "fast-querystring@npm:1.1.1" + dependencies: + fast-decode-uri-component: ^1.0.1 + checksum: 86d2b75b9b299a552353532fb1a542f09730ee2a61e657d68710971d9a2afc9a3c5c7b7e106b6534f4cc506d2ff1c08ab0fda4ae614b4e7720798c9ac2a88e02 + languageName: node + linkType: hard + "fast-redact@npm:^3.0.0": version: 3.1.1 resolution: "fast-redact@npm:3.1.1" @@ -5032,6 +5457,15 @@ __metadata: languageName: node linkType: hard +"fast-url-parser@npm:^1.1.3": + version: 1.1.3 + resolution: "fast-url-parser@npm:1.1.3" + dependencies: + punycode: ^1.3.2 + checksum: 5043d0c4a8d775ff58504d56c096563c11b113e4cb8a2668c6f824a1cd4fb3812e2fdf76537eb24a7ce4ae7def6bd9747da630c617cf2a4b6ce0c42514e4f21c + languageName: node + linkType: hard + "fastq@npm:^1.6.0": version: 1.13.0 resolution: "fastq@npm:1.13.0" @@ -5392,14 +5826,14 @@ __metadata: languageName: node linkType: hard -"glob-promise@npm:^3.4.0": - version: 3.4.0 - resolution: "glob-promise@npm:3.4.0" +"glob-promise@npm:^4.2.2": + version: 4.2.2 + resolution: "glob-promise@npm:4.2.2" dependencies: - "@types/glob": "*" + "@types/glob": ^7.1.3 peerDependencies: - glob: "*" - checksum: 84a2c076e7581c9f8aa7a8a151ad5f9352c4118ba03c5673ecfcf540f4c53aa75f8d32fe493c2286d471dccd7a75932b9bfe97bf782564c1f4a50b9c7954e3b6 + glob: ^7.1.6 + checksum: c1a3d95f7c8393e4151d4899ec4e42bb2e8237160f840ad1eccbe9247407da8b6c13e28f463022e011708bc40862db87b9b77236d35afa3feb8aa86d518f2dfe languageName: node linkType: hard @@ -5523,24 +5957,28 @@ __metadata: languageName: node linkType: hard -"graphql-config@npm:^4.3.1": - version: 4.3.1 - resolution: "graphql-config@npm:4.3.1" +"graphql-config@npm:^4.5.0": + version: 4.5.0 + resolution: "graphql-config@npm:4.5.0" dependencies: - "@endemolshinegroup/cosmiconfig-typescript-loader": 3.0.2 "@graphql-tools/graphql-file-loader": ^7.3.7 "@graphql-tools/json-file-loader": ^7.3.7 "@graphql-tools/load": ^7.5.5 "@graphql-tools/merge": ^8.2.6 "@graphql-tools/url-loader": ^7.9.7 - "@graphql-tools/utils": ^8.6.5 - cosmiconfig: 7.0.1 - cosmiconfig-toml-loader: 1.0.0 - minimatch: 4.2.1 + "@graphql-tools/utils": ^9.0.0 + cosmiconfig: 8.0.0 + jiti: 1.17.1 + minimatch: 4.2.3 string-env-interpolation: 1.0.1 + tslib: ^2.4.0 peerDependencies: + cosmiconfig-toml-loader: ^1.0.0 graphql: ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 - checksum: 29c22fafaaf3889572d168b399f3f54194750269f832b241883f82011c40923b5eb9f8c24f9f1214b716f74726a061372af583b499a70299a71c5b3b53b42281 + peerDependenciesMeta: + cosmiconfig-toml-loader: + optional: true + checksum: 8ab1a3ce3534598ddac2df213b6af2eecd9ebcca2268d39cc3e87a6e55749483389a6df222e9e0acd638dd2479378a5c8e8d90f980e6a54e700c4c4ae3522123 languageName: node linkType: hard @@ -5558,16 +5996,15 @@ __metadata: languageName: node linkType: hard -"graphql-request@npm:^4.0.0": - version: 4.3.0 - resolution: "graphql-request@npm:4.3.0" +"graphql-request@npm:^6.0.0": + version: 6.0.0 + resolution: "graphql-request@npm:6.0.0" dependencies: + "@graphql-typed-document-node/core": ^3.2.0 cross-fetch: ^3.1.5 - extract-files: ^9.0.0 - form-data: ^3.0.0 peerDependencies: graphql: 14 - 16 - checksum: 3124afd01aee781cd5a2e9ac30063526b677a6754032566104fc36270b5f9be03f17a32e49f34c71ca968d533151550c37f7a0194d11c36ff59977bd73e2abc3 + checksum: e3acc6267c3d25d6dea7144823b3552c3533489269272e651e2a16c45575ccb357bb1dd3c78c1b6c379d4080377bc5694f589bd681d822c9647f6b1736239428 languageName: node linkType: hard @@ -5582,6 +6019,15 @@ __metadata: languageName: node linkType: hard +"graphql-ws@npm:5.12.1": + version: 5.12.1 + resolution: "graphql-ws@npm:5.12.1" + peerDependencies: + graphql: ">=0.11 <=16" + checksum: 88d587c431feba681957faecd96101bb3860e1a4765f34b8cae1c514e7f98754b5f31c6b3127775e4732f26883b0802fe426bf9f7031c16cd0b25a27ad90ec9c + languageName: node + linkType: hard + "graphql-ws@npm:^5.4.1": version: 5.9.0 resolution: "graphql-ws@npm:5.9.0" @@ -5768,6 +6214,16 @@ __metadata: languageName: node linkType: hard +"http-proxy-agent@npm:^6.0.0": + version: 6.0.1 + resolution: "http-proxy-agent@npm:6.0.1" + dependencies: + agent-base: ^7.0.1 + debug: ^4.3.4 + checksum: 59f446ece640828b8265163259b2cca7bf27a141959fb863746cf83a762645983495c0755c8cdd322281c0817185c8e019be9c9a29b3d5208dbdd0e929f47f35 + languageName: node + linkType: hard + "http-signature@npm:~1.2.0": version: 1.2.0 resolution: "http-signature@npm:1.2.0" @@ -5796,6 +6252,16 @@ __metadata: languageName: node linkType: hard +"https-proxy-agent@npm:^6.0.0": + version: 6.1.0 + resolution: "https-proxy-agent@npm:6.1.0" + dependencies: + agent-base: ^7.0.1 + debug: 4 + checksum: ede8d54fcca7611cc1afeca8d522a43a5ce138e7d8fb393d5c6714ae824e893a35bc8fd261ce5f02ca48ab948e424f0e51be00c1beea1d52ea80cc1257d39ea6 + languageName: node + linkType: hard + "humanize-ms@npm:^1.2.1": version: 1.2.1 resolution: "humanize-ms@npm:1.2.1" @@ -6050,7 +6516,7 @@ __metadata: languageName: node linkType: hard -"is-glob@npm:4.0.3, is-glob@npm:^4.0.1, is-glob@npm:~4.0.1": +"is-glob@npm:4.0.3, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1": version: 4.0.3 resolution: "is-glob@npm:4.0.3" dependencies: @@ -6178,17 +6644,7 @@ __metadata: languageName: node linkType: hard -"isomorphic-fetch@npm:^3.0.0": - version: 3.0.0 - resolution: "isomorphic-fetch@npm:3.0.0" - dependencies: - node-fetch: ^2.6.1 - whatwg-fetch: ^3.4.1 - checksum: e5ab79a56ce5af6ddd21265f59312ad9a4bc5a72cebc98b54797b42cb30441d5c5f8d17c5cd84a99e18101c8af6f90c081ecb8d12fd79e332be1778d58486d75 - languageName: node - linkType: hard - -"isomorphic-ws@npm:^5.0.0": +"isomorphic-ws@npm:5.0.0, isomorphic-ws@npm:^5.0.0": version: 5.0.0 resolution: "isomorphic-ws@npm:5.0.0" peerDependencies: @@ -6232,6 +6688,31 @@ __metadata: languageName: node linkType: hard +"jiti@npm:1.17.1": + version: 1.17.1 + resolution: "jiti@npm:1.17.1" + bin: + jiti: bin/jiti.js + checksum: 56c6d8488e7e9cc6ee66a0f0d5e18db6669cb12b2e93364f393442289a9bc75a8e8c796249f59015e01c3ebdf9478e2ca8b76c30e29072c678ee00d39de757c7 + languageName: node + linkType: hard + +"jiti@npm:^1.17.1": + version: 1.18.2 + resolution: "jiti@npm:1.18.2" + bin: + jiti: bin/jiti.js + checksum: 46c41cd82d01c6efdee3fc0ae9b3e86ed37457192d6366f19157d863d64961b07982ab04e9d5879576a1af99cc4d132b0b73b336094f86a5ce9fb1029ec2d29f + languageName: node + linkType: hard + +"jose@npm:^4.11.4": + version: 4.14.4 + resolution: "jose@npm:4.14.4" + checksum: 2d820a91a8fd97c05d8bc8eedc373b944a0cd7f5fe41063086da233d0473c73fb523912a9f026ea870782bd221f4a515f441a2d3af4de48c6f2c76dac5082377 + languageName: node + linkType: hard + "js-levenshtein@npm:^1.1.6": version: 1.1.6 resolution: "js-levenshtein@npm:1.1.6" @@ -6366,37 +6847,27 @@ __metadata: languageName: node linkType: hard -"json-schema-ref-parser@npm:^9.0.6": - version: 9.0.9 - resolution: "json-schema-ref-parser@npm:9.0.9" - dependencies: - "@apidevtools/json-schema-ref-parser": 9.0.9 - checksum: e05166a84c702f54f192edb2eb2e39236c3b03c30561777d63fd156ecd3aa3d2fffc0806a5703384bfba3c78800b1dc05f8da1ea25e6470b35a823210f7d48c4 - languageName: node - linkType: hard - -"json-schema-to-typescript@npm:10.1.5": - version: 10.1.5 - resolution: "json-schema-to-typescript@npm:10.1.5" +"json-schema-to-typescript@npm:13.0.1": + version: 13.0.1 + resolution: "json-schema-to-typescript@npm:13.0.1" dependencies: - "@types/json-schema": ^7.0.6 - "@types/lodash": ^4.14.168 - "@types/prettier": ^2.1.5 - cli-color: ^2.0.0 + "@bcherny/json-schema-ref-parser": 10.0.5-fork + "@types/json-schema": ^7.0.11 + "@types/lodash": ^4.14.182 + "@types/prettier": ^2.6.1 + cli-color: ^2.0.2 get-stdin: ^8.0.0 glob: ^7.1.6 - glob-promise: ^3.4.0 - is-glob: ^4.0.1 - json-schema-ref-parser: ^9.0.6 - json-stringify-safe: ^5.0.1 - lodash: ^4.17.20 - minimist: ^1.2.5 + glob-promise: ^4.2.2 + is-glob: ^4.0.3 + lodash: ^4.17.21 + minimist: ^1.2.6 mkdirp: ^1.0.4 mz: ^2.7.0 - prettier: ^2.2.0 + prettier: ^2.6.2 bin: json2ts: dist/src/cli.js - checksum: ab154931ca6b3804497e25b204a568edbd16b9a9a996881a512be9125f84ce19d7a09122a9f282113921edec8272e8ed67b780c15d86f72293a0d06b6ac35a4a + checksum: 8e95510628b159fdde37092a04bfb8cc1f35a7abd752965f724bd6a913d3ce9eb3c776884f67244ea3c7968ccabb8fac9bd17aed525644b2379a6c0fe899eecb languageName: node linkType: hard @@ -6490,7 +6961,7 @@ __metadata: languageName: node linkType: hard -"jsonwebtoken@npm:8.5.1, jsonwebtoken@npm:^8.5.1": +"jsonwebtoken@npm:8.5.1": version: 8.5.1 resolution: "jsonwebtoken@npm:8.5.1" dependencies: @@ -6629,13 +7100,6 @@ __metadata: languageName: node linkType: hard -"lodash.get@npm:^4": - version: 4.4.2 - resolution: "lodash.get@npm:4.4.2" - checksum: e403047ddb03181c9d0e92df9556570e2b67e0f0a930fcbbbd779370972368f5568e914f913e93f3b08f6d492abc71e14d4e9b7a18916c31fa04bd2306efe545 - languageName: node - linkType: hard - "lodash.includes@npm:^4.3.0": version: 4.3.0 resolution: "lodash.includes@npm:4.3.0" @@ -6858,13 +7322,6 @@ __metadata: languageName: node linkType: hard -"make-error@npm:^1, make-error@npm:^1.1.1": - version: 1.3.6 - resolution: "make-error@npm:1.3.6" - checksum: b86e5e0e25f7f777b77fabd8e2cbf15737972869d852a22b7e73c17623928fccb826d8e46b9951501d3f20e51ad74ba8c59ed584f610526a48f8ccf88aaec402 - languageName: node - linkType: hard - "make-fetch-happen@npm:^10.0.3": version: 10.1.7 resolution: "make-fetch-happen@npm:10.1.7" @@ -6988,6 +7445,18 @@ __metadata: languageName: node linkType: hard +"meros@npm:^1.2.1": + version: 1.2.1 + resolution: "meros@npm:1.2.1" + peerDependencies: + "@types/node": ">=13" + peerDependenciesMeta: + "@types/node": + optional: true + checksum: 2201c3f7c58ad2a5b5f7d6b1c644d79bde513e25cb64b51a8c41381ec74bc02cd3423425e34f60c96bf3991f1ec51d65dc8b8e3354cbb060cc9f8226b4666a5a + languageName: node + linkType: hard + "methods@npm:~1.1.2": version: 1.1.2 resolution: "methods@npm:1.1.2" @@ -6995,7 +7464,7 @@ __metadata: languageName: node linkType: hard -"micromatch@npm:^4.0.4": +"micromatch@npm:^4.0.4, micromatch@npm:^4.0.5": version: 4.0.5 resolution: "micromatch@npm:4.0.5" dependencies: @@ -7055,12 +7524,12 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:4.2.1": - version: 4.2.1 - resolution: "minimatch@npm:4.2.1" +"minimatch@npm:4.2.3": + version: 4.2.3 + resolution: "minimatch@npm:4.2.3" dependencies: brace-expansion: ^1.1.7 - checksum: 2b1514e3d0f29a549912f0db7ae7b82c5cab4a8f2dd0369f1c6451a325b3f12b2cf473c95873b6157bb8df183d6cf6db82ff03614b6adaaf1d7e055beccdfd01 + checksum: 3392388e3ef7de7ae9a3a48d48a27a323934452f4af81b925dfbe85ce2dc07da855e3dbcc69229888be4e5118f6c0b79847d30f3e7c0e0017b25e423c11c0409 languageName: node linkType: hard @@ -7346,6 +7815,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^3.2.1": + version: 3.2.1 + resolution: "node-addon-api@npm:3.2.1" + dependencies: + node-gyp: latest + checksum: 2369986bb0881ccd9ef6bacdf39550e07e089a9c8ede1cbc5fc7712d8e2faa4d50da0e487e333d4125f8c7a616c730131d1091676c9d499af1d74560756b4a18 + languageName: node + linkType: hard + "node-domexception@npm:1.0.0, node-domexception@npm:^1.0.0": version: 1.0.0 resolution: "node-domexception@npm:1.0.0" @@ -7378,6 +7856,17 @@ __metadata: languageName: node linkType: hard +"node-gyp-build@npm:^4.3.0": + version: 4.6.0 + resolution: "node-gyp-build@npm:4.6.0" + bin: + node-gyp-build: bin.js + node-gyp-build-optional: optional.js + node-gyp-build-test: build-test.js + checksum: 25d78c5ef1f8c24291f4a370c47ba52fcea14f39272041a90a7894cd50d766f7c8cb8fb06c0f42bf6f69b204b49d9be3c8fc344aac09714d5bdb95965499eb15 + languageName: node + linkType: hard + "node-gyp@npm:latest": version: 9.0.0 resolution: "node-gyp@npm:9.0.0" @@ -7973,7 +8462,7 @@ __metadata: languageName: node linkType: hard -"prettier@npm:2.8.8": +"prettier@npm:2.8.8, prettier@npm:^2.6.2": version: 2.8.8 resolution: "prettier@npm:2.8.8" bin: @@ -7982,15 +8471,6 @@ __metadata: languageName: node linkType: hard -"prettier@npm:^2.2.0": - version: 2.7.1 - resolution: "prettier@npm:2.7.1" - bin: - prettier: bin-prettier.js - checksum: 55a4409182260866ab31284d929b3cb961e5fdb91fe0d2e099dac92eaecec890f36e524b4c19e6ceae839c99c6d7195817579cdffc8e2c80da0cb794463a748b - languageName: node - linkType: hard - "pretty-format@npm:^27.0.2, pretty-format@npm:^27.5.1": version: 27.5.1 resolution: "pretty-format@npm:27.5.1" @@ -8083,7 +8563,7 @@ __metadata: languageName: node linkType: hard -"punycode@npm:^1.4.1": +"punycode@npm:^1.3.2, punycode@npm:^1.4.1": version: 1.4.1 resolution: "punycode@npm:1.4.1" checksum: fa6e698cb53db45e4628559e557ddaf554103d2a96a1d62892c8f4032cd3bc8871796cae9eabc1bc700e2b6677611521ce5bb1d9a27700086039965d0cf34518 @@ -8097,6 +8577,22 @@ __metadata: languageName: node linkType: hard +"pvtsutils@npm:^1.3.2": + version: 1.3.2 + resolution: "pvtsutils@npm:1.3.2" + dependencies: + tslib: ^2.4.0 + checksum: 9b8155611363e2f40276879f2454e60204b45be0cd0482f9373f369308a2e9c76d5d74cdf661a3f5aae8022d75ea159eb0ba38ee78fc782ee3051e4722db98d0 + languageName: node + linkType: hard + +"pvutils@npm:^1.1.3": + version: 1.1.3 + resolution: "pvutils@npm:1.1.3" + checksum: 2ee26a9e5176c348977d6ec00d8ee80bff62f51743b1c5fe8abeeb4c5d29d9959cdfe0ce146707a9e6801bce88190fed3002d720b072dc87d031c692820b44c9 + languageName: node + linkType: hard + "qs@npm:6.10.3": version: 6.10.3 resolution: "qs@npm:6.10.3" @@ -8257,13 +8753,6 @@ __metadata: languageName: node linkType: hard -"replaceall@npm:^0.1.6": - version: 0.1.6 - resolution: "replaceall@npm:0.1.6" - checksum: 2396fdc6f10b7ed7c8c0298455840473a028a81e29087bc02a37cc3ff1e85467534e0992a1b45bcdb3e99370db656d260bec7bd422819b708009f060ba8b4dbe - languageName: node - linkType: hard - "request-promise-core@npm:1.1.4": version: 1.1.4 resolution: "request-promise-core@npm:1.1.4" @@ -8708,6 +9197,13 @@ __metadata: languageName: node linkType: hard +"shell-quote@npm:^1.7.3": + version: 1.8.1 + resolution: "shell-quote@npm:1.8.1" + checksum: 5f01201f4ef504d4c6a9d0d283fa17075f6770bfbe4c5850b074974c68062f37929ca61700d95ad2ac8822e14e8c4b990ca0e6e9272e64befd74ce5e19f0736b + languageName: node + linkType: hard + "side-channel@npm:^1.0.4": version: 1.0.4 resolution: "side-channel@npm:1.0.4" @@ -8831,17 +9327,7 @@ __metadata: languageName: node linkType: hard -"source-map-support@npm:^0.5.17": - version: 0.5.21 - resolution: "source-map-support@npm:0.5.21" - dependencies: - buffer-from: ^1.0.0 - source-map: ^0.6.0 - checksum: 43e98d700d79af1d36f859bdb7318e601dfc918c7ba2e98456118ebc4c4872b327773e5a1df09b0524e9e5063bb18f0934538eace60cca2710d1fa687645d137 - languageName: node - linkType: hard - -"source-map@npm:^0.6.0, source-map@npm:^0.6.1, source-map@npm:~0.6.1": +"source-map@npm:^0.6.1, source-map@npm:~0.6.1": version: 0.6.1 resolution: "source-map@npm:0.6.1" checksum: 59ce8640cf3f3124f64ac289012c2b8bd377c238e316fb323ea22fbfe83da07d81e000071d7242cad7a23cd91c7de98e4df8830ec3f133cb6133a5f6e9f67bc2 @@ -9095,7 +9581,7 @@ __metadata: languageName: node linkType: hard -"sync-fetch@npm:0.4.1, sync-fetch@npm:^0.4.0": +"sync-fetch@npm:^0.4.0": version: 0.4.1 resolution: "sync-fetch@npm:0.4.1" dependencies: @@ -9308,27 +9794,6 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^9": - version: 9.1.1 - resolution: "ts-node@npm:9.1.1" - dependencies: - arg: ^4.1.0 - create-require: ^1.1.0 - diff: ^4.0.1 - make-error: ^1.1.1 - source-map-support: ^0.5.17 - yn: 3.1.1 - peerDependencies: - typescript: ">=2.7" - bin: - ts-node: dist/bin.js - ts-node-script: dist/bin-script.js - ts-node-transpile-only: dist/bin-transpile.js - ts-script: dist/bin-script-deprecated.js - checksum: 356e2647b8b1e6ab00380c0537fa569b63bd9b6f006cc40fd650f81fae1817bd8fecc075300036950d8f45c1d85b95be33cd1e48a1a424a7d86c3dbb42bf60e5 - languageName: node - linkType: hard - "tsc-multi@npm:1.0.0": version: 1.0.0 resolution: "tsc-multi@npm:1.0.0" @@ -9373,20 +9838,20 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2, tslib@npm:^2.0.3, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:~2.4.0": - version: 2.4.0 - resolution: "tslib@npm:2.4.0" - checksum: 8c4aa6a3c5a754bf76aefc38026134180c053b7bd2f81338cb5e5ebf96fefa0f417bff221592bf801077f5bf990562f6264fecbc42cd3309b33872cb6fc3b113 - languageName: node - linkType: hard - -"tslib@npm:^2.5.0": +"tslib@npm:^2.0.0, tslib@npm:^2.3.1, tslib@npm:^2.5.0, tslib@npm:~2.5.0": version: 2.5.0 resolution: "tslib@npm:2.5.0" checksum: ae3ed5f9ce29932d049908ebfdf21b3a003a85653a9a140d614da6b767a93ef94f460e52c3d787f0e4f383546981713f165037dc2274df212ea9f8a4541004e1 languageName: node linkType: hard +"tslib@npm:^2.0.3, tslib@npm:^2.1.0, tslib@npm:^2.4.0, tslib@npm:~2.4.0": + version: 2.4.0 + resolution: "tslib@npm:2.4.0" + checksum: 8c4aa6a3c5a754bf76aefc38026134180c053b7bd2f81338cb5e5ebf96fefa0f417bff221592bf801077f5bf990562f6264fecbc42cd3309b33872cb6fc3b113 + languageName: node + linkType: hard + "tsscmp@npm:1.0.6": version: 1.0.6 resolution: "tsscmp@npm:1.0.6" @@ -9662,6 +10127,13 @@ __metadata: languageName: node linkType: hard +"urlpattern-polyfill@npm:^8.0.0": + version: 8.0.2 + resolution: "urlpattern-polyfill@npm:8.0.2" + checksum: d2cc0905a613c77e330c426e8697ee522dd9640eda79ac51160a0f6350e103f09b8c327623880989f8ba7325e8d95267b745aa280fdcc2aead80b023e16bd09d + languageName: node + linkType: hard + "util-deprecate@npm:^1.0.1": version: 1.0.2 resolution: "util-deprecate@npm:1.0.2" @@ -9723,6 +10195,13 @@ __metadata: languageName: node linkType: hard +"value-or-promise@npm:^1.0.12": + version: 1.0.12 + resolution: "value-or-promise@npm:1.0.12" + checksum: f53a66c75b7447c90bbaf946a757ca09c094629cb80ba742f59c980ec3a69be0a385a0e75505dedb4e757862f1a994ca4beaf083a831f24d3ffb3d4bb18cd1e1 + languageName: node + linkType: hard + "vary@npm:^1, vary@npm:~1.1.2": version: 1.1.2 resolution: "vary@npm:1.1.2" @@ -9992,13 +10471,26 @@ __metadata: languageName: node linkType: hard -"web-streams-polyfill@npm:^3.0.3, web-streams-polyfill@npm:^3.2.0": +"web-streams-polyfill@npm:^3.0.3, web-streams-polyfill@npm:^3.2.0, web-streams-polyfill@npm:^3.2.1": version: 3.2.1 resolution: "web-streams-polyfill@npm:3.2.1" checksum: b119c78574b6d65935e35098c2afdcd752b84268e18746606af149e3c424e15621b6f1ff0b42b2676dc012fc4f0d313f964b41a4b5031e525faa03997457da02 languageName: node linkType: hard +"webcrypto-core@npm:^1.7.7": + version: 1.7.7 + resolution: "webcrypto-core@npm:1.7.7" + dependencies: + "@peculiar/asn1-schema": ^2.3.6 + "@peculiar/json-schema": ^1.1.12 + asn1js: ^3.0.1 + pvtsutils: ^1.3.2 + tslib: ^2.4.0 + checksum: 1dc5aedb250372dd95e175a671b990ae50e36974f99c4efc85d88e6528c1bc52dd964d44a41b68043c21fb26aabfe8aad4f05a1c39ca28d61de5ca7388413d52 + languageName: node + linkType: hard + "webidl-conversions@npm:^3.0.0": version: 3.0.1 resolution: "webidl-conversions@npm:3.0.1" @@ -10054,13 +10546,6 @@ __metadata: languageName: node linkType: hard -"whatwg-fetch@npm:^3.4.1": - version: 3.6.2 - resolution: "whatwg-fetch@npm:3.6.2" - checksum: ee976b7249e7791edb0d0a62cd806b29006ad7ec3a3d89145921ad8c00a3a67e4be8f3fb3ec6bc7b58498724fd568d11aeeeea1f7827e7e1e5eae6c8a275afed - languageName: node - linkType: hard - "whatwg-mimetype@npm:^2.2.0, whatwg-mimetype@npm:^2.3.0": version: 2.3.0 resolution: "whatwg-mimetype@npm:2.3.0" @@ -10228,6 +10713,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:8.13.0, ws@npm:^8.12.0": + version: 8.13.0 + resolution: "ws@npm:8.13.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 53e991bbf928faf5dc6efac9b8eb9ab6497c69feeb94f963d648b7a3530a720b19ec2e0ec037344257e05a4f35bd9ad04d9de6f289615ffb133282031b18c61c + languageName: node + linkType: hard + "ws@npm:^7.0.0": version: 7.5.8 resolution: "ws@npm:7.5.8" @@ -10431,13 +10931,6 @@ __metadata: languageName: node linkType: hard -"yn@npm:3.1.1": - version: 3.1.1 - resolution: "yn@npm:3.1.1" - checksum: 2c487b0e149e746ef48cda9f8bad10fc83693cd69d7f9dcd8be4214e985de33a29c9e24f3c0d6bcf2288427040a8947406ab27f7af67ee9456e6b84854f02dd6 - languageName: node - linkType: hard - "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"