CPP-2161 parallel tasks

core/cli/bin/run

@@ -30,7 +30,7 @@ async function main() {
       const showHelp = require('../lib/help').default
       await showHelp(rootLogger, argv._)
     } else {
-      const { runTasks } = require('../lib')
+      const { runCommands } = require('../lib')
       if (argv['--'].length > 0) {
         // The `--` in a command such as `dotcom-tool-kit test:staged --`
         // delineates between hooks and file patterns. For example, when the
@@ -39,9 +39,9 @@ async function main() {
         // the command becomes something like `dotcom-tool-kit test:staged --
         // index.js`. When this command is executed it runs the configured task
         // where the file path arguments would then be extracted.
-        await runTasks(rootLogger, argv._, argv['--'])
+        await runCommands(rootLogger, argv._, argv['--'])
       } else {
-        await runTasks(rootLogger, argv._)
+        await runCommands(rootLogger, argv._)
       }
     }
   } catch (error) {

core/cli/package.json

@@ -25,6 +25,7 @@
     "@types/lodash": "^4.17.20",
     "@types/pluralize": "^0.0.33",
     "globby": "^10.0.2",
+    "type-fest": "^4.41.0",
     "winston": "^3.17.0",
     "zod": "^3.24.4"
   },

core/cli/src/index.ts

@@ -4,7 +4,7 @@ import util from 'util'
 import { formatPluginTree } from './messages'
 import { loadHookInstallations } from './install'
 
-export { runTasks } from './tasks'
+export { runCommands } from './tasks'
 export { shouldDisableNativeFetch } from './fetch'
 
 export async function listPlugins(logger: Logger): Promise<void> {

core/cli/src/plugin/entry-point.ts

@@ -6,6 +6,7 @@ import { Validated, invalid, valid } from '@dotcom-tool-kit/validated'
 import { __importDefault } from 'tslib'
 import type * as z from 'zod'
 import { indentReasons } from '../messages'
+import type { ReadonlyDeep } from 'type-fest'
 
 function guessIsZodSchema(schema: unknown): schema is z.ZodSchema {
   return typeof schema === 'object' && schema !== null && '_def' in schema
@@ -16,7 +17,7 @@ function guessIsZodSchema(schema: unknown): schema is z.ZodSchema {
  * are the correct shape right now so let's leave this as any and hope for the
  * best.
  **/
-async function requireEntrypoint(entryPoint: EntryPoint): Promise<Validated<any>> {
+async function requireEntrypoint(entryPoint: ReadonlyDeep<EntryPoint>): Promise<Validated<any>> {
   const resolvedPath = require.resolve(entryPoint.modulePath, { paths: [entryPoint.plugin.root] })
 
   if (!resolvedPath) {
@@ -63,7 +64,7 @@ export async function importSchemaEntryPoint(
 // the constructor from the type bound here so you can actually pass in a subclass
 export async function importEntryPoint<T extends { name: string } & Omit<typeof Base, 'new'>>(
   type: T,
-  entryPoint: EntryPoint
+  entryPoint: ReadonlyDeep<EntryPoint>
 ): Promise<Validated<{ baseClass: T; schema?: z.ZodSchema }>> {
   const pluginModuleResult = await requireEntrypoint(entryPoint)
   return pluginModuleResult.flatMap((pluginModule) => {

core/cli/src/tasks.ts

@@ -14,19 +14,20 @@ import { type TaskOptions, TaskSchemas } from '@dotcom-tool-kit/schemas'
 import { OptionsForTask } from '@dotcom-tool-kit/plugin'
 import type { RootOptions } from '@dotcom-tool-kit/plugin/src/root-schema'
 import pluralize from 'pluralize'
+import type { ReadonlyDeep } from 'type-fest'
 
-type ErrorSummary = {
+export type ErrorSummary = {
   task: string
   error: Error
 }
 
-const loadTasks = async (
+export async function loadTasks(
   logger: Logger,
   tasks: OptionsForTask[],
-  config: ValidConfig
-): Promise<Validated<Task[]>> => {
+  config: ReadonlyDeep<ValidConfig>
+): Promise<Validated<Task[]>> {
   const taskResults = await Promise.all(
-    tasks.map(async ({ task: taskId, options }) => {
+    tasks.map(async ({ task: taskId, options, plugin }) => {
       const entryPoint = config.tasks[taskId]
       const taskResult = await importEntryPoint(Task, entryPoint)
 
@@ -45,7 +46,8 @@ const loadTasks = async (
             logger,
             taskId,
             config.pluginOptions[entryPoint.plugin.id]?.options ?? {},
-            parsedOptions.data
+            parsedOptions.data,
+            plugin
           )
           return valid(task)
         } else {
@@ -58,7 +60,54 @@ const loadTasks = async (
   return reduceValidated(taskResults)
 }
 
-export async function runTasksFromConfig(
+export function handleTaskErrors(errors: ErrorSummary[], command: string) {
+  throw new AggregateError(
+    errors.map(({ task, error }) => {
+      error.name = `${styles.task(task)} → ${error.name}`
+      return error
+    }),
+    `${pluralize('error', errors.length, true)} running tasks for ${styles.command(command)}`
+  )
+}
+
+export async function runTasks(
+  logger: Logger,
+  config: ValidConfig,
+  tasks: Task[],
+  command: string,
+  files?: string[]
+) {
+  const errors: ErrorSummary[] = []
+
+  if (tasks.length === 0) {
+    logger.warn(`no task configured for ${command}: skipping assignment...`)
+  }
+
+  for (const task of tasks) {
+    try {
+      logger.info(styles.taskHeader(`running ${styles.task(task.id)} task`))
+      await task.run({ files, command, cwd: config.root, config })
+    } catch (error) {
+      // if there's an exit code, that's a request from the task to exit early
+      if (error instanceof ToolKitError && error.exitCode) {
+        throw error
+      }
+
+      // if not, we allow subsequent hook tasks to run on error
+      // TODO use validated for this
+      errors.push({
+        task: task.id,
+        error: error as Error
+      })
+    }
+  }
+
+  if (errors.length > 0) {
+    handleTaskErrors(errors, command)
+  }
+}
+
+export async function runCommandsFromConfig(
   logger: Logger,
   config: ValidConfig,
   commands: string[],
@@ -90,45 +139,12 @@ export async function runTasksFromConfig(
   Object.freeze(config)
 
   for (const { command, tasks } of commandTasks) {
-    const errors: ErrorSummary[] = []
-
-    if (tasks.length === 0) {
-      logger.warn(`no task configured for ${command}: skipping assignment...`)
-    }
-
-    for (const task of tasks) {
-      try {
-        logger.info(styles.taskHeader(`running ${styles.task(task.id)} task`))
-        await task.run({ files, command, cwd: config.root, config })
-      } catch (error) {
-        // if there's an exit code, that's a request from the task to exit early
-        if (error instanceof ToolKitError && error.exitCode) {
-          throw error
-        }
-
-        // if not, we allow subsequent hook tasks to run on error
-        // TODO use validated for this
-        errors.push({
-          task: task.id,
-          error: error as Error
-        })
-      }
-    }
-
-    if (errors.length > 0) {
-      throw new AggregateError(
-        errors.map(({ task, error }) => {
-          error.name = `${styles.task(task)} → ${error.name}`
-          return error
-        }),
-        `${pluralize('error', errors.length, true)} running tasks for ${styles.command(command)}`
-      )
-    }
+    await runTasks(logger, config, tasks, command, files)
   }
 }
 
-export async function runTasks(logger: Logger, commands: string[], files?: string[]): Promise<void> {
+export async function runCommands(logger: Logger, commands: string[], files?: string[]): Promise<void> {
   const config = await loadConfig(logger, { root: process.cwd() })
 
-  return runTasksFromConfig(logger, config, commands, files)
+  return runCommandsFromConfig(logger, config, commands, files)
 }

docs/extending-tool-kit.md

@@ -121,6 +121,26 @@ class Rollup extends Task {
 module.exports = Rollup
 ```
 
+> [!NOTE]
+> If you're writing a task that runs something as an ongoing process, such as a server or a watch-mode build, you should also implement the `Task#stop` method to allow it to be stopped on error [when running tasks in parallel](../plugins/parallel), e.g.:
+> ```js
+> class Rollup extends Task {
+>   async run() {
+>     // ...
+>     if(this.options.watch) {
+>       this.watcher = rollup.watch(options)
+>       // ...
+>     } else {
+>       // ...
+>     }
+>   }
+>
+>   stop() {
+>     this.watcher?.close()
+>   }
+> }
+> ```
+
 Then, in the plugin's `.toolkitrc.yml`, you can provide the default commands this task will run on. It's preferable to do this in the plugin `.toolkitrc.yml` instead of your top-level `.toolkitrc.yml` so your plugin is self-contained and can be more easily moved into its own repo or Tool Kit itself, if it's something that can be shared between multiple repos/teams.
 
 ```yml

jest.config.base.js

@@ -1,3 +1,5 @@
+const path = require('path')
+
 const tsJestConfig = {
   tsconfig: 'tsconfig.settings.json',
   isolatedModules: true
@@ -10,6 +12,11 @@ module.exports.config = {
   testMatch: ['**/__tests__/**/*.[jt]s?(x)', '**/*.+(spec|test).[jt]s?(x)'],
   testPathIgnorePatterns: ['/node_modules/', '<rootDir>/.+/lib/', '/test/files'],
   clearMocks: true,
+  snapshotSerializers: [
+    '@relmify/jest-serializer-strip-ansi/always',
+    path.resolve(__dirname, './jest/serializers/aggregate-error.js'),
+    path.resolve(__dirname, './jest/serializers/tool-kit-error.js')
+  ],
   transform: {
     '^.+\\.tsx?$': ['ts-jest', tsJestConfig]
   }

jest/serializers/aggregate-error.js

@@ -0,0 +1,15 @@
+module.exports = {
+  test(value) {
+    return value instanceof Error && value.name === 'AggregateError'
+  },
+
+  serialize(val, config, indentation, depth, refs, printer) {
+    return `AggregateError ${printer(val.message, config, indentation, depth, refs)} ${printer(
+      { errors: val.errors },
+      config,
+      indentation,
+      depth,
+      refs
+    )}`
+  }
+}

jest/serializers/tool-kit-error.js

@@ -0,0 +1,15 @@
+module.exports = {
+  test(value) {
+    return value instanceof Error && value.constructor.name === 'ToolKitError'
+  },
+
+  serialize(val, config, indentation, depth, refs, printer) {
+    return `${printer(val.name, config, indentation, depth, refs)} ${printer(
+      val.message,
+      config,
+      indentation,
+      depth,
+      refs
+    )} ${printer({ details: val.details }, config, indentation, depth, refs)}`
+  }
+}

lib/base/src/task.ts

@@ -3,6 +3,7 @@ import { Base } from './base'
 import { taskSymbol, typeSymbol } from './symbols'
 import type { Logger } from 'winston'
 import type { ValidConfig } from '@dotcom-tool-kit/config'
+import { Plugin } from '@dotcom-tool-kit/plugin'
 import type { Default } from './type-utils'
 import type { ReadonlyDeep } from 'type-fest'
 
@@ -33,21 +34,27 @@ export abstract class Task<
     logger: Logger,
     public id: string,
     public pluginOptions: z.output<Default<Options['plugin'], z.ZodObject<Record<string, never>>>>,
-    public options: z.output<Default<Options['task'], z.ZodObject<Record<string, never>>>>
+    public options: z.output<Default<Options['task'], z.ZodObject<Record<string, never>>>>,
+    public plugin: Plugin
   ) {
     super()
     this.logger = logger.child({ task: id })
   }
 
   abstract run(runContext: TaskRunContext): Promise<void>
+
+  // not abstract for default behaviour of doing nothing
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async stop(): Promise<void> {}
 }
 
 export type TaskConstructor = {
   new <O extends { plugin: z.ZodTypeAny; task: z.ZodTypeAny }>(
     logger: Logger,
     id: string,
     pluginOptions: Partial<z.infer<O['plugin']>>,
-    options: Partial<z.infer<O['task']>>
+    options: Partial<z.infer<O['task']>>,
+    plugin: Plugin
   ): Task<O>
 }