Add option for each to run tasks in parallel ignoring dependency order

CHANGELOG.md

@@ -7,8 +7,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
-...
-
+### Added
+- Add an option `:parallel-unordered` to the `each` task that
+  processes projects in parallel as quickly as possible, ignoring dependency order.
 
 ## [1.10.1] - 2024-10-15
 

README.md

@@ -113,6 +113,8 @@ In addition to targeting options, `each` accepts:
 - `:report` show a detailed timing report after the tasks finish executing.
 - `:silent` suppress task output for successful projects.
 - `:output` path to a directory to save individual build output in.
+- `:parallel-unordered` like `:parallel`, but processes projects
+  as quickly as possible, ignoring dependency order.
 
 #### Incremental Builds
 

project.clj

@@ -1,4 +1,4 @@
-(defproject lein-monolith "1.10.1"
+(defproject lein-monolith "1.11.0-SNAPSHOT"
   :description "Leiningen plugin for managing subrojects within a monorepo."
   :url "https://github.com/amperity/lein-monolith"
   :license {:name "Apache License 2.0"

src/lein_monolith/task/each.clj

@@ -34,6 +34,7 @@
   (merge
     target/selection-opts
     {:parallel 1
+     :parallel-unordered 1
      :endure 0
      :report 0
      :silent 0
@@ -52,6 +53,8 @@
   (concat
     (when-let [threads (:parallel opts)]
       [:parallel threads])
+    (when-let [threads (:parallel-unordered opts)]
+      [:parallel-unordered threads])
     (when (:endure opts)
       [:endure])
     (when (:report opts)
@@ -301,7 +304,7 @@
             (print (str task-output))
             (flush)))
         ;; Print convenience resume tip for user.
-        (when-not (or (:parallel opts) (:endure opts))
+        (when-not (or (:parallel opts) (:parallel-unordered opts) (:endure opts))
           (let [resume-args (into
                               ["lein" "monolith" "each"]
                               (map u/shell-escape)
@@ -366,18 +369,47 @@
         (mapv (comp deref computations second) targets)))))
 
 
+(defn- run-parallel-unordered!
+  "Runs the tasks for targets in multiple worker threads, ignoring dependency
+  order. Returns a vector of result maps in the order the tasks finished
+  executing."
+  [ctx threads targets]
+  (let [thread-pool (executor/fixed-thread-executor threads)]
+    (resolve-tasks (:monolith ctx) (:task ctx))
+    (->
+      (reduce
+        (fn future-builder
+          [computations [_ target]]
+          (let [task-future (d/future-with thread-pool
+                              (lein/debug "Starting project" target)
+                              (run-task! ctx target))]
+            (assoc computations target task-future)))
+        {}
+        targets)
+      (as-> computations
+        (mapv (comp deref computations second) targets)))))
+
+
 (defn- run-all*
   "Run all tasks, using the `:parallel` option to determine whether to run them
   serially or concurrently."
   [ctx targets]
-  (if-let [threads (get-in ctx [:opts :parallel])]
-    (run-parallel! ctx (Integer/parseInt threads) targets)
-    (run-linear! ctx targets)))
+  (let [parallel (get-in ctx [:opts :parallel])
+        parallel-unordered (get-in ctx [:opts :parallel-unordered])]
+    (cond
+      parallel
+      (run-parallel! ctx (Integer/parseInt parallel) targets)
+
+      parallel-unordered
+      (run-parallel-unordered! ctx (Integer/parseInt parallel-unordered) targets)
+
+      :else
+      (run-linear! ctx targets))))
 
 
 (defn- run-all!
-  "Run all tasks, using the `:parallel`, `:silent`, and `:output` options to
-  determine behavior."
+  "Run all tasks, using the `:parallel`, `:parallel-unordered`, `:silent`, and
+  `:output` options to determine behavior."
   [ctx targets]
   (if (or (get-in ctx [:opts :silent])
           (get-in ctx [:opts :output]))

src/leiningen/monolith.clj

@@ -157,6 +157,13 @@
     :silent                 Don't print task output unless a subproject fails.
     :output <path>          Save each project's individual output in the given directory.
 
+    :parallel-unordered <threads> Like `:parallel`, but process projects as quickly
+                                  as possible, ignoring dependency order. This can yield
+                                  a speedup for some shapes of depdendency graphs,
+                                  but should only be used for tasks for which project order
+                                  doesn't matter. Otherwise, it will produce errors or incorrect results!
+                                  Mutually exclusive with `:parallel`.
+
   Targeting Options:
     :in <names>             Add the named projects directly to the targets.
     :upstream               Add the transitive dependencies of the current project to the targets.
@@ -183,8 +190,10 @@
   (let [[opts task] (u/parse-kw-args each/task-opts args)]
     (when (empty? task)
       (lein/abort "Cannot run each without a task argument!"))
-    (when (and (:start opts) (:parallel opts))
-      (lein/abort "The :parallel and :start options are not compatible!"))
+    (when (and (:start opts) (or (:parallel opts) (:parallel-unordered opts)))
+      (lein/abort "The :start option is not compatible with parallel processing!"))
+    (when (and (:parallel opts) (:parallel-unordered opts))
+      (lein/abort "The :parallel and :parallel-unordered options cannot be used at the same time!"))
     (each/run-tasks project opts task)))
 
 

test/example-tests.sh

@@ -40,6 +40,7 @@ test_monolith each :parallel 3 :report :endure pprint :group
 test_monolith each :refresh foo install
 test_monolith each :refresh foo install
 test_monolith each :parallel 3 :refresh bar install
+test_monolith each :parallel-unordered 3 test
 test_monolith changed
 test_monolith clear-fingerprints :upstream-of lib-b
 test_monolith mark-fresh :upstream-of lib-b foo bar