From f0383bf58087f9cc14b2d70d9f93b64e0607eed4 Mon Sep 17 00:00:00 2001 From: Brandon Vincent Date: Wed, 13 Aug 2025 15:52:41 -0700 Subject: [PATCH] Add option to runs tasks in parallel ignoring dependency order --- CHANGELOG.md | 5 ++-- README.md | 2 ++ project.clj | 2 +- src/lein_monolith/task/each.clj | 44 ++++++++++++++++++++++++++++----- src/leiningen/monolith.clj | 13 ++++++++-- test/example-tests.sh | 1 + 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3d16e7..eacc037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,9 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] -... - +### Added +- Add an option `:parallel-unordered` to the `each` task that + processes projects in parallel as quickly as possible, ignoring dependency order. ## [1.10.1] - 2024-10-15 diff --git a/README.md b/README.md index 41fa859..4098876 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ In addition to targeting options, `each` accepts: - `:report` show a detailed timing report after the tasks finish executing. - `:silent` suppress task output for successful projects. - `:output` path to a directory to save individual build output in. +- `:parallel-unordered` like `:parallel`, but processes projects + as quickly as possible, ignoring dependency order. #### Incremental Builds diff --git a/project.clj b/project.clj index ec80f04..770a886 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject lein-monolith "1.10.1" +(defproject lein-monolith "1.11.0-SNAPSHOT" :description "Leiningen plugin for managing subrojects within a monorepo." :url "https://github.com/amperity/lein-monolith" :license {:name "Apache License 2.0" diff --git a/src/lein_monolith/task/each.clj b/src/lein_monolith/task/each.clj index 4359338..b9795f2 100644 --- a/src/lein_monolith/task/each.clj +++ b/src/lein_monolith/task/each.clj @@ -34,6 +34,7 @@ (merge target/selection-opts {:parallel 1 + :parallel-unordered 1 :endure 0 :report 0 :silent 0 @@ -52,6 +53,8 @@ (concat (when-let [threads (:parallel opts)] [:parallel threads]) + (when-let [threads (:parallel-unordered opts)] + [:parallel-unordered threads]) (when (:endure opts) [:endure]) (when (:report opts) @@ -301,7 +304,7 @@ (print (str task-output)) (flush))) ;; Print convenience resume tip for user. - (when-not (or (:parallel opts) (:endure opts)) + (when-not (or (:parallel opts) (:parallel-unordered opts) (:endure opts)) (let [resume-args (into ["lein" "monolith" "each"] (map u/shell-escape) @@ -366,18 +369,47 @@ (mapv (comp deref computations second) targets))))) +(defn- run-parallel-unordered! + "Runs the tasks for targets in multiple worker threads, ignoring dependency + order. Returns a vector of result maps in the order the tasks finished + executing." + [ctx threads targets] + (let [thread-pool (executor/fixed-thread-executor threads)] + (resolve-tasks (:monolith ctx) (:task ctx)) + (-> + (reduce + (fn future-builder + [computations [_ target]] + (let [task-future (d/future-with thread-pool + (lein/debug "Starting project" target) + (run-task! ctx target))] + (assoc computations target task-future))) + {} + targets) + (as-> computations + (mapv (comp deref computations second) targets))))) + + (defn- run-all* "Run all tasks, using the `:parallel` option to determine whether to run them serially or concurrently." [ctx targets] - (if-let [threads (get-in ctx [:opts :parallel])] - (run-parallel! ctx (Integer/parseInt threads) targets) - (run-linear! ctx targets))) + (let [parallel (get-in ctx [:opts :parallel]) + parallel-unordered (get-in ctx [:opts :parallel-unordered])] + (cond + parallel + (run-parallel! ctx (Integer/parseInt parallel) targets) + + parallel-unordered + (run-parallel-unordered! ctx (Integer/parseInt parallel-unordered) targets) + + :else + (run-linear! ctx targets)))) (defn- run-all! - "Run all tasks, using the `:parallel`, `:silent`, and `:output` options to - determine behavior." + "Run all tasks, using the `:parallel`, `:parallel-unordered`, `:silent`, and + `:output` options to determine behavior." [ctx targets] (if (or (get-in ctx [:opts :silent]) (get-in ctx [:opts :output])) diff --git a/src/leiningen/monolith.clj b/src/leiningen/monolith.clj index f232068..3606eff 100644 --- a/src/leiningen/monolith.clj +++ b/src/leiningen/monolith.clj @@ -157,6 +157,13 @@ :silent Don't print task output unless a subproject fails. :output Save each project's individual output in the given directory. + :parallel-unordered Like `:parallel`, but process projects as quickly + as possible, ignoring dependency order. This can yield + a speedup for some shapes of depdendency graphs, + but should only be used for tasks for which project order + doesn't matter. Otherwise, it will produce errors or incorrect results! + Mutually exclusive with `:parallel`. + Targeting Options: :in Add the named projects directly to the targets. :upstream Add the transitive dependencies of the current project to the targets. @@ -183,8 +190,10 @@ (let [[opts task] (u/parse-kw-args each/task-opts args)] (when (empty? task) (lein/abort "Cannot run each without a task argument!")) - (when (and (:start opts) (:parallel opts)) - (lein/abort "The :parallel and :start options are not compatible!")) + (when (and (:start opts) (or (:parallel opts) (:parallel-unordered opts))) + (lein/abort "The :start option is not compatible with parallel processing!")) + (when (and (:parallel opts) (:parallel-unordered opts)) + (lein/abort "The :parallel and :parallel-unordered options cannot be used at the same time!")) (each/run-tasks project opts task))) diff --git a/test/example-tests.sh b/test/example-tests.sh index f67f55f..7555e12 100755 --- a/test/example-tests.sh +++ b/test/example-tests.sh @@ -40,6 +40,7 @@ test_monolith each :parallel 3 :report :endure pprint :group test_monolith each :refresh foo install test_monolith each :refresh foo install test_monolith each :parallel 3 :refresh bar install +test_monolith each :parallel-unordered 3 test test_monolith changed test_monolith clear-fingerprints :upstream-of lib-b test_monolith mark-fresh :upstream-of lib-b foo bar