Clean up benchmarks folder and CI

.github/workflows/AdvancedPS.yml

@@ -0,0 +1,40 @@
+name: Integration Test (AdvancedPS)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    name: AdvancedPS.jl
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: "1"
+
+      - uses: julia-actions/cache@v2
+
+      - uses: actions/checkout@v6
+        with:
+          repository: TuringLang/AdvancedPS.jl
+          ref: main
+          path: downstream
+
+      - name: Run downstream tests with this Libtask
+        shell: julia --color=yes --project=downstream {0}
+        run: |
+          using Pkg
+          try
+            Pkg.develop(PackageSpec(path="."))
+            Pkg.update()
+            Pkg.test()
+          catch err
+            err isa Pkg.Resolve.ResolverError || rethrow()
+            @info "Not compatible with this release. No problem." exception=err
+            exit(0)
+          end

.github/workflows/Benchmark.yml

@@ -0,0 +1,20 @@
+name: Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  benchmarks:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: "1"
+      - uses: julia-actions/cache@v2
+      - name: Run benchmarks
+        run: julia --project=benchmarks -e 'using Pkg; Pkg.instantiate(); include("benchmarks/benchmark.jl")'

.github/workflows/CI.yml

@@ -0,0 +1,36 @@
+name: Libtask Testing
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 'min'
+          - '1.11'
+          - '1'
+          # TODO(mhauru) Reenable the below once there is a 'pre' version different from '1'.
+          # - 'pre'
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+
+      - uses: julia-actions/cache@v2
+
+      - uses: julia-actions/julia-buildpkg@v1
+
+      - uses: julia-actions/julia-runtest@v1

.github/workflows/Turing.yml

@@ -0,0 +1,57 @@
+name: Turing integration tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+
+# Cancel existing tests on the same PR if a new commit is added to a pull request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  turing:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        version:
+          - 'min'
+          - '1.11'
+          - '1'
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+
+      - uses: julia-actions/cache@v2
+
+      - name: Instantiate
+        id: instantiate
+        working-directory: test/integration/turing
+        continue-on-error: true
+        run: julia --project=. --color=yes -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Report incompatibility
+        if: steps.instantiate.outcome == 'failure'
+        run: |
+          echo "::warning::Turing is incompatible with the current version of Libtask. Skipping integration tests."
+          echo "This likely means Turing has an compat bound on Libtask that excludes the currently checked out version."
+
+      - name: Run Turing integration tests
+        if: steps.instantiate.outcome == 'success'
+        working-directory: test/integration/turing
+        run: julia --project=. --color=yes main.jl

benchmarks/Project.toml

@@ -0,0 +1,7 @@
+[deps]
+Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
+Libtask = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[sources]
+Libtask = {path = "../"}

benchmarks/benchmark.jl

@@ -0,0 +1,83 @@
+using Libtask
+using LinearAlgebra
+using Chairmarks: @b
+
+# Each benchmark function takes a `maybe_produce` as its last argument, defaulting to
+# `identity` (a no-op). The benchmark driver calls the function twice: once with the
+# default (measuring raw performance) and once via a TapedTask that passes `produce` as the
+# last argument (measuring the overhead of the produce/consume machinery).
+function benchmark(f, x...)
+    printstyled(string(f), "\n"; bold=true)
+
+    # Baseline: call f directly with maybe_produce=identity (the default).
+    baseline = @b $f($(x)...)
+
+    # TapedTask: pass `produce` so every `maybe_produce(...)` call yields a value.
+    function f_via_task(f, x)
+        tt = TapedTask(nothing, f, x..., produce)
+        n = 0
+        while consume(tt) !== nothing
+            n += 1
+        end
+        return n
+    end
+    n_produces = f_via_task(f, x)
+    taped = @b $f_via_task($f, $x)
+
+    noun = n_produces == 1 ? "produce" : "produces"
+    label = "taped ($n_produces $noun)"
+    print(rpad("baseline", length(label)), "  ")
+    display(baseline)
+    print(label, "  ")
+    display(taped)
+    ratio = round(taped.time / baseline.time; digits=1)
+    println(rpad("ratio", length(label)), "  ", "$(ratio)x")
+    println()
+    return nothing
+end
+
+function rosenbrock(x, maybe_produce=identity)
+    i = x[2:end]
+    j = x[1:(end - 1)]
+    ret = sum((1 .- j) .^ 2 + 100 * (i - j .^ 2) .^ 2)
+    maybe_produce(ret)
+    return ret
+end
+benchmark(rosenbrock, rand(100_000))
+
+function ackley(x::AbstractVector, maybe_produce=identity)
+    a, b, c = 20.0, -0.2, 2.0 * π
+    len_recip = inv(length(x))
+    sum_sqrs = zero(eltype(x))
+    sum_cos = sum_sqrs
+    for i in x
+        sum_cos += cos(c * i)
+        sum_sqrs += i^2
+        maybe_produce(sum_sqrs)
+    end
+    return -a * exp(b * sqrt(len_recip * sum_sqrs)) - exp(len_recip * sum_cos) +
+           a +
+           MathConstants.e
+end
+benchmark(ackley, rand(100_000))
+
+function matrix_test(x, maybe_produce=identity)
+    n = 100
+    a = reshape(x[1:(n^2)], n, n)
+    b = reshape(x[(n^2 + 1):(2n^2)], n, n)
+    ret = log.((a * b) + a - b)
+    maybe_produce(ret)
+    return ret
+end
+benchmark(matrix_test, collect(1.0:(2 * 100^2 + 100)))
+
+relu(x) = log.(1.0 .+ exp.(x))
+sigmoid(n) = 1.0 / (1.0 + exp(-n))
+function neural_net(w1, w2, w3, x1, maybe_produce=identity)
+    x2 = relu(w1 * x1)
+    x3 = relu(w2 * x2)
+    ret = sigmoid(LinearAlgebra.dot(w3, x3))
+    maybe_produce(ret)
+    return ret
+end
+benchmark(neural_net, randn(10, 10), randn(10, 10), randn(10), rand(10))