From e4fca3a30e2812c9819f6a031a113c97988f511c Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 01:39:15 +0000 Subject: [PATCH 01/17] WIP --- .github/actions/test_ruby_gem_uploads/Gemfile | 5 +++ .../actions/test_ruby_gem_uploads/Rakefile | 5 +++ .../actions/test_ruby_gem_uploads/action.yaml | 34 ++++++++++++++++++- .../test_ruby_gem_uploads/spec/spec_helper.rb | 5 +++ .github/workflows/release_ruby_gem.yml | 1 + .github/workflows/smoke_test.yml | 2 ++ .github/workflows/smoke_test_main.yml | 2 ++ .../lib/trunk_spec_helper.rb | 9 ++++- 8 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 .github/actions/test_ruby_gem_uploads/Rakefile diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index b5732cf4..0f2df80d 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -3,3 +3,8 @@ source 'https://rubygems.org' gem 'rspec' +gem 'knapsack_pro' +gem 'rake' +gem 'rake-compiler' +# donotland +# gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/Rakefile b/.github/actions/test_ruby_gem_uploads/Rakefile new file mode 100644 index 00000000..8aa8ab52 --- /dev/null +++ b/.github/actions/test_ruby_gem_uploads/Rakefile @@ -0,0 +1,5 @@ +require 'rake/testtask' +require 'rake/extensiontask' +require 'knapsack_pro' +require 'rspec/core/rake_task' +KnapsackPro.load_tasks if defined?(KnapsackPro) \ No newline at end of file diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index cde857a4..3e5bf163 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -24,6 +24,9 @@ inputs: platform: description: The platform that the gem was built for required: true + knapsack-pro-test-suite-token-rspec: + description: Optional Knapsack Pro test suite token for RSpec + required: true runs: using: composite @@ -64,7 +67,7 @@ runs: mv rspec_trunk_flaky_tests/rspec_trunk_flaky_tests*/lib rspec_trunk_flaky_tests/lib bundle install bundle lock --remove-platform ruby || true - echo "gem 'rspec_trunk_flaky_tests', '${GEM_VERSION}', :path => './rspec_trunk_flaky_tests'" >> Gemfile + printf "\ngem 'rspec_trunk_flaky_tests', '%s', :path => './rspec_trunk_flaky_tests'\n" "${GEM_VERSION}" >> Gemfile bundle install --gemfile Gemfile --local - name: Run regular tests @@ -117,3 +120,32 @@ runs: TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} TRUNK_API_TOKEN: ${{ inputs.trunk-token }} TRUNK_VARIANT: smoke-test-variant + + - name: Run knapsack_pro queue and verify a single local bundle + shell: bash + if: inputs.knapsack-pro-test-suite-token-rspec != '' + working-directory: ${{ github.action_path }} + env: + KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} + KNAPSACK_PRO_CI_NODE_TOTAL: 3 + KNAPSACK_PRO_CI_NODE_INDEX: 1 + TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + TRUNK_LOCAL_UPLOAD_DIR: bundles + TRUNK_LOCAL_UPLOAD_DIR_ALLOW_MULTIPLE: true # donotland + run: | + set -euo pipefail + rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" + mkdir -p "${TRUNK_LOCAL_UPLOAD_DIR}" + set +e + bundle exec rake "knapsack_pro:rspec" + set -e + ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" + mapfile -t bundle_files < <(find "${TRUNK_LOCAL_UPLOAD_DIR}" -maxdepth 1 -type f -name '*.bin') + if [ "${#bundle_files[@]}" -ne 1 ]; then + echo "Expected exactly one bundle in ${TRUNK_LOCAL_UPLOAD_DIR}, found ${#bundle_files[@]}" + ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" + exit 1 + fi + echo "Knapsack run correctly generated exactly one bundle at ${bundle_files[0]}" diff --git a/.github/actions/test_ruby_gem_uploads/spec/spec_helper.rb b/.github/actions/test_ruby_gem_uploads/spec/spec_helper.rb index c4abdd50..87c9e5d4 100644 --- a/.github/actions/test_ruby_gem_uploads/spec/spec_helper.rb +++ b/.github/actions/test_ruby_gem_uploads/spec/spec_helper.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true require 'trunk_spec_helper' +if ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC'] + # Enable Knapsack queueing when CI provides a suite token + require 'knapsack_pro' + KnapsackPro::Adapters::RSpecAdapter.bind +end # This file was generated by the `rspec --init` command. Conventionally, all # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. # The generated `.rspec` file contains `--require spec_helper` which will cause diff --git a/.github/workflows/release_ruby_gem.yml b/.github/workflows/release_ruby_gem.yml index 547b1461..663f8d68 100644 --- a/.github/workflows/release_ruby_gem.yml +++ b/.github/workflows/release_ruby_gem.yml @@ -126,6 +126,7 @@ jobs: trunk-token: ${{ secrets.TRUNK_STAGING_ORG_API_TOKEN }} platform: ${{ matrix.platform.name }} artifact-pattern: cross-gem-${{ matrix.platform.name }} + knapsack-pro-test-suite-token-rspec: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }} publish_ruby_gem: runs-on: ubuntu-latest diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index e43b2f14..987f1ef1 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -127,6 +127,7 @@ jobs: trunk-org-slug: trunk-staging-org platform: x86_64-linux artifact-pattern: "" + knapsack-pro-test-suite-token-rspec: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }} test_ruby_gem_production: name: Test Ruby gem on production @@ -151,6 +152,7 @@ jobs: trunk-org-slug: trunk platform: x86_64-linux artifact-pattern: "" + knapsack-pro-test-suite-token-rspec: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }} production-slack-workflow-status: if: always() && github.ref == 'refs/heads/main' && (needs.build_cli.outputs.production-success != 'true' || needs.test_ruby_gem_production.result != 'success') diff --git a/.github/workflows/smoke_test_main.yml b/.github/workflows/smoke_test_main.yml index 01e3ac23..0ff067bf 100644 --- a/.github/workflows/smoke_test_main.yml +++ b/.github/workflows/smoke_test_main.yml @@ -228,6 +228,7 @@ jobs: trunk-org-slug: trunk-staging-org platform: x86_64-linux artifact-pattern: "" + knapsack-pro-test-suite-token-rspec: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }} test_ruby_gem_production: name: Test Ruby gem on production @@ -252,6 +253,7 @@ jobs: trunk-org-slug: trunk platform: x86_64-linux artifact-pattern: "" + knapsack-pro-test-suite-token-rspec: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC }} production-slack-workflow-status: if: always() && github.ref == 'refs/heads/main' && (needs.test_cli.result != 'success' || needs.test_ruby_gem_production.result != 'success') diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 5e12c49a..ec6aa360 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -65,7 +65,14 @@ def knapsack_detector_mode? def knapsack_detector_command? command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip - command_line.include?('knapsack_pro:rspec_test_example_detector') + # donotland + # command_line.include?('knapsack_pro:rspec_test_example_detector') + if command_line.include?('knapsack_pro:rspec_test_example_detector') + puts "Detected knapsack example detector command: #{command_line}".yellow + true + else + false + end end def trunk_disabled From 312769640b23f36bd08592f7e5010f186e8f21e6 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 16:14:49 +0000 Subject: [PATCH 02/17] try with logger --- .github/actions/test_ruby_gem_uploads/Gemfile | 1 + .../actions/test_ruby_gem_uploads/action.yaml | 96 +++++++++---------- .github/workflows/smoke_test_main.yml | 4 + 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index 0f2df80d..c322eaab 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -6,5 +6,6 @@ gem 'rspec' gem 'knapsack_pro' gem 'rake' gem 'rake-compiler' +gem 'logger' # donotland # gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 3e5bf163..99fb7164 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -70,56 +70,56 @@ runs: printf "\ngem 'rspec_trunk_flaky_tests', '%s', :path => './rspec_trunk_flaky_tests'\n" "${GEM_VERSION}" >> Gemfile bundle install --gemfile Gemfile --local - - name: Run regular tests - id: run-regular-tests - shell: bash - working-directory: ${{ github.action_path }} - run: | - bundle exec rspec spec/test_spec.rb --format documentation - env: - TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + # - name: Run regular tests + # id: run-regular-tests + # shell: bash + # working-directory: ${{ github.action_path }} + # run: | + # bundle exec rspec spec/test_spec.rb --format documentation + # env: + # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - - name: Run variant quarantine test without variant (should fail - not quarantined) - id: run-variant-test-no-variant - shell: bash - working-directory: ${{ github.action_path }} - run: | - set +e - bundle exec rspec spec/variant_quarantine_spec.rb --format documentation - EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - echo "ERROR: Variant quarantine test without variant should have failed (test should not be quarantined)" - exit 1 - else - echo "SUCCESS: Variant quarantine test without variant failed as expected (test is not quarantined)" - fi - env: - TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - TRUNK_VARIANT: "" + # - name: Run variant quarantine test without variant (should fail - not quarantined) + # id: run-variant-test-no-variant + # shell: bash + # working-directory: ${{ github.action_path }} + # run: | + # set +e + # bundle exec rspec spec/variant_quarantine_spec.rb --format documentation + # EXIT_CODE=$? + # if [ $EXIT_CODE -eq 0 ]; then + # echo "ERROR: Variant quarantine test without variant should have failed (test should not be quarantined)" + # exit 1 + # else + # echo "SUCCESS: Variant quarantine test without variant failed as expected (test is not quarantined)" + # fi + # env: + # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + # TRUNK_VARIANT: "" - - name: Run variant quarantine test with variant (should pass - quarantined) - id: run-variant-test-with-variant - shell: bash - working-directory: ${{ github.action_path }} - run: | - bundle exec rspec spec/variant_quarantine_spec.rb --format documentation - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo "ERROR: Variant quarantine test with variant should have passed (test should be quarantined)" - echo "Note: The variant_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" - exit 1 - else - echo "SUCCESS: Variant quarantine test with variant passed (test was quarantined)" - fi - env: - TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - TRUNK_VARIANT: smoke-test-variant + # - name: Run variant quarantine test with variant (should pass - quarantined) + # id: run-variant-test-with-variant + # shell: bash + # working-directory: ${{ github.action_path }} + # run: | + # bundle exec rspec spec/variant_quarantine_spec.rb --format documentation + # EXIT_CODE=$? + # if [ $EXIT_CODE -ne 0 ]; then + # echo "ERROR: Variant quarantine test with variant should have passed (test should be quarantined)" + # echo "Note: The variant_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" + # exit 1 + # else + # echo "SUCCESS: Variant quarantine test with variant passed (test was quarantined)" + # fi + # env: + # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + # TRUNK_VARIANT: smoke-test-variant - name: Run knapsack_pro queue and verify a single local bundle shell: bash diff --git a/.github/workflows/smoke_test_main.yml b/.github/workflows/smoke_test_main.yml index 0ff067bf..addf3d0d 100644 --- a/.github/workflows/smoke_test_main.yml +++ b/.github/workflows/smoke_test_main.yml @@ -3,6 +3,10 @@ permissions: actions: read on: workflow_dispatch: {} + # donotland + push: + branches: + - tyler/knapsack-smoke-test schedule: - cron: 0 0/2 * * * repository_dispatch: From 41b66ef99b8eb3dd9df1b105e3822ad164f4dc3c Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 16:38:56 +0000 Subject: [PATCH 03/17] more print --- rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index ec6aa360..e97c6bdd 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -71,6 +71,7 @@ def knapsack_detector_command? puts "Detected knapsack example detector command: #{command_line}".yellow true else + puts "Detected knapsack example detector other command: #{command_line}".red false end end From f664cd2d546418fff21646cc7934307e624e3e78 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 16:50:06 +0000 Subject: [PATCH 04/17] one more --- rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index e97c6bdd..0bc21e15 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -68,10 +68,10 @@ def knapsack_detector_command? # donotland # command_line.include?('knapsack_pro:rspec_test_example_detector') if command_line.include?('knapsack_pro:rspec_test_example_detector') - puts "Detected knapsack example detector command: #{command_line}".yellow + puts "Detected knapsack example detector command: #{command_line}\n\n".yellow true else - puts "Detected knapsack example detector other command: #{command_line}".red + puts "Detected knapsack other command: #{command_line}\n\n".yellow false end end From c434f2c7255b93edd736d1676ad88ae7e4231e83 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 18:59:46 +0000 Subject: [PATCH 05/17] queue mode --- .github/actions/test_ruby_gem_uploads/action.yaml | 3 ++- rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 99fb7164..d862736f 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -129,6 +129,7 @@ runs: KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} KNAPSACK_PRO_CI_NODE_TOTAL: 3 KNAPSACK_PRO_CI_NODE_INDEX: 1 + KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES: true TRUNK_API_TOKEN: ${{ inputs.trunk-token }} TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} @@ -139,7 +140,7 @@ runs: rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" mkdir -p "${TRUNK_LOCAL_UPLOAD_DIR}" set +e - bundle exec rake "knapsack_pro:rspec" + bundle exec rake "knapsack_pro:queue:rspec" set -e ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" mapfile -t bundle_files < <(find "${TRUNK_LOCAL_UPLOAD_DIR}" -maxdepth 1 -type f -name '*.bin') diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 0bc21e15..2f623516 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -68,6 +68,7 @@ def knapsack_detector_command? # donotland # command_line.include?('knapsack_pro:rspec_test_example_detector') if command_line.include?('knapsack_pro:rspec_test_example_detector') + # knapsack_pro:queue:rspec:initialize puts "Detected knapsack example detector command: #{command_line}\n\n".yellow true else From 78d2f49c144e97069fe549c86cba2dba587b1dbc Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 19:32:39 +0000 Subject: [PATCH 06/17] reset me! --- .../spec/variant_quarantine_spec_alt.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb diff --git a/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb b/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb new file mode 100644 index 00000000..b181a817 --- /dev/null +++ b/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +describe 'variant_quarantine_test_alt' do + it 'should be quarantined when run with variant' do + # This test should fail when run without a variant (not quarantined) + # and be quarantined (not fail) when run with a variant + expect(2 + 2).to eq(5) + end +end From 9f17534c64d5eb726c83bb79b15986217dcee295 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 19:49:19 +0000 Subject: [PATCH 07/17] refinement --- .github/actions/test_ruby_gem_uploads/Gemfile | 2 +- .../actions/test_ruby_gem_uploads/README.md | 27 +++++++++++++++++++ .../actions/test_ruby_gem_uploads/action.yaml | 5 +++- .../spec/variant_quarantine_spec_alt.rb | 9 ------- 4 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 .github/actions/test_ruby_gem_uploads/README.md delete mode 100644 .github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index c322eaab..fd8cbc30 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -8,4 +8,4 @@ gem 'rake' gem 'rake-compiler' gem 'logger' # donotland -# gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' +gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/README.md b/.github/actions/test_ruby_gem_uploads/README.md new file mode 100644 index 00000000..e5dfbf92 --- /dev/null +++ b/.github/actions/test_ruby_gem_uploads/README.md @@ -0,0 +1,27 @@ +# Running Ruby Tests Locally + +If you want to locally run the ruby smoke tests that are run in CI, run: + +From `cd rspec-trunk-flaky-tests`: + +1. `cd rspec-trunk-flaky-tests` +2. `bundle install` +3. `bundle exec rake build` +4. `cd ../.github/actions/test_ruby_gem_uploads` +5. Add `gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests'` to [Gemfile](./Gemfile) +6. Run `bundle exec rspec spec/variant_quarantine_spec.rb --format documentation` + +After `cd .github/actions/test_ruby_gem_uploads` + +See more in the [rspec README.md](../../../rspec-trunk-flaky-tests/README.md). + +## Knapsack Pro + +Repeat steps 1-5 above, and run: + +1. `KNAPSACK_PRO_CI_NODE_BUILD_ID=$(openssl rand -base64 32)` +2. `KNAPSACK_PRO_TEST_DIR=spec KNAPSACK_PRO_TEST_FILE_PATTERN="**/*.rb" KNAPSACK_PRO_PROJECT_DIR=. KNAPSACK_PRO_REPOSITORY_ADAPTER=git KNAPSACK_PRO_LOG_LEVEL=debug KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC="" KNAPSACK_PRO_CI_NODE_TOTAL=1 KNAPSACK_PRO_CI_NODE_INDEX=0 KNAPSACK_PRO_FIXED_QUEUE_SPLIT=false bundle exec rake "knapsack_pro:queue:rspec"` + +### Reference + +- [More information about Queue Mode](https://docs.knapsackpro.com/ruby/queue-mode/) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index d862736f..fb4ea4b4 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -129,7 +129,8 @@ runs: KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} KNAPSACK_PRO_CI_NODE_TOTAL: 3 KNAPSACK_PRO_CI_NODE_INDEX: 1 - KNAPSACK_PRO_RSPEC_SPLIT_BY_TEST_EXAMPLES: true + # https://docs.knapsackpro.com/ruby/queue-mode/#dynamic-split-vs-fixed-split + KNAPSACK_PRO_FIXED_QUEUE_SPLIT: false TRUNK_API_TOKEN: ${{ inputs.trunk-token }} TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} @@ -140,6 +141,8 @@ runs: rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" mkdir -p "${TRUNK_LOCAL_UPLOAD_DIR}" set +e + # Initialize is not required, but some custom setups will use it + bundle exec rake "knapsack_pro:queue:rspec:initialize" bundle exec rake "knapsack_pro:queue:rspec" set -e ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" diff --git a/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb b/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb deleted file mode 100644 index b181a817..00000000 --- a/.github/actions/test_ruby_gem_uploads/spec/variant_quarantine_spec_alt.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -describe 'variant_quarantine_test_alt' do - it 'should be quarantined when run with variant' do - # This test should fail when run without a variant (not quarantined) - # and be quarantined (not fail) when run with a variant - expect(2 + 2).to eq(5) - end -end From c62d91566b87637fc2d763c73bbe3c6361df2caa Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:05:43 +0000 Subject: [PATCH 08/17] cleanup --- .github/actions/test_ruby_gem_uploads/Gemfile | 2 +- .github/actions/test_ruby_gem_uploads/README.md | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index fd8cbc30..c322eaab 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -8,4 +8,4 @@ gem 'rake' gem 'rake-compiler' gem 'logger' # donotland -gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' +# gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/README.md b/.github/actions/test_ruby_gem_uploads/README.md index e5dfbf92..95019806 100644 --- a/.github/actions/test_ruby_gem_uploads/README.md +++ b/.github/actions/test_ruby_gem_uploads/README.md @@ -19,8 +19,21 @@ See more in the [rspec README.md](../../../rspec-trunk-flaky-tests/README.md). Repeat steps 1-5 above, and run: -1. `KNAPSACK_PRO_CI_NODE_BUILD_ID=$(openssl rand -base64 32)` -2. `KNAPSACK_PRO_TEST_DIR=spec KNAPSACK_PRO_TEST_FILE_PATTERN="**/*.rb" KNAPSACK_PRO_PROJECT_DIR=. KNAPSACK_PRO_REPOSITORY_ADAPTER=git KNAPSACK_PRO_LOG_LEVEL=debug KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC="" KNAPSACK_PRO_CI_NODE_TOTAL=1 KNAPSACK_PRO_CI_NODE_INDEX=0 KNAPSACK_PRO_FIXED_QUEUE_SPLIT=false bundle exec rake "knapsack_pro:queue:rspec"` +```bash +export KNAPSACK_PRO_CI_NODE_BUILD_ID=$(openssl rand -base64 32) +export KNAPSACK_PRO_TEST_DIR=spec +export KNAPSACK_PRO_TEST_FILE_PATTERN="**/*.rb" +export KNAPSACK_PRO_PROJECT_DIR=. +export KNAPSACK_PRO_REPOSITORY_ADAPTER=git +export KNAPSACK_PRO_LOG_LEVEL=debug +export KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC="" +export KNAPSACK_PRO_CI_NODE_TOTAL=1 +export KNAPSACK_PRO_CI_NODE_INDEX=0 +export KNAPSACK_PRO_FIXED_QUEUE_SPLIT=false + +bundle exec rake "knapsack_pro:queue:rspec:initialize" +bundle exec rake "knapsack_pro:queue:rspec" +``` ### Reference From 1a872e1d2494fda9c23ea7342b6e8e7839a24af8 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:07:56 +0000 Subject: [PATCH 09/17] cleanup gemfile edits --- .github/actions/test_ruby_gem_uploads/Gemfile | 3 +-- .github/actions/test_ruby_gem_uploads/README.md | 2 +- .github/actions/test_ruby_gem_uploads/action.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index c322eaab..027aa6dc 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -7,5 +7,4 @@ gem 'knapsack_pro' gem 'rake' gem 'rake-compiler' gem 'logger' -# donotland -# gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' +gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/README.md b/.github/actions/test_ruby_gem_uploads/README.md index 95019806..d16525ef 100644 --- a/.github/actions/test_ruby_gem_uploads/README.md +++ b/.github/actions/test_ruby_gem_uploads/README.md @@ -8,7 +8,7 @@ From `cd rspec-trunk-flaky-tests`: 2. `bundle install` 3. `bundle exec rake build` 4. `cd ../.github/actions/test_ruby_gem_uploads` -5. Add `gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests'` to [Gemfile](./Gemfile) +5. Verify [Gemfile](./Gemfile) points to the built gem, i.e. `gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests'` 6. Run `bundle exec rspec spec/variant_quarantine_spec.rb --format documentation` After `cd .github/actions/test_ruby_gem_uploads` diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index fb4ea4b4..4d7a33fd 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -66,8 +66,8 @@ runs: gem unpack "${GEM_FILE}" --target=rspec_trunk_flaky_tests mv rspec_trunk_flaky_tests/rspec_trunk_flaky_tests*/lib rspec_trunk_flaky_tests/lib bundle install + sed -i "s|gem 'rspec_trunk_flaky_tests'.*|gem 'rspec_trunk_flaky_tests', '${GEM_VERSION}', :path => './rspec_trunk_flaky_tests'|" Gemfile bundle lock --remove-platform ruby || true - printf "\ngem 'rspec_trunk_flaky_tests', '%s', :path => './rspec_trunk_flaky_tests'\n" "${GEM_VERSION}" >> Gemfile bundle install --gemfile Gemfile --local # - name: Run regular tests From 01ed5de7eded29e3d2fa23fbd0507a3bd0297beb Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:21:10 +0000 Subject: [PATCH 10/17] almost there --- .github/actions/test_ruby_gem_uploads/action.yaml | 8 +++++++- rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb | 7 +++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 4d7a33fd..0d6bb99f 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -135,7 +135,6 @@ runs: TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} TRUNK_LOCAL_UPLOAD_DIR: bundles - TRUNK_LOCAL_UPLOAD_DIR_ALLOW_MULTIPLE: true # donotland run: | set -euo pipefail rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" @@ -143,6 +142,13 @@ runs: set +e # Initialize is not required, but some custom setups will use it bundle exec rake "knapsack_pro:queue:rspec:initialize" + + if [ -d "${TRUNK_LOCAL_UPLOAD_DIR}" ]; then + echo "knapsack pro manual queue initialization should not trigger an upload" + ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" + exit 1 + fi + bundle exec rake "knapsack_pro:queue:rspec" set -e ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 2f623516..b6128fe9 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -65,13 +65,12 @@ def knapsack_detector_mode? def knapsack_detector_command? command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip - # donotland - # command_line.include?('knapsack_pro:rspec_test_example_detector') - if command_line.include?('knapsack_pro:rspec_test_example_detector') - # knapsack_pro:queue:rspec:initialize + if command_line.include?('knapsack_pro:rspec_test_example_detector') || command_line.include?('knapsack_pro:queue:rspec:initialize') + # donotland puts "Detected knapsack example detector command: #{command_line}\n\n".yellow true else + # donotland puts "Detected knapsack other command: #{command_line}\n\n".yellow false end From 26d039592ed869d69d511c4cde0c27c85b0b68f0 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:34:19 +0000 Subject: [PATCH 11/17] final test before cleanup --- .../actions/test_ruby_gem_uploads/action.yaml | 22 +++++++++++++------ .../lib/trunk_spec_helper.rb | 3 ++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 0d6bb99f..e6b243d5 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -144,18 +144,26 @@ runs: bundle exec rake "knapsack_pro:queue:rspec:initialize" if [ -d "${TRUNK_LOCAL_UPLOAD_DIR}" ]; then - echo "knapsack pro manual queue initialization should not trigger an upload" + echo "knapsack_pro manual queue initialization should not trigger an upload" ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" exit 1 fi - bundle exec rake "knapsack_pro:queue:rspec" + OUTPUT_FILE=$(mktemp) + bundle exec rake "knapsack_pro:queue:rspec" 2>&1 | tee "${OUTPUT_FILE}" set -e - ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" - mapfile -t bundle_files < <(find "${TRUNK_LOCAL_UPLOAD_DIR}" -maxdepth 1 -type f -name '*.bin') - if [ "${#bundle_files[@]}" -ne 1 ]; then - echo "Expected exactly one bundle in ${TRUNK_LOCAL_UPLOAD_DIR}, found ${#bundle_files[@]}" - ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" + OUTPUT=$(cat "${OUTPUT_FILE}") + rm -f "${OUTPUT_FILE}" + + if [ ! -d "${TRUNK_LOCAL_UPLOAD_DIR}" ]; then + echo "Expected ${TRUNK_LOCAL_UPLOAD_DIR} directory to be created" + exit 1 + fi + + REPORT_COUNT=$(echo "${OUTPUT}" | grep -c "Local Flaky tests report generated" || echo "0") + if [ "${REPORT_COUNT}" -ne 1 ]; then + echo "Expected 'Local Flaky tests report generated' to appear exactly once, found ${REPORT_COUNT} times" exit 1 fi + echo "Knapsack run correctly generated exactly one bundle at ${bundle_files[0]}" diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index b6128fe9..980f2093 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -65,7 +65,8 @@ def knapsack_detector_mode? def knapsack_detector_command? command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip - if command_line.include?('knapsack_pro:rspec_test_example_detector') || command_line.include?('knapsack_pro:queue:rspec:initialize') + if command_line.include?('knapsack_pro:rspec_test_example_detector') || + command_line.include?('knapsack_pro:queue:rspec:initialize') # donotland puts "Detected knapsack example detector command: #{command_line}\n\n".yellow true From 40c7723fe1af512972b9509aa5e9a97b0a2cdcb8 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:47:26 +0000 Subject: [PATCH 12/17] one more fix --- .github/actions/test_ruby_gem_uploads/action.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index e6b243d5..0211120a 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -143,7 +143,8 @@ runs: # Initialize is not required, but some custom setups will use it bundle exec rake "knapsack_pro:queue:rspec:initialize" - if [ -d "${TRUNK_LOCAL_UPLOAD_DIR}" ]; then + FILE_COUNT=$(find "${TRUNK_LOCAL_UPLOAD_DIR}" -maxdepth 1 -type f 2>/dev/null | wc -l) + if [ "${FILE_COUNT}" -ne 0 ]; then echo "knapsack_pro manual queue initialization should not trigger an upload" ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" exit 1 @@ -155,8 +156,10 @@ runs: OUTPUT=$(cat "${OUTPUT_FILE}") rm -f "${OUTPUT_FILE}" - if [ ! -d "${TRUNK_LOCAL_UPLOAD_DIR}" ]; then - echo "Expected ${TRUNK_LOCAL_UPLOAD_DIR} directory to be created" + FILE_COUNT=$(find "${TRUNK_LOCAL_UPLOAD_DIR}" -maxdepth 1 -type f 2>/dev/null | wc -l) + if [ "${FILE_COUNT}" -ne 1 ]; then + echo "knapsack_pro execution run should trigger an upload" + ls -la "${TRUNK_LOCAL_UPLOAD_DIR}" exit 1 fi From d06b61bfa3f29eaca1a864af0c8de398e3275163 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 20:56:54 +0000 Subject: [PATCH 13/17] final final --- .github/actions/test_ruby_gem_uploads/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 0211120a..7bbd4e49 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -127,8 +127,8 @@ runs: working-directory: ${{ github.action_path }} env: KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} - KNAPSACK_PRO_CI_NODE_TOTAL: 3 - KNAPSACK_PRO_CI_NODE_INDEX: 1 + KNAPSACK_PRO_CI_NODE_TOTAL: 1 + KNAPSACK_PRO_CI_NODE_INDEX: 0 # https://docs.knapsackpro.com/ruby/queue-mode/#dynamic-split-vs-fixed-split KNAPSACK_PRO_FIXED_QUEUE_SPLIT: false TRUNK_API_TOKEN: ${{ inputs.trunk-token }} @@ -169,4 +169,4 @@ runs: exit 1 fi - echo "Knapsack run correctly generated exactly one bundle at ${bundle_files[0]}" + echo "Knapsack run correctly generated exactly one bundle" From 0450c164cef4e8196b0da557fa946b45584ad6f7 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 21:00:53 +0000 Subject: [PATCH 14/17] cleanup --- .../actions/test_ruby_gem_uploads/action.yaml | 100 +++++++++--------- .github/workflows/smoke_test_main.yml | 4 - .../lib/trunk_spec_helper.rb | 12 +-- 3 files changed, 52 insertions(+), 64 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 7bbd4e49..7e40930b 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -70,56 +70,56 @@ runs: bundle lock --remove-platform ruby || true bundle install --gemfile Gemfile --local - # - name: Run regular tests - # id: run-regular-tests - # shell: bash - # working-directory: ${{ github.action_path }} - # run: | - # bundle exec rspec spec/test_spec.rb --format documentation - # env: - # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - - # - name: Run variant quarantine test without variant (should fail - not quarantined) - # id: run-variant-test-no-variant - # shell: bash - # working-directory: ${{ github.action_path }} - # run: | - # set +e - # bundle exec rspec spec/variant_quarantine_spec.rb --format documentation - # EXIT_CODE=$? - # if [ $EXIT_CODE -eq 0 ]; then - # echo "ERROR: Variant quarantine test without variant should have failed (test should not be quarantined)" - # exit 1 - # else - # echo "SUCCESS: Variant quarantine test without variant failed as expected (test is not quarantined)" - # fi - # env: - # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - # TRUNK_VARIANT: "" - - # - name: Run variant quarantine test with variant (should pass - quarantined) - # id: run-variant-test-with-variant - # shell: bash - # working-directory: ${{ github.action_path }} - # run: | - # bundle exec rspec spec/variant_quarantine_spec.rb --format documentation - # EXIT_CODE=$? - # if [ $EXIT_CODE -ne 0 ]; then - # echo "ERROR: Variant quarantine test with variant should have passed (test should be quarantined)" - # echo "Note: The variant_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" - # exit 1 - # else - # echo "SUCCESS: Variant quarantine test with variant passed (test was quarantined)" - # fi - # env: - # TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} - # TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} - # TRUNK_API_TOKEN: ${{ inputs.trunk-token }} - # TRUNK_VARIANT: smoke-test-variant + - name: Run regular tests + id: run-regular-tests + shell: bash + working-directory: ${{ github.action_path }} + run: | + bundle exec rspec spec/test_spec.rb --format documentation + env: + TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + + - name: Run variant quarantine test without variant (should fail - not quarantined) + id: run-variant-test-no-variant + shell: bash + working-directory: ${{ github.action_path }} + run: | + set +e + bundle exec rspec spec/variant_quarantine_spec.rb --format documentation + EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + echo "ERROR: Variant quarantine test without variant should have failed (test should not be quarantined)" + exit 1 + else + echo "SUCCESS: Variant quarantine test without variant failed as expected (test is not quarantined)" + fi + env: + TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + TRUNK_VARIANT: "" + + - name: Run variant quarantine test with variant (should pass - quarantined) + id: run-variant-test-with-variant + shell: bash + working-directory: ${{ github.action_path }} + run: | + bundle exec rspec spec/variant_quarantine_spec.rb --format documentation + EXIT_CODE=$? + if [ $EXIT_CODE -ne 0 ]; then + echo "ERROR: Variant quarantine test with variant should have passed (test should be quarantined)" + echo "Note: The variant_quarantine_test must be quarantined with variant 'smoke-test-variant' in the system" + exit 1 + else + echo "SUCCESS: Variant quarantine test with variant passed (test was quarantined)" + fi + env: + TRUNK_PUBLIC_API_ADDRESS: ${{ inputs.trunk-public-api-address }} + TRUNK_ORG_URL_SLUG: ${{ inputs.trunk-org-slug }} + TRUNK_API_TOKEN: ${{ inputs.trunk-token }} + TRUNK_VARIANT: smoke-test-variant - name: Run knapsack_pro queue and verify a single local bundle shell: bash diff --git a/.github/workflows/smoke_test_main.yml b/.github/workflows/smoke_test_main.yml index addf3d0d..0ff067bf 100644 --- a/.github/workflows/smoke_test_main.yml +++ b/.github/workflows/smoke_test_main.yml @@ -3,10 +3,6 @@ permissions: actions: read on: workflow_dispatch: {} - # donotland - push: - branches: - - tyler/knapsack-smoke-test schedule: - cron: 0 0/2 * * * repository_dispatch: diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 980f2093..8e859014 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -65,16 +65,8 @@ def knapsack_detector_mode? def knapsack_detector_command? command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip - if command_line.include?('knapsack_pro:rspec_test_example_detector') || - command_line.include?('knapsack_pro:queue:rspec:initialize') - # donotland - puts "Detected knapsack example detector command: #{command_line}\n\n".yellow - true - else - # donotland - puts "Detected knapsack other command: #{command_line}\n\n".yellow - false - end + command_line.include?('knapsack_pro:rspec_test_example_detector') || + command_line.include?('knapsack_pro:queue:rspec:initialize') end def trunk_disabled From d62c9f5d5500417818048fe495e4204b5a4eb257 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 21:06:46 +0000 Subject: [PATCH 15/17] ensure execution --- .github/actions/test_ruby_gem_uploads/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 7e40930b..44e6061f 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -126,6 +126,8 @@ runs: if: inputs.knapsack-pro-test-suite-token-rspec != '' working-directory: ${{ github.action_path }} env: + KNAPSACK_PRO_TEST_DIR: spec + KNAPSACK_PRO_TEST_FILE_PATTERN: "**/*.rb" KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} KNAPSACK_PRO_CI_NODE_TOTAL: 1 KNAPSACK_PRO_CI_NODE_INDEX: 0 @@ -139,6 +141,7 @@ runs: set -euo pipefail rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" mkdir -p "${TRUNK_LOCAL_UPLOAD_DIR}" + export KNAPSACK_PRO_CI_NODE_BUILD_ID=$(openssl rand -base64 32) set +e # Initialize is not required, but some custom setups will use it bundle exec rake "knapsack_pro:queue:rspec:initialize" From 17d91b034ff883b656b2b08a5707ae04f1ff1a33 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 21:19:55 +0000 Subject: [PATCH 16/17] simplify? --- .github/actions/test_ruby_gem_uploads/action.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 44e6061f..7e40930b 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -126,8 +126,6 @@ runs: if: inputs.knapsack-pro-test-suite-token-rspec != '' working-directory: ${{ github.action_path }} env: - KNAPSACK_PRO_TEST_DIR: spec - KNAPSACK_PRO_TEST_FILE_PATTERN: "**/*.rb" KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} KNAPSACK_PRO_CI_NODE_TOTAL: 1 KNAPSACK_PRO_CI_NODE_INDEX: 0 @@ -141,7 +139,6 @@ runs: set -euo pipefail rm -rf "${TRUNK_LOCAL_UPLOAD_DIR}" mkdir -p "${TRUNK_LOCAL_UPLOAD_DIR}" - export KNAPSACK_PRO_CI_NODE_BUILD_ID=$(openssl rand -base64 32) set +e # Initialize is not required, but some custom setups will use it bundle exec rake "knapsack_pro:queue:rspec:initialize" From 41608e8242a7b2469eca798b1e5ba564cfb4b16b Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 28 Jan 2026 21:28:38 +0000 Subject: [PATCH 17/17] specify tests --- .github/actions/test_ruby_gem_uploads/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 7e40930b..cc2ca0a0 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -126,6 +126,9 @@ runs: if: inputs.knapsack-pro-test-suite-token-rspec != '' working-directory: ${{ github.action_path }} env: + KNAPSACK_PRO_TEST_DIR: spec + # Required when using queue intialization + KNAPSACK_PRO_TEST_FILE_PATTERN: spec/*.rb KNAPSACK_PRO_TEST_SUITE_TOKEN_RSPEC: ${{ inputs.knapsack-pro-test-suite-token-rspec }} KNAPSACK_PRO_CI_NODE_TOTAL: 1 KNAPSACK_PRO_CI_NODE_INDEX: 0