Skip to content

Commit 3e38fe2

Browse files
denikclaude
denik
and
claude
authored
Always write out.test.toml; check drift on CI with git diff (#4737)
## Changes Instead of failing tests when out.test.toml disagrees, always write it to the source directory. CI now runs git diff --exit-code after tests to catch stale files. ## Why We avoid failing test just because out.test.toml is different. Otherwise you (or agent) needs multiple steps: "make test" to see the difference, run test-update or generate-out-test-toml to fix it then run test again. We still enforce that out.test.toml is up to date by checking "git diff" afterwards. --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 96d017a commit 3e38fe2

File tree

8 files changed

+23
-36
lines changed

8 files changed

+23
-36
lines changed

‎.github/workflows/push.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ jobs:
145145
- name: Analyze slow tests
146146
run: make slowest
147147

148+
- name: Check out.test.toml files are up to date
149+
shell: bash
150+
run: |
151+
if ! git diff --exit-code; then
152+
echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'make generate-out-test-toml' to update."
153+
exit 1
154+
fi
155+
148156
test-exp-aitools:
149157
needs:
150158
- cleanups

‎acceptance/acceptance_test.go‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -296,13 +296,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
296296
config, configPath := internal.LoadConfig(t, dir)
297297
skipReason := getSkipReason(&config, configPath)
298298

299-
if testdiff.OverwriteMode || OnlyOutTestToml {
300-
// Generate materialized config for this test
301-
// We do this before skipping the test, so the configs are generated for all tests.
302-
materializedConfig, err := internal.GenerateMaterializedConfig(config)
303-
require.NoError(t, err)
304-
testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig)
305-
}
299+
// Generate materialized config for this test.
300+
// We do this before skipping the test, so the configs are generated for all tests.
301+
materializedConfig, err := internal.GenerateMaterializedConfig(config)
302+
require.NoError(t, err)
303+
testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig)
306304

307305
// If only regenerating out.test.toml, skip the actual test execution
308306
if OnlyOutTestToml {
@@ -539,17 +537,13 @@ func runTest(t *testing.T,
539537
testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
540538

541539
// Generate materialized config for this test
542-
materializedConfig, err := internal.GenerateMaterializedConfig(config)
543-
require.NoError(t, err)
544-
testutil.WriteFile(t, filepath.Join(tmpDir, internal.MaterializedConfigFile), materializedConfig)
545-
546540
inputs := make(map[string]bool, 2)
547541
outputs := make(map[string]bool, 2)
548542
err = CopyDir(dir, tmpDir, inputs, outputs)
549543
require.NoError(t, err)
550544

551-
// Add materialized config to outputs for comparison
552-
outputs[internal.MaterializedConfigFile] = true
545+
// out.test.toml is written to the source dir during test discovery, not compared.
546+
delete(outputs, internal.MaterializedConfigFile)
553547

554548
timeout := config.Timeout
555549

@@ -666,11 +660,7 @@ func runTest(t *testing.T,
666660
continue
667661
}
668662

669-
skipRepls := false
670-
if relPath == internal.MaterializedConfigFile {
671-
skipRepls = true
672-
}
673-
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, skipRepls)
663+
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
674664
}
675665

676666
// Make sure there are not unaccounted for new files
@@ -704,7 +694,7 @@ func runTest(t *testing.T,
704694
if strings.HasPrefix(relPath, "out") {
705695
// We have a new file starting with "out"
706696
// Show the contents & support overwrite mode for it:
707-
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false)
697+
doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
708698
}
709699
}
710700

@@ -777,7 +767,7 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext,
777767
return append(env, key+"="+newValue)
778768
}
779769

780-
func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool, skipRepls bool) {
770+
func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
781771
pathRef := filepath.Join(dirRef, relPath)
782772
pathNew := filepath.Join(dirNew, relPath)
783773
bufRef, okRef := tryReading(t, pathRef)
@@ -792,7 +782,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
792782

793783
// Apply replacements to the new value only.
794784
// The reference value is stored after applying replacements.
795-
if !NoRepl && !skipRepls {
785+
if !NoRepl {
796786
valueNew = repls.Replace(valueNew)
797787
}
798788

‎acceptance/bundle/artifacts/whl_change_version/output.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl
3737
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py"
3838
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py"
3939
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt"
40-
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.test.toml"
4140
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt"
4241
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json"
4342
"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script"

‎acceptance/bundle/sync/dryrun/output.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11

22
>>> [CLI] bundle sync --dry-run --output text
33
Warn: Running in dry-run mode. No actual changes will be made.
4-
Action: PUT: .gitignore, databricks.yml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
4+
Action: PUT: .gitignore, databricks.yml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
55
Initial Sync Complete
66
Uploaded .gitignore
77
Uploaded databricks.yml
8-
Uploaded out.test.toml
98
Uploaded project-folder
109
Uploaded project-folder/app.py
1110
Uploaded project-folder/app.yaml

‎acceptance/bundle/sync/output.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
>>> [CLI] bundle sync --output text
3-
Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, out.test.toml
3+
Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py
44
Initial Sync Complete
55
Uploaded .gitignore
66
Uploaded databricks.yml
@@ -10,4 +10,3 @@ Uploaded dryrun/out.test.toml
1010
Uploaded ignored-folder/folder1
1111
Uploaded ignored-folder/folder1/script.py
1212
Uploaded ignored-folder/script.py
13-
Uploaded out.test.toml

‎acceptance/cmd/sync-from-file/output.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11

22
>>> [CLI] sync . /Users/[USERNAME] --exclude-from ignore.test-fixture
3-
Action: PUT: .gitignore, ignore.test-fixture, out.test.toml, project-folder/app.py, project-folder/app.yaml
3+
Action: PUT: .gitignore, ignore.test-fixture, project-folder/app.py, project-folder/app.yaml
44
Initial Sync Complete
55
Uploaded .gitignore
66
Uploaded ignore.test-fixture
7-
Uploaded out.test.toml
87
Uploaded project-folder
98
Uploaded project-folder/app.py
109
Uploaded project-folder/app.yaml

‎acceptance/cmd/sync/dryrun/output.txt‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
Warn: Running in dry-run mode. No actual changes will be made.
44
Initial Sync Complete
55
Uploaded .gitignore
6-
Uploaded out.test.toml
76
Uploaded project-folder
87
Uploaded project-folder/app.py
98
Uploaded project-folder/app.yaml
@@ -13,23 +12,20 @@ Uploaded project-folder/query.sql
1312
Warn: Running in dry-run mode. No actual changes will be made.
1413
Initial Sync Complete
1514
Uploaded .gitignore
16-
Uploaded out.test.toml
1715
Uploaded project-folder
1816
Uploaded project-folder/query.sql
1917

2018
>>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql
2119
Warn: Running in dry-run mode. No actual changes will be made.
2220
Initial Sync Complete
2321
Uploaded .gitignore
24-
Uploaded out.test.toml
2522

2623
>>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/*.py
2724
Warn: Running in dry-run mode. No actual changes will be made.
2825
Initial Sync Complete
2926
Uploaded .gitignore
3027
Uploaded ignored-folder
3128
Uploaded ignored-folder/script.py
32-
Uploaded out.test.toml
3329

3430
>>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/**/*.py
3531
Warn: Running in dry-run mode. No actual changes will be made.
@@ -38,7 +34,6 @@ Uploaded .gitignore
3834
Uploaded ignored-folder/folder1
3935
Uploaded ignored-folder/folder1/script.py
4036
Uploaded ignored-folder/script.py
41-
Uploaded out.test.toml
4237

4338
>>> [CLI] sync . /Users/[USERNAME] --dry-run --include ignored-folder/** --include !ignored-folder/folder1/big-blob
4439
Warn: Running in dry-run mode. No actual changes will be made.
@@ -48,7 +43,6 @@ Uploaded ignored-folder/folder1
4843
Uploaded ignored-folder/folder1/script.py
4944
Uploaded ignored-folder/folder1/script.yaml
5045
Uploaded ignored-folder/script.py
51-
Uploaded out.test.toml
5246
Uploaded project-folder
5347
Uploaded project-folder/app.py
5448
Uploaded project-folder/app.yaml

‎acceptance/cmd/sync/output.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11

22
>>> [CLI] sync . /Users/[USERNAME]
3-
Action: PUT: .gitignore, dryrun/out.test.toml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
3+
Action: PUT: .gitignore, dryrun/out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
44
Initial Sync Complete
55
Uploaded .gitignore
66
Uploaded dryrun
77
Uploaded dryrun/out.test.toml
8-
Uploaded out.test.toml
98
Uploaded project-folder
109
Uploaded project-folder/app.py
1110
Uploaded project-folder/app.yaml

0 commit comments

Comments
 (0)