Preserve empty lines in config-remote-sync (#4514)

## Changes

Yaml round-trip strips empty lines when diff is applied to the file. To
fix that, we mark empty lines with a marker comment and then restore
them after processing.

## Why

Empty line removals cause unexpected diff

## Tests

Added more acceptance tests and also added unit test for the
preservation logic to cover edge cases .

It is expected to see updated output in other config-remote-sync tests
because previously empty lines were removed

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: ilyakuz-db

acceptance/bundle/config-remote-sync/cli_defaults/output.txt

@@ -29,39 +29,22 @@ Resource: resources.pipelines.pipeline1
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,9 +1,7 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- targets:
-   default:
-     mode: development
--
- resources:
-   jobs:
-@@ -13,4 +11,5 @@
+@@ -13,4 +13,5 @@
            interval: 1
            unit: DAYS
 +        pause_status: UNPAUSED
        tasks:
          - task_key: main
-@@ -21,5 +20,8 @@
-             node_type_id: [NODE_TYPE_ID]
+@@ -22,4 +23,8 @@
              num_workers: 1
--
+
 +      tags:
 +        dev: default_tag_changed
 +      max_concurrent_runs: 5
 +      name: Custom Job Name
      job2:
        tasks:
-@@ -31,5 +33,4 @@
-             node_type_id: [NODE_TYPE_ID]
-             num_workers: 1
--
-   pipelines:
-     pipeline1:
-@@ -38,2 +39,4 @@
+@@ -38,2 +43,4 @@
          - notebook:
              path: /Users/{{workspace_user_name}}/notebook
 +      channel: PREVIEW

acceptance/bundle/config-remote-sync/config_edits/output.txt

@@ -34,26 +34,14 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,5 +1,4 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- resources:
-   jobs:
-@@ -13,5 +12,4 @@
-             node_type_id: [NODE_TYPE_ID]
-             num_workers: 1
--
- targets:
-   default:
-@@ -24,5 +22,5 @@
+@@ -24,5 +24,5 @@
                - success@example.com
              on_failure:
 -              - config-failure@example.com
 +              - remote-failure@example.com
            parameters:
              - name: catalog
-@@ -35,7 +33,6 @@
+@@ -35,7 +35,6 @@
                unit: DAYS
            tags:
 -            env: config-production

acceptance/bundle/config-remote-sync/formatting_preserved/databricks.yml.tmpl

@@ -2,18 +2,29 @@
 bundle:
   name: test-bundle-$UNIQUE_NAME
 
-# Resources section with extra spacing
+
+# Resources section with extra spacing (consecutive blank lines above)
 resources:
   jobs:
     my_job:
       # Comment about max concurrent runs
       max_concurrent_runs: 1
 
+      description: |
+        Main ETL job for data processing.
+
+        Runs daily at 2am UTC.
+
       # Task configuration
       tasks:
         - task_key: main
+
+          description: >-
+            Main processing task
+            that runs the notebook.
           notebook_task:
             notebook_path: /Users/{{workspace_user_name}}/notebook
+
           new_cluster:
             spark_version: $DEFAULT_SPARK_VERSION
             node_type_id: $NODE_TYPE_ID

acceptance/bundle/config-remote-sync/formatting_preserved/output.txt

@@ -3,12 +3,14 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
-=== Modify max_concurrent_runs from 1 to 5
+=== Modify max_concurrent_runs, description, and add timeout
 === Detect and save changes
 Detected changes in 1 resource(s):
 
 Resource: resources.jobs.my_job
+  description: replace
   max_concurrent_runs: replace
+  timeout_seconds: add
 
 
 
@@ -17,31 +19,33 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -2,5 +2,4 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- # Resources section with extra spacing
- resources:
-@@ -8,6 +7,5 @@
+@@ -9,10 +9,10 @@
      my_job:
        # Comment about max concurrent runs
 -      max_concurrent_runs: 1
--
 +      max_concurrent_runs: 5
+
+-      description: |
+-        Main ETL job for data processing.
++      description: |-
++        Updated ETL job.
+
+-        Runs daily at 2am UTC.
++        Runs hourly.
+
        # Task configuration
-       tasks:
-@@ -19,10 +17,8 @@
-             node_type_id: [NODE_TYPE_ID]
-             num_workers: 1 # inline comment about workers
--
-       # Tags for categorization
-       tags:
-         env: dev # environment tag
-         team: data-eng
--
-       # Flow-style formatting (should be preserved)
-       parameters:
+@@ -21,6 +21,5 @@
+
+           description: >-
+-            Main processing task
+-            that runs the notebook.
++            Main processing task that runs the notebook.
+           notebook_task:
+             notebook_path: /Users/{{workspace_user_name}}/notebook
+@@ -40,2 +39,3 @@
+         - {name: catalog, default: main}
+         - {name: schema, default: dev}
++      timeout_seconds: 3600
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/config-remote-sync/formatting_preserved/script

@@ -11,9 +11,11 @@ touch dummy.whl
 $CLI bundle deploy
 job_id="$(read_id.py my_job)"
 
-title "Modify max_concurrent_runs from 1 to 5"
+title "Modify max_concurrent_runs, description, and add timeout"
 edit_resource.py jobs $job_id <<EOF
 r["max_concurrent_runs"] = 5
+r["description"] = "Updated ETL job.\n\nRuns hourly."
+r["timeout_seconds"] = 3600
 EOF
 
 title "Detect and save changes"

acceptance/bundle/config-remote-sync/job_fields/output.txt

@@ -25,13 +25,7 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,5 +1,4 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- resources:
-   jobs:
-@@ -8,13 +7,19 @@
+@@ -8,13 +8,19 @@
          on_success:
            - success@example.com
 +        no_alert_for_skipped_runs: true
@@ -58,14 +52,13 @@ Resource: resources.jobs.my_job
 +            - samples.nyctaxi.trips
        environments:
          - environment_key: default
-@@ -31,5 +36,6 @@
+@@ -31,4 +37,6 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
--
 +      tags:
 +        team: data
+
  targets:
-   default:
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/config-remote-sync/job_multiple_tasks/output.txt

@@ -21,13 +21,7 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,5 +1,4 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- resources:
-   jobs:
-@@ -13,13 +12,11 @@
+@@ -13,13 +13,11 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
 -        - task_key: d_task
@@ -47,7 +41,7 @@ Resource: resources.jobs.my_job
 +          task_key: e_task
          - task_key: c_task
            notebook_task:
-@@ -28,7 +25,8 @@
+@@ -28,7 +26,8 @@
              spark_version: 13.3.x-snapshot-scala2.12
              node_type_id: [NODE_TYPE_ID]
 -            num_workers: 2
@@ -58,12 +52,6 @@ Resource: resources.jobs.my_job
 +          timeout_seconds: 3600
          - task_key: a_task
            notebook_task:
-@@ -40,5 +38,4 @@
-           depends_on:
-             - task_key: c_task
--
-     rename_task_job:
-       tasks:
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
 Deploying resources...
 Updating deployment state...
@@ -88,7 +76,7 @@ Resource: resources.jobs.rename_task_job
 >>> diff.py databricks.yml.backup2 databricks.yml
 --- databricks.yml.backup2
 +++ databricks.yml
-@@ -40,14 +40,14 @@
+@@ -42,14 +42,14 @@
      rename_task_job:
        tasks:
 -        - task_key: b_task
@@ -109,14 +97,14 @@ Resource: resources.jobs.rename_task_job
 +            - task_key: b_task_renamed
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/d_task
-@@ -58,5 +58,5 @@
+@@ -60,5 +60,5 @@
          - task_key: c_task
            depends_on:
 -            - task_key: b_task
 +            - task_key: b_task_renamed
            notebook_task:
              notebook_path: /Users/{{workspace_user_name}}/c_task
-@@ -67,7 +67,14 @@
+@@ -69,7 +69,14 @@
          - task_key: a_task
            notebook_task:
 -            notebook_path: /Users/{{workspace_user_name}}/a_task

acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt

@@ -21,23 +21,14 @@ Resource: resources.pipelines.my_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,14 +1,12 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- resources:
-   pipelines:
+@@ -6,5 +6,5 @@
      my_pipeline:
        name: test-pipeline-[UNIQUE_NAME]
 -      development: false
 +      development: true
        libraries:
          - notebook:
-             path: /Users/{{workspace_user_name}}/notebook
--
-   jobs:
-     my_job:
-@@ -17,3 +15,3 @@
+@@ -17,3 +17,3 @@
            pipeline_task:
              pipeline_id: ${resources.pipelines.my_pipeline.id}
 -            full_refresh: false

acceptance/bundle/config-remote-sync/multiple_resources/output.txt

@@ -22,29 +22,24 @@ Resource: resources.jobs.job_two
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -1,9 +1,8 @@
- bundle:
-   name: test-bundle-[UNIQUE_NAME]
--
- resources:
+@@ -5,5 +5,5 @@
    jobs:
      job_one:
 -      max_concurrent_runs: 1
 +      max_concurrent_runs: 5
        tasks:
          - task_key: main
-@@ -14,7 +13,8 @@
-             node_type_id: [NODE_TYPE_ID]
+@@ -15,6 +15,8 @@
              num_workers: 1
--
+
 +      tags:
 +        team: data
      job_two:
 -      max_concurrent_runs: 2
 +      max_concurrent_runs: 10
        tasks:
          - task_key: main
-@@ -25,2 +25,4 @@
+@@ -25,2 +27,4 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
 +      tags:

acceptance/bundle/config-remote-sync/output_json/output.txt

@@ -8,7 +8,7 @@ Deployment complete!
     {
       "path": "[TEST_TMP_DIR]/databricks.yml",
       "originalContent": "bundle:\n  name: test-bundle-[UNIQUE_NAME]\n\nresources:\n  jobs:\n    test_job:\n      max_concurrent_runs: 1\n      tasks:\n        - task_key: main\n          notebook_task:\n            notebook_path: /Users/{{workspace_user_name}}/notebook\n          new_cluster:\n            spark_version: 13.3.x-snapshot-scala2.12\n            node_type_id: [NODE_TYPE_ID]\n            num_workers: 1\n",
-      "modifiedContent": "bundle:\n  name: test-bundle-[UNIQUE_NAME]\nresources:\n  jobs:\n    test_job:\n      max_concurrent_runs: 3\n      tasks:\n        - task_key: main\n          notebook_task:\n            notebook_path: /Users/{{workspace_user_name}}/notebook\n          new_cluster:\n            spark_version: 13.3.x-snapshot-scala2.12\n            node_type_id: [NODE_TYPE_ID]\n            num_workers: 1\n      tags:\n        env: test\n"
+      "modifiedContent": "bundle:\n  name: test-bundle-[UNIQUE_NAME]\n\nresources:\n  jobs:\n    test_job:\n      max_concurrent_runs: 3\n      tasks:\n        - task_key: main\n          notebook_task:\n            notebook_path: /Users/{{workspace_user_name}}/notebook\n          new_cluster:\n            spark_version: 13.3.x-snapshot-scala2.12\n            node_type_id: [NODE_TYPE_ID]\n            num_workers: 1\n      tags:\n        env: test\n"
     }
   ],
   "changes": {