databricks · shreyas-goenka · Sep 23, 2026 · Sep 23, 2026 · Sep 23, 2026 · Sep 23, 2026
@@ -0,0 +1 @@
+* Ignore the backend-provided `spark.sql.ansi.enabled: "true"` pipeline configuration default when detecting direct-engine drift, while preserving explicit configuration changes. ([#6813](https://github.com/databricks/cli/pull/6813))
@@ -33,6 +33,8 @@ resources:
 
   pipelines:
     pipeline1:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       libraries:
         - notebook:

@@ -46,7 +46,7 @@ Resource: resources.pipelines.pipeline1
 +      name: Custom Job Name
      job2:
        tasks:
-@@ -38,2 +43,4 @@
+@@ -40,2 +45,4 @@
          - notebook:
              path: /Users/{{workspace_user_name}}/notebook
 +      channel: PREVIEW

@@ -4,6 +4,8 @@ bundle:
 resources:
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       development: false
       libraries:

@@ -22,13 +22,13 @@ Resource: resources.pipelines.my_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -11,4 +11,5 @@
+@@ -13,4 +13,5 @@
              path: /Users/{{workspace_user_name}}/notebook
 
 +      continuous: true
    jobs:
      my_job:
-@@ -17,5 +18,5 @@
+@@ -19,5 +20,5 @@
            pipeline_task:
              pipeline_id: ${resources.pipelines.my_pipeline.id}
 -            full_refresh: false

@@ -8,6 +8,7 @@ resources:
       catalog: main
       schema: default
       configuration:
+        spark.sql.ansi.enabled: "true"
         key1: value1
       notifications:
         - email_recipients:

@@ -24,7 +24,7 @@ Resource: resources.pipelines.my_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -5,20 +5,27 @@
+@@ -5,21 +5,28 @@
    pipelines:
      my_pipeline:
 -      name: test-pipeline-[UNIQUE_NAME]
@@ -33,6 +33,7 @@ Resource: resources.pipelines.my_pipeline
 -      schema: default
 +      schema: prod
        configuration:
+         spark.sql.ansi.enabled: "true"
          key1: value1
 +        key2: value2
        notifications:

@@ -30,6 +30,8 @@ variables:
 resources:
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       development: false
       libraries:

@@ -37,7 +37,7 @@ Resource: resources.pipelines.my_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -45,14 +45,14 @@
+@@ -47,14 +47,14 @@
          dependencies:
            - ${var.my_env_dep}
 -          - --editable ${workspace.file_path}
@@ -56,7 +56,7 @@ Resource: resources.pipelines.my_pipeline
 -          default: ${var.target_env}
          - name: file_val
            default: ${var.file_override_var}
-@@ -66,4 +66,16 @@
+@@ -68,4 +68,16 @@
          - name: curated
            default: ${var.curated_schema}
 +        - default: ${var.my_catalog}
@@ -73,14 +73,14 @@ Resource: resources.pipelines.my_pipeline
 +          name: some_schema
        tasks:
          - task_key: main
-@@ -72,5 +84,5 @@
+@@ -74,5 +86,5 @@
              base_parameters:
                # Compound interpolation: mixes ${var.X} and ${bundle.X} refs.
 -              source_path: /mnt/${var.my_catalog}/${bundle.target}/raw/landing
 +              source_path: /mnt/${var.my_catalog}/${bundle.target}/raw/landing_v2
          # Resource reference: ${resources.pipelines.my_pipeline.id} resolves to
          # the deployed pipeline's ID. Used to test that resource refs are
-@@ -79,5 +91,13 @@
+@@ -81,5 +93,13 @@
            pipeline_task:
              pipeline_id: ${resources.pipelines.my_pipeline.id}
 -            full_refresh: false
@@ -95,7 +95,7 @@ Resource: resources.pipelines.my_pipeline
 +          task_key: secondary
        # Job environments use ${var.my_env_dep} via a pure variable reference.
        # The script adds a separate environments entry and modifies the existing
-@@ -89,4 +109,12 @@
+@@ -91,4 +111,12 @@
              dependencies:
                - ${var.my_env_dep}
 +              - nonexistent-other-pkg==2.0

@@ -23,6 +23,7 @@ resources:
       catalog: main
       schema: default
       configuration:
+        spark.sql.ansi.enabled: "true"
         key1: value1
       libraries:
         - notebook:

@@ -29,8 +29,8 @@ Resource: resources.pipelines.my_pipeline
 +      description: updated remotely
        job_clusters:
          - job_cluster_key: test_cluster
-@@ -25,4 +25,5 @@
-       configuration:
+@@ -26,4 +26,5 @@
+         spark.sql.ansi.enabled: "true"
          key1: value1
 +        key2: value2
        libraries:

@@ -9,6 +9,8 @@ bundle:
 resources:
   pipelines:
     split_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       catalog: main
       schema: default
@@ -25,6 +27,8 @@ targets:
     resources:
       pipelines:
         split_pipeline:
+          configuration:
+            spark.sql.ansi.enabled: "true"
           clusters:
             - label: maintenance
               num_workers: 1
@@ -18,14 +18,14 @@ Resource: resources.pipelines.split_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -15,5 +15,5 @@
+@@ -17,5 +17,5 @@
        clusters:
          - label: default
 -          num_workers: 2
 +          num_workers: 5
        libraries:
          - notebook:
-@@ -28,3 +28,3 @@
+@@ -32,3 +32,3 @@
            clusters:
              - label: maintenance
 -              num_workers: 1
@@ -46,7 +46,7 @@ Resource: resources.pipelines.split_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -28,3 +28,2 @@
+@@ -32,3 +32,2 @@
            clusters:
              - label: maintenance
 -              num_workers: 3

@@ -4,6 +4,8 @@ bundle:
 resources:
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       root_path: ./pipeline_root
       libraries:

@@ -23,14 +23,14 @@ Resource: resources.pipelines.my_pipeline
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -6,5 +6,5 @@
-     my_pipeline:
+@@ -8,5 +8,5 @@
+         spark.sql.ansi.enabled: "true"
        name: test-pipeline-[UNIQUE_NAME]
 -      root_path: ./pipeline_root
 +      root_path: ./pipeline_root_v2
        libraries:
          - notebook:
-@@ -16,9 +16,13 @@
+@@ -18,9 +18,13 @@
          - task_key: main
            notebook_task:
 -            notebook_path: ./src/notebook.py

@@ -20,5 +20,6 @@ resources:
         - file:
             path: pipeline.py
       configuration:
+        spark.sql.ansi.enabled: "true"
         europris.swipe.egress_streaming_schema: "${resources.schemas.my_schema.catalog_name}.${resources.schemas.my_schema.name}"
         europris.azure.subscription: ${var.AZURE.subscription}
@@ -7,6 +7,8 @@ workspace:
 resources:
   pipelines:
     test_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: "Snapshot Test Pipeline ${BUNDLE_NAME}"
       libraries:
         - notebook:

@@ -4,6 +4,8 @@ bundle:
 resources:
   pipelines:
     foo:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline-$UNIQUE_NAME
       libraries:
         - notebook:

@@ -30,6 +30,7 @@ Terraform will perform the following actions:
       ~ state                  = "IDLE" -> (known after apply)
       ~ storage                = "/Shared/old_storage" -> "/Shared/new_storage" # forces replacement
       ~ url                    = "[DATABRICKS_URL]/#joblist/pipelines/[NEW_PIPELINE_ID]" -> (known after apply)
+        # (1 unchanged attribute hidden)
 
       + deployment {
           + kind               = "BUNDLE"

@@ -7,6 +7,9 @@
   "path": "/api/2.0/pipelines",
   "body": {
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json"

@@ -2,6 +2,9 @@
   "pipelines": {
     "foo": {
       "channel": "CURRENT",
+      "configuration": {
+        "spark.sql.ansi.enabled": "true"
+      },
       "deployment": {
         "kind": "BUNDLE",
         "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate-[UNIQUE_NAME]/default/state/metadata.json"

@@ -4,6 +4,8 @@ bundle:
 resources:
   pipelines:
     foo:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline
       libraries:
         - notebook:

@@ -16,7 +16,7 @@ Terraform will perform the following actions:
         id                     = "[NEW_PIPELINE_ID]"
       ~ name                   = "lakeflow-pipeline" -> "test-pipeline"
       - root_path              = "/Workspace/Users/[USERNAME]/lakeflow_pipeline" -> null
-        # (12 unchanged attributes hidden)
+        # (13 unchanged attributes hidden)
 
       + deployment {
           + kind               = "BUNDLE"

@@ -18,6 +18,9 @@
   "body": {
     "catalog": "main",
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"

@@ -18,6 +18,9 @@
   "body": {
     "catalog": "main",
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"

@@ -12,6 +12,9 @@
       "foo": {
         "catalog": "main",
         "channel": "CURRENT",
+        "configuration": {
+          "spark.sql.ansi.enabled": "true"
+        },
         "deployment": {
           "kind": "BUNDLE",
           "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"

@@ -4,6 +4,8 @@ bundle:
 resources:
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-pipeline
       libraries:
         - file:

@@ -24,6 +24,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 {
   "spec": {
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/metadata.json"

@@ -12,6 +12,8 @@ resources:
             notebook_path: "./bar.py"
   pipelines:
     bar:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: test-bundle-pipeline-$UNIQUE_NAME
       libraries:
         - file:

@@ -11,6 +11,8 @@ resources:
     bar:
       # Unique: the Pipelines API refuses a duplicate name, so two overlapping runs on the
       # same workspace would collide. Jobs allow duplicates, which is why foo above does not.
+      configuration:
+        spark.sql.ansi.enabled: "true"
       name: bar-$UNIQUE_NAME
       catalog: main
       schema: default

@@ -75,6 +75,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
   "body": {
     "catalog": "main",
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "deployment_id": "[NUMID]",
       "kind": "BUNDLE",
@@ -119,6 +122,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
   "body": {
     "catalog": "main",
     "channel": "CURRENT",
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "deployment_id": "[NUMID]",
       "kind": "BUNDLE",

@@ -50,6 +50,8 @@ resources:
 
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       libraries:
         - file:
             path: pipeline.py

@@ -78,6 +78,8 @@ resources:
       name: test-model
   pipelines:
     my_pipeline:
+      configuration:
+        spark.sql.ansi.enabled: "true"
       budget_policy_id: ''
       clusters:
         - aws_attributes:

@@ -44,6 +44,9 @@
         "label": "default"
       }
     ],
+    "configuration": {
+      "spark.sql.ansi.enabled": "true"
+    },
     "deployment": {
       "kind": "BUNDLE",
       "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/direct/state/metadata.json"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		* Ignore the backend-provided `spark.sql.ansi.enabled: "true"` pipeline configuration default when detecting direct-engine drift, while preserving explicit configuration changes. ([#6813](https://github.com/databricks/cli/pull/6813))