Sort tasks by task_key before generating the terraform configuration

bundle/config/mutator/sort_job_tasks.go

@@ -0,0 +1,33 @@
+package mutator
+
+import (
+	"context"
+	"sort"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+)
+
+type sortJobTasks struct{}
+
+// SortJobTasks sorts the tasks of each job in the bundle by task key. Sorting
+// the task keys ensures that the diff computed by terraform is correct. For
+// more details see the NOTE at https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage
+// and https://github.com/databricks/terraform-provider-databricks/issues/4011.
+func SortJobTasks() bundle.Mutator {
+	return &sortJobTasks{}
+}
+
+func (m *sortJobTasks) Name() string {
+	return "SortJobTasks"
+}
+
+func (m *sortJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	for _, job := range b.Config.Resources.Jobs {
+		sort.Slice(job.Tasks, func(i, j int) bool {
+			return job.Tasks[i].TaskKey < job.Tasks[j].TaskKey
+		})
+	}
+
+	return nil
+}

bundle/config/mutator/sort_job_tasks_test.go

@@ -0,0 +1,80 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSortJobClusters(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"foo": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									TaskKey:        "c",
+									EnvironmentKey: "3",
+								},
+								{
+									TaskKey:        "a",
+									EnvironmentKey: "1",
+								},
+								{
+									TaskKey:        "b",
+									EnvironmentKey: "2",
+								},
+							},
+						},
+					},
+					"bar": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									TaskKey: "d",
+								},
+								{
+									TaskKey: "e",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, SortJobTasks())
+	assert.NoError(t, diags.Error())
+
+	assert.Equal(t, []jobs.Task{
+		{
+			TaskKey:        "a",
+			EnvironmentKey: "1",
+		},
+		{
+			TaskKey:        "b",
+			EnvironmentKey: "2",
+		},
+		{
+			TaskKey:        "c",
+			EnvironmentKey: "3",
+		},
+	}, b.Config.Resources.Jobs["foo"].Tasks)
+
+	assert.Equal(t, []jobs.Task{
+		{
+			TaskKey: "d",
+		},
+		{
+			TaskKey: "e",
+		},
+	}, b.Config.Resources.Jobs["bar"].Tasks)
+}

bundle/phases/initialize.go

@@ -62,6 +62,11 @@ func Initialize() bundle.Mutator {
 			mutator.DefaultQueueing(),
 			mutator.ExpandPipelineGlobPaths(),
 
+			// We sort job tasks by their task key to ensure sane diffs by the
+			// Databricks Terraform provider. This is done after variable resolution
+			// to ensure that the task key is fully resolved to what terraform will see.
+			mutator.SortJobTasks(),
+
 			// Configure use of WSFS for reads if the CLI is running on Databricks.
 			mutator.ConfigureWSFS(),