diff --git a/bundle/config/mutator/sort_job_tasks.go b/bundle/config/mutator/sort_job_tasks.go new file mode 100644 index 000000000..8b9ca7f17 --- /dev/null +++ b/bundle/config/mutator/sort_job_tasks.go @@ -0,0 +1,33 @@ +package mutator + +import ( + "context" + "sort" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type sortJobTasks struct{} + +// SortJobTasks sorts the tasks of each job in the bundle by task key. Sorting +// the task keys ensures that the diff computed by terraform is correct. For +// more details see the NOTE at https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/job#example-usage +// and https://github.com/databricks/terraform-provider-databricks/issues/4011. +func SortJobTasks() bundle.Mutator { + return &sortJobTasks{} +} + +func (m *sortJobTasks) Name() string { + return "SortJobTasks" +} + +func (m *sortJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + sort.Slice(job.Tasks, func(i, j int) bool { + return job.Tasks[i].TaskKey < job.Tasks[j].TaskKey + }) + } + + return nil +} diff --git a/bundle/config/mutator/sort_job_tasks_test.go b/bundle/config/mutator/sort_job_tasks_test.go new file mode 100644 index 000000000..c5de826d4 --- /dev/null +++ b/bundle/config/mutator/sort_job_tasks_test.go @@ -0,0 +1,80 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" +) + +func TestSortJobClusters(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "foo": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "c", + EnvironmentKey: "3", + }, + { + TaskKey: "a", + EnvironmentKey: "1", + }, + { + TaskKey: "b", + EnvironmentKey: "2", + }, + }, + }, + }, + "bar": { + JobSettings: &jobs.JobSettings{ + Tasks: []jobs.Task{ + { + TaskKey: "d", + }, + { + TaskKey: "e", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, SortJobTasks()) + assert.NoError(t, diags.Error()) + + assert.Equal(t, []jobs.Task{ + { + TaskKey: "a", + EnvironmentKey: "1", + }, + { + TaskKey: "b", + EnvironmentKey: "2", + }, + { + TaskKey: "c", + EnvironmentKey: "3", + }, + }, b.Config.Resources.Jobs["foo"].Tasks) + + assert.Equal(t, []jobs.Task{ + { + TaskKey: "d", + }, + { + TaskKey: "e", + }, + }, b.Config.Resources.Jobs["bar"].Tasks) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8039a4f13..4d737c642 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -62,6 +62,11 @@ func Initialize() bundle.Mutator { mutator.DefaultQueueing(), mutator.ExpandPipelineGlobPaths(), + // We sort job tasks by their task key to ensure sane diffs by the + // Databricks Terraform provider. This is done after variable resolution + // to ensure that the task key is fully resolved to what terraform will see. + mutator.SortJobTasks(), + // Configure use of WSFS for reads if the CLI is running on Databricks. mutator.ConfigureWSFS(),