diff --git a/acceptance/bundle/override/job_cluster_var/databricks.yml b/acceptance/bundle/override/job_cluster_var/databricks.yml index 546cc2d8a..48e68c926 100644 --- a/acceptance/bundle/override/job_cluster_var/databricks.yml +++ b/acceptance/bundle/override/job_cluster_var/databricks.yml @@ -20,7 +20,6 @@ targets: jobs: foo: job_clusters: - # This does not work because merging is done before resolution - job_cluster_key: "${var.mykey}" new_cluster: node_type_id: i3.xlarge diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt index dee2a3b5b..cb76de5a8 100644 --- a/acceptance/bundle/override/job_cluster_var/output.txt +++ b/acceptance/bundle/override/job_cluster_var/output.txt @@ -9,17 +9,12 @@ "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "job_clusters": [ - { - "job_cluster_key": "key", - "new_cluster": { - "spark_version": "13.3.x-scala2.12" - } - }, { "job_cluster_key": "key", "new_cluster": { "node_type_id": "i3.xlarge", - "num_workers": 1 + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" } } ], @@ -51,17 +46,12 @@ Validation OK! "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "job_clusters": [ - { - "job_cluster_key": "key", - "new_cluster": { - "spark_version": "13.3.x-scala2.12" - } - }, { "job_cluster_key": "key", "new_cluster": { "node_type_id": "i3.2xlarge", - "num_workers": 4 + "num_workers": 4, + "spark_version": "13.3.x-scala2.12" } } ], diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index f7b3cd608..913685bcf 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -33,10 +33,6 @@ func Initialize() bundle.Mutator { // If it is an ancestor, this updates all paths to be relative to the sync root path. mutator.SyncInferRoot(), - mutator.MergeJobClusters(), - mutator.MergeJobParameters(), - mutator.MergeJobTasks(), - mutator.MergePipelineClusters(), mutator.InitializeWorkspaceClient(), mutator.PopulateCurrentUser(), mutator.LoadGitDetails(), @@ -70,6 +66,12 @@ func Initialize() bundle.Mutator { "workspace", "variables", ), + + mutator.MergeJobClusters(), + mutator.MergeJobParameters(), + mutator.MergeJobTasks(), + mutator.MergePipelineClusters(), + // Provide permission config errors & warnings after initializing all variables permissions.PermissionDiagnostics(), mutator.SetRunAs(),