From 2a4fdd911e2b8ece6a9675bfa409ade9343847cb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 9 Jan 2025 11:14:47 +0100 Subject: [PATCH] Add a test showing bug in merge if variables are used for keys (#2101) Since merge happens first, before variable resolution, the two jobs are seen as different. I also updated override/job_cluster/script to include more of the output. --- .../bundle/override/job_cluster/output.txt | 66 ++++++++++----- acceptance/bundle/override/job_cluster/script | 4 +- .../override/job_cluster_var/databricks.yml | 37 ++++++++ .../override/job_cluster_var/output.txt | 84 +++++++++++++++++++ .../bundle/override/job_cluster_var/script | 4 + 5 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 acceptance/bundle/override/job_cluster_var/databricks.yml create mode 100644 acceptance/bundle/override/job_cluster_var/output.txt create mode 100644 acceptance/bundle/override/job_cluster_var/script diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt index dc7a5f75b..947d19032 100644 --- a/acceptance/bundle/override/job_cluster/output.txt +++ b/acceptance/bundle/override/job_cluster/output.txt @@ -1,30 +1,56 @@ >>> $CLI bundle validate -o json -t development { - "name": "job", - "job_clusters": [ - { - "job_cluster_key": "key", - "new_cluster": { - "node_type_id": "i3.xlarge", - "num_workers": 1, - "spark_version": "13.3.x-scala2.12" + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "key", + "new_cluster": { + "node_type_id": "i3.xlarge", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + } } - } - ] + ], + "name": "job", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {} + } } >>> $CLI bundle validate -o json -t staging { - "name": "job", - "job_clusters": [ - { - "job_cluster_key": "key", - "new_cluster": { - "node_type_id": "i3.2xlarge", - "num_workers": 4, - "spark_version": "13.3.x-scala2.12" + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "key", + "new_cluster": { + "node_type_id": "i3.2xlarge", + "num_workers": 4, + "spark_version": "13.3.x-scala2.12" + } } - } - ] + ], + "name": "job", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {} + } } diff --git a/acceptance/bundle/override/job_cluster/script b/acceptance/bundle/override/job_cluster/script index 3f6827bb7..4a26c433a 100644 --- a/acceptance/bundle/override/job_cluster/script +++ b/acceptance/bundle/override/job_cluster/script @@ -1,2 +1,2 @@ -trace $CLI bundle validate -o json -t development | jq '.resources.jobs.foo | {name,job_clusters}' -trace $CLI bundle validate -o json -t staging | jq '.resources.jobs.foo | {name,job_clusters}' +trace $CLI bundle validate -o json -t development | jq '.resources.jobs' +trace $CLI bundle validate -o json -t staging | jq '.resources.jobs' diff --git a/acceptance/bundle/override/job_cluster_var/databricks.yml b/acceptance/bundle/override/job_cluster_var/databricks.yml new file mode 100644 index 000000000..546cc2d8a --- /dev/null +++ b/acceptance/bundle/override/job_cluster_var/databricks.yml @@ -0,0 +1,37 @@ +bundle: + name: override_job_cluster + +variables: + mykey: + default: key + +resources: + jobs: + foo: + name: job + job_clusters: + - job_cluster_key: key + new_cluster: + spark_version: 13.3.x-scala2.12 + +targets: + development: + resources: + jobs: + foo: + job_clusters: + # This does not work because merging is done before resolution + - job_cluster_key: "${var.mykey}" + new_cluster: + node_type_id: i3.xlarge + num_workers: 1 + + staging: + resources: + jobs: + foo: + job_clusters: + - job_cluster_key: "${var.mykey}" + new_cluster: + node_type_id: i3.2xlarge + num_workers: 4 diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt new file mode 100644 index 000000000..dee2a3b5b --- /dev/null +++ b/acceptance/bundle/override/job_cluster_var/output.txt @@ -0,0 +1,84 @@ + +>>> $CLI bundle validate -o json -t development +{ + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "key", + "new_cluster": { + "spark_version": "13.3.x-scala2.12" + } + }, + { + "job_cluster_key": "key", + "new_cluster": { + "node_type_id": "i3.xlarge", + "num_workers": 1 + } + } + ], + "name": "job", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {} + } +} + +>>> $CLI bundle validate -t development +Name: override_job_cluster +Target: development +Workspace: + User: tester@databricks.com + Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development + +Validation OK! + +>>> $CLI bundle validate -o json -t staging +{ + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "key", + "new_cluster": { + "spark_version": "13.3.x-scala2.12" + } + }, + { + "job_cluster_key": "key", + "new_cluster": { + "node_type_id": "i3.2xlarge", + "num_workers": 4 + } + } + ], + "name": "job", + "permissions": [], + "queue": { + "enabled": true + }, + "tags": {} + } +} + +>>> $CLI bundle validate -t staging +Name: override_job_cluster +Target: staging +Workspace: + User: tester@databricks.com + Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging + +Validation OK! diff --git a/acceptance/bundle/override/job_cluster_var/script b/acceptance/bundle/override/job_cluster_var/script new file mode 100644 index 000000000..1cf373828 --- /dev/null +++ b/acceptance/bundle/override/job_cluster_var/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json -t development | jq '.resources.jobs' +trace $CLI bundle validate -t development +trace $CLI bundle validate -o json -t staging | jq '.resources.jobs' +trace $CLI bundle validate -t staging