Add a test showing bug in merge if variables are used for keys (#2101)

Since merge happens first, before variable resolution, the two jobs are
seen as different.

I also updated override/job_cluster/script to include more of the
output.

acceptance/bundle/override/job_cluster/output.txt

@@ -1,30 +1,56 @@
 
 >>> $CLI bundle validate -o json -t development
 {
-  "name": "job",
+  "foo": {
-  "job_clusters": [
+    "deployment": {
-    {
+      "kind": "BUNDLE",
-      "job_cluster_key": "key",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
-      "new_cluster": {
+    },
-        "node_type_id": "i3.xlarge",
+    "edit_mode": "UI_LOCKED",
-        "num_workers": 1,
+    "format": "MULTI_TASK",
-        "spark_version": "13.3.x-scala2.12"
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1,
+          "spark_version": "13.3.x-scala2.12"
+        }
       }
-    }
+    ],
-  ]
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
 }
 
 >>> $CLI bundle validate -o json -t staging
 {
-  "name": "job",
+  "foo": {
-  "job_clusters": [
+    "deployment": {
-    {
+      "kind": "BUNDLE",
-      "job_cluster_key": "key",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
-      "new_cluster": {
+    },
-        "node_type_id": "i3.2xlarge",
+    "edit_mode": "UI_LOCKED",
-        "num_workers": 4,
+    "format": "MULTI_TASK",
-        "spark_version": "13.3.x-scala2.12"
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4,
+          "spark_version": "13.3.x-scala2.12"
+        }
       }
-    }
+    ],
-  ]
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
 }

acceptance/bundle/override/job_cluster/script

@@ -1,2 +1,2 @@
-trace $CLI bundle validate -o json -t development | jq '.resources.jobs.foo | {name,job_clusters}'
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
-trace $CLI bundle validate -o json -t staging | jq '.resources.jobs.foo | {name,job_clusters}'
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'

acceptance/bundle/override/job_cluster_var/databricks.yml

@@ -0,0 +1,37 @@
+bundle:
+  name: override_job_cluster
+
+variables:
+  mykey:
+    default: key
+
+resources:
+  jobs:
+    foo:
+      name: job
+      job_clusters:
+        - job_cluster_key: key
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+
+targets:
+  development:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            # This does not work because merging is done before resolution
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.xlarge
+                num_workers: 1
+
+  staging:
+    resources:
+      jobs:
+        foo:
+          job_clusters:
+            - job_cluster_key: "${var.mykey}"
+              new_cluster:
+                node_type_id: i3.2xlarge
+                num_workers: 4

acceptance/bundle/override/job_cluster_var/output.txt

@@ -0,0 +1,84 @@
+
+>>> $CLI bundle validate -o json -t development
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.xlarge",
+          "num_workers": 1
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t development
+Name: override_job_cluster
+Target: development
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/development
+
+Validation OK!
+
+>>> $CLI bundle validate -o json -t staging
+{
+  "foo": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "spark_version": "13.3.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "key",
+        "new_cluster": {
+          "node_type_id": "i3.2xlarge",
+          "num_workers": 4
+        }
+      }
+    ],
+    "name": "job",
+    "permissions": [],
+    "queue": {
+      "enabled": true
+    },
+    "tags": {}
+  }
+}
+
+>>> $CLI bundle validate -t staging
+Name: override_job_cluster
+Target: staging
+Workspace:
+  User: tester@databricks.com
+  Path: /Workspace/Users/tester@databricks.com/.bundle/override_job_cluster/staging
+
+Validation OK!

acceptance/bundle/override/job_cluster_var/script

@@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json -t development | jq '.resources.jobs'
+trace $CLI bundle validate -t development
+trace $CLI bundle validate -o json -t staging | jq '.resources.jobs'
+trace $CLI bundle validate -t staging