From b808d4d6f1b860f6286b2988a6226c5d506910b3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 9 Jan 2025 10:03:03 +0100 Subject: [PATCH] Add test for overriding list variable (#2099) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add override for "libraries". - Remove complexvar - it serves no purpose - we already have map variable ‘cluster’. --- .../bundle/variables/complex/databricks.yml | 19 +++------- .../bundle/variables/complex/out.default.json | 14 ------- .../bundle/variables/complex/out.dev.json | 37 ++++--------------- .../bundle/variables/complex/output.txt | 14 +++++++ acceptance/bundle/variables/complex/script | 10 +++-- 5 files changed, 33 insertions(+), 61 deletions(-) create mode 100644 acceptance/bundle/variables/complex/output.txt diff --git a/acceptance/bundle/variables/complex/databricks.yml b/acceptance/bundle/variables/complex/databricks.yml index 500f374e3..5dcc30b08 100644 --- a/acceptance/bundle/variables/complex/databricks.yml +++ b/acceptance/bundle/variables/complex/databricks.yml @@ -11,6 +11,7 @@ resources: - task_key: test job_cluster_key: key libraries: ${variables.libraries.value} + # specific fields of complex variable are referenced: task_key: "task with spark version ${var.cluster.spark_version} and jar ${var.libraries[0].jar}" variables: @@ -35,14 +36,6 @@ variables: - jar: "/path/to/jar" - egg: "/path/to/egg" - whl: "/path/to/whl" - complexvar: - type: complex - description: "A complex variable" - default: - key1: "value1" - key2: "value2" - key3: "value3" - targets: default: @@ -51,15 +44,13 @@ targets: variables: node_type: "Standard_DS3_v3" cluster: + # complex variables are not merged, so missing variables (policy_id) are not inherited spark_version: "14.2.x-scala2.11" node_type_id: ${var.node_type} num_workers: 4 spark_conf: spark.speculation: false spark.databricks.delta.retentionDurationCheck.enabled: false - complexvar: - type: complex - default: - key1: "1" - key2: "2" - key3: "3" + libraries: + - jar: "/newpath/to/jar" + - whl: "/newpath/to/whl" diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json index bbdb0f8de..6454562a6 100644 --- a/acceptance/bundle/variables/complex/out.default.json +++ b/acceptance/bundle/variables/complex/out.default.json @@ -76,20 +76,6 @@ "spark_version": "13.2.x-scala2.11" } }, - "complexvar": { - "default": { - "key1": "value1", - "key2": "value2", - "key3": "value3" - }, - "description": "A complex variable", - "type": "complex", - "value": { - "key1": "value1", - "key2": "value2", - "key3": "value3" - } - }, "libraries": { "default": [ { diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json index 330518c85..cede5feb2 100644 --- a/acceptance/bundle/variables/complex/out.dev.json +++ b/acceptance/bundle/variables/complex/out.dev.json @@ -32,16 +32,13 @@ "job_cluster_key": "key", "libraries": [ { - "jar": "/path/to/jar" + "jar": "/newpath/to/jar" }, { - "egg": "/path/to/egg" - }, - { - "whl": "/path/to/whl" + "whl": "/newpath/to/whl" } ], - "task_key": "task with spark version 14.2.x-scala2.11 and jar /path/to/jar" + "task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar" } ] } @@ -70,43 +67,23 @@ "spark_version": "14.2.x-scala2.11" } }, - "complexvar": { - "default": { - "key1": "1", - "key2": "2", - "key3": "3" - }, - "description": "A complex variable", - "type": "complex", - "value": { - "key1": "1", - "key2": "2", - "key3": "3" - } - }, "libraries": { "default": [ { - "jar": "/path/to/jar" + "jar": "/newpath/to/jar" }, { - "egg": "/path/to/egg" - }, - { - "whl": "/path/to/whl" + "whl": "/newpath/to/whl" } ], "description": "A libraries definition", "type": "complex", "value": [ { - "jar": "/path/to/jar" + "jar": "/newpath/to/jar" }, { - "egg": "/path/to/egg" - }, - { - "whl": "/path/to/whl" + "whl": "/newpath/to/whl" } ] }, diff --git a/acceptance/bundle/variables/complex/output.txt b/acceptance/bundle/variables/complex/output.txt new file mode 100644 index 000000000..ce295421f --- /dev/null +++ b/acceptance/bundle/variables/complex/output.txt @@ -0,0 +1,14 @@ + +>>> $CLI bundle validate -o json + +>>> jq .resources.jobs.my_job.tasks[0].task_key out.default.json +"task with spark version 13.2.x-scala2.11 and jar /path/to/jar" + +>>> $CLI bundle validate -o json -t dev + +>>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json +"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar" +policy_id and spark_conf.spark_random fields do not exist in dev target: + +>>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json +null diff --git a/acceptance/bundle/variables/complex/script b/acceptance/bundle/variables/complex/script index 9ee0ab02a..f8b61f18d 100644 --- a/acceptance/bundle/variables/complex/script +++ b/acceptance/bundle/variables/complex/script @@ -1,4 +1,8 @@ -$CLI bundle validate -o json | jq '{resources,variables}' > out.default.json +trace $CLI bundle validate -o json | jq '{resources,variables}' > out.default.json +trace jq .resources.jobs.my_job.tasks[0].task_key out.default.json | grep "task with spark version 13.2.x-scala2.11 and jar /path/to/jar" -# spark.random and policy_id should be empty in this target: -$CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json +trace $CLI bundle validate -o json -t dev | jq '{resources,variables}' > out.dev.json +trace jq .resources.jobs.my_job.tasks[0].task_key out.dev.json | grep "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar" + +echo policy_id and spark_conf.spark_random fields do not exist in dev target: +trace jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json | grep null