diff --git a/acceptance/bundle/libraries/maven/.gitignore b/acceptance/bundle/libraries/maven/.gitignore new file mode 100644 index 000000000..15bcc6dd0 --- /dev/null +++ b/acceptance/bundle/libraries/maven/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/acceptance/bundle/libraries/maven/databricks.yml b/acceptance/bundle/libraries/maven/databricks.yml new file mode 100644 index 000000000..785142626 --- /dev/null +++ b/acceptance/bundle/libraries/maven/databricks.yml @@ -0,0 +1,27 @@ +bundle: + name: maven + + +resources: + jobs: + testjob: + name: test-job + tasks: + - task_key: dbt + spark_jar_task: + main_class_name: com.databricks.example.Main + + libraries: + - maven: + coordinates: org.jsoup:jsoup:1.7.2 + + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode diff --git a/acceptance/bundle/libraries/maven/out.job.libraries.txt b/acceptance/bundle/libraries/maven/out.job.libraries.txt new file mode 100644 index 000000000..2b4a0d5f5 --- /dev/null +++ b/acceptance/bundle/libraries/maven/out.job.libraries.txt @@ -0,0 +1,7 @@ +[ + { + "maven": { + "coordinates": "org.jsoup:jsoup:1.7.2" + } + } +] diff --git a/acceptance/bundle/libraries/maven/output.txt b/acceptance/bundle/libraries/maven/output.txt new file mode 100644 index 000000000..fd72d8d14 --- /dev/null +++ b/acceptance/bundle/libraries/maven/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate -o json +[ + { + "maven": { + "coordinates": "org.jsoup:jsoup:1.7.2" + } + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/maven/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/libraries/maven/script b/acceptance/bundle/libraries/maven/script new file mode 100644 index 000000000..06d1b6409 --- /dev/null +++ b/acceptance/bundle/libraries/maven/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.testjob.tasks[0].libraries' +trace $CLI bundle deploy +cat out.requests.txt | jq 'select(.path == "/api/2.1/jobs/create")' | jq '.body.tasks[0].libraries' > out.job.libraries.txt +rm out.requests.txt diff --git a/acceptance/bundle/libraries/maven/test.toml b/acceptance/bundle/libraries/maven/test.toml new file mode 100644 index 000000000..911438c24 --- /dev/null +++ b/acceptance/bundle/libraries/maven/test.toml @@ -0,0 +1,5 @@ +# We run this test only locally for now because we need to figure out how to do +# bundle destroy on script.cleanup first. +LocalOnly = true + +RecordRequests = true diff --git a/acceptance/bundle/libraries/pypi/.gitignore b/acceptance/bundle/libraries/pypi/.gitignore new file mode 100644 index 000000000..15bcc6dd0 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/.gitignore @@ -0,0 +1 @@ +.databricks diff --git a/acceptance/bundle/libraries/pypi/databricks.yml b/acceptance/bundle/libraries/pypi/databricks.yml new file mode 100644 index 000000000..67f3da254 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: pypi + + +resources: + jobs: + testjob: + name: test-job + tasks: + - task_key: dbt + dbt_task: + project_directory: ./ + profiles_directory: dbt_profiles/ + commands: + - 'dbt deps --target=${bundle.target}' + - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' + + libraries: + - pypi: + package: dbt-databricks>=1.8.0,<2.0.0 + + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + data_security_mode: SINGLE_USER + num_workers: 0 + spark_conf: + spark.master: "local[*, 4]" + spark.databricks.cluster.profile: singleNode + custom_tags: + ResourceClass: SingleNode diff --git a/acceptance/bundle/libraries/pypi/out.job.libraries.txt b/acceptance/bundle/libraries/pypi/out.job.libraries.txt new file mode 100644 index 000000000..ddc7d84a5 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/out.job.libraries.txt @@ -0,0 +1,7 @@ +[ + { + "pypi": { + "package": "dbt-databricks>=1.8.0,<2.0.0" + } + } +] diff --git a/acceptance/bundle/libraries/pypi/output.txt b/acceptance/bundle/libraries/pypi/output.txt new file mode 100644 index 000000000..002677d64 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate -o json +[ + { + "pypi": { + "package": "dbt-databricks>=1.8.0,<2.0.0" + } + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pypi/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/libraries/pypi/script b/acceptance/bundle/libraries/pypi/script new file mode 100644 index 000000000..06d1b6409 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json | jq '.resources.jobs.testjob.tasks[0].libraries' +trace $CLI bundle deploy +cat out.requests.txt | jq 'select(.path == "/api/2.1/jobs/create")' | jq '.body.tasks[0].libraries' > out.job.libraries.txt +rm out.requests.txt diff --git a/acceptance/bundle/libraries/pypi/test.toml b/acceptance/bundle/libraries/pypi/test.toml new file mode 100644 index 000000000..911438c24 --- /dev/null +++ b/acceptance/bundle/libraries/pypi/test.toml @@ -0,0 +1,5 @@ +# We run this test only locally for now because we need to figure out how to do +# bundle destroy on script.cleanup first. +LocalOnly = true + +RecordRequests = true