steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add acceptance tests for Schema depenendency auto-detection

This commit is contained in:
Shreyas Goenka 2025-01-16 17:15:48 +01:00
parent fa43a4de38
commit c0d3c4fbbf
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
9 changed files with 279 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml Normal file
View File

@ -0,0 +1,43 @@
bundle:
name: dlt-schema-field-dep-on-uc-schema
resources:
schemas:
schema1:
catalog_name: catalog1
name: foobar
schema2:
catalog_name: catalog2
name: foobar
schema3:
catalog_name: catalog1
name: barfoo
pipelines:
pipeline1:
catalog: catalog1
schema: foobar
pipeline2:
catalog: catalog2
schema: foobar
pipeline3:
catalog: catalog1
schema: barfoo
pipeline4:
catalog: catalogX
schema: foobar
pipeline5:
catalog: catalog1
schema: schemaX
pipeline6:
schema: foobar
pipeline7:
name: whatever

63
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json Normal file
View File

@ -0,0 +1,63 @@
{
"pipeline1": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "${resources.schemas.schema1.name}"
},
"pipeline2": {
"catalog": "catalog2",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "${resources.schemas.schema2.name}"
},
"pipeline3": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "${resources.schemas.schema3.name}"
},
"pipeline4": {
"catalog": "catalogX",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "foobar"
},
"pipeline5": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "schemaX"
},
"pipeline6": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"schema": "foobar"
},
"pipeline7": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"name": "whatever",
"permissions": []
}
}

1
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.pipelines > out.json

43
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml Normal file
View File

@ -0,0 +1,43 @@
bundle:
name: dlt-schema-field-dep-on-uc-schema
resources:
schemas:
schema1:
catalog_name: catalog1
name: foobar
schema2:
catalog_name: catalog2
name: foobar
schema3:
catalog_name: catalog1
name: barfoo
pipelines:
pipeline1:
catalog: catalog1
target: foobar
pipeline2:
catalog: catalog2
target: foobar
pipeline3:
catalog: catalog1
target: barfoo
pipeline4:
catalog: catalogX
target: foobar
pipeline5:
catalog: catalog1
target: schemaX
pipeline6:
target: foobar
pipeline7:
name: whatever

63
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json Normal file
View File

@ -0,0 +1,63 @@
{
"pipeline1": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "${resources.schemas.schema1.name}"
},
"pipeline2": {
"catalog": "catalog2",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "${resources.schemas.schema2.name}"
},
"pipeline3": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "${resources.schemas.schema3.name}"
},
"pipeline4": {
"catalog": "catalogX",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "foobar"
},
"pipeline5": {
"catalog": "catalog1",
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "schemaX"
},
"pipeline6": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"permissions": [],
"target": "foobar"
},
"pipeline7": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
},
"name": "whatever",
"permissions": []
}
}

1
acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.pipelines > out.json

37
acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml Normal file
View File

@ -0,0 +1,37 @@
bundle:
name: volume-deps-on-schema
resources:
schemas:
schema1:
catalog_name: catalog1
name: foobar
schema2:
catalog_name: catalog2
name: foobar
schema3:
catalog_name: catalog1
name: barfoo
volumes:
volume1:
catalog_name: catalog1
schema_name: ${resources.schemas.schema1.name}
volume2:
catalog_name: catalog2
schema_name: ${resources.schemas.schema2.name}
volume3:
catalog_name: catalog1
schema_name: ${resources.schemas.schema3.name}
volume4:
catalog_name: catalogX
schema_name: foobar
volume5:
catalog_name: catalog1
schema_name: schemaX

27
acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json Normal file
View File

@ -0,0 +1,27 @@
{
"volume1": {
"catalog_name": "catalog1",
"schema_name": "${resources.schemas.schema1.name}",
"volume_type": "MANAGED"
},
"volume2": {
"catalog_name": "catalog2",
"schema_name": "${resources.schemas.schema2.name}",
"volume_type": "MANAGED"
},
"volume3": {
"catalog_name": "catalog1",
"schema_name": "${resources.schemas.schema3.name}",
"volume_type": "MANAGED"
},
"volume4": {
"catalog_name": "catalogX",
"schema_name": "foobar",
"volume_type": "MANAGED"
},
"volume5": {
"catalog_name": "catalog1",
"schema_name": "schemaX",
"volume_type": "MANAGED"
}
}

1
acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script Normal file
View File

@ -0,0 +1 @@
$CLI bundle validate -o json | jq .resources.volumes > out.json