From c0d3c4fbbfa050cb307aa862168f4747df790156 Mon Sep 17 00:00:00 2001
From: Shreyas Goenka <shreyas.goenka@databricks.com>
Date: Thu, 16 Jan 2025 17:15:48 +0100
Subject: [PATCH] Add acceptance tests for Schema depenendency auto-detection

---
 .../dlt_schema/databricks.yml                 | 43 +++++++++++++
 .../dlt_schema/out.json                       | 63 +++++++++++++++++++
 .../dlt_schema/script                         |  1 +
 .../dlt_target/databricks.yml                 | 43 +++++++++++++
 .../dlt_target/out.json                       | 63 +++++++++++++++++++
 .../dlt_target/script                         |  1 +
 .../volume/databricks.yml                     | 37 +++++++++++
 .../capture_schema_dependency/volume/out.json | 27 ++++++++
 .../capture_schema_dependency/volume/script   |  1 +
 9 files changed, 279 insertions(+)
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json
 create mode 100644 acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script

diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml
new file mode 100644
index 000000000..690df4443
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml
@@ -0,0 +1,43 @@
+bundle:
+  name: dlt-schema-field-dep-on-uc-schema
+
+resources:
+  schemas:
+    schema1:
+      catalog_name: catalog1
+      name: foobar
+
+    schema2:
+      catalog_name: catalog2
+      name: foobar
+
+    schema3:
+      catalog_name: catalog1
+      name: barfoo
+
+  pipelines:
+    pipeline1:
+      catalog: catalog1
+      schema: foobar
+
+    pipeline2:
+      catalog: catalog2
+      schema: foobar
+
+    pipeline3:
+      catalog: catalog1
+      schema: barfoo
+
+    pipeline4:
+      catalog: catalogX
+      schema: foobar
+
+    pipeline5:
+      catalog: catalog1
+      schema: schemaX
+
+    pipeline6:
+      schema: foobar
+
+    pipeline7:
+      name: whatever
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json
new file mode 100644
index 000000000..b6b4c1f73
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json
@@ -0,0 +1,63 @@
+{
+  "pipeline1": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "${resources.schemas.schema1.name}"
+  },
+  "pipeline2": {
+    "catalog": "catalog2",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "${resources.schemas.schema2.name}"
+  },
+  "pipeline3": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "${resources.schemas.schema3.name}"
+  },
+  "pipeline4": {
+    "catalog": "catalogX",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "foobar"
+  },
+  "pipeline5": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "schemaX"
+  },
+  "pipeline6": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "schema": "foobar"
+  },
+  "pipeline7": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "name": "whatever",
+    "permissions": []
+  }
+}
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script
new file mode 100644
index 000000000..f30782db6
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.pipelines > out.json
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml
new file mode 100644
index 000000000..075ade5b4
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml
@@ -0,0 +1,43 @@
+bundle:
+  name: dlt-schema-field-dep-on-uc-schema
+
+resources:
+  schemas:
+    schema1:
+      catalog_name: catalog1
+      name: foobar
+
+    schema2:
+      catalog_name: catalog2
+      name: foobar
+
+    schema3:
+      catalog_name: catalog1
+      name: barfoo
+
+  pipelines:
+    pipeline1:
+      catalog: catalog1
+      target: foobar
+
+    pipeline2:
+      catalog: catalog2
+      target: foobar
+
+    pipeline3:
+      catalog: catalog1
+      target: barfoo
+
+    pipeline4:
+      catalog: catalogX
+      target: foobar
+
+    pipeline5:
+      catalog: catalog1
+      target: schemaX
+
+    pipeline6:
+      target: foobar
+
+    pipeline7:
+      name: whatever
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json
new file mode 100644
index 000000000..3ad7ac41e
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json
@@ -0,0 +1,63 @@
+{
+  "pipeline1": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "${resources.schemas.schema1.name}"
+  },
+  "pipeline2": {
+    "catalog": "catalog2",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "${resources.schemas.schema2.name}"
+  },
+  "pipeline3": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "${resources.schemas.schema3.name}"
+  },
+  "pipeline4": {
+    "catalog": "catalogX",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "foobar"
+  },
+  "pipeline5": {
+    "catalog": "catalog1",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "schemaX"
+  },
+  "pipeline6": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "permissions": [],
+    "target": "foobar"
+  },
+  "pipeline7": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/dlt-schema-field-dep-on-uc-schema/default/state/metadata.json"
+    },
+    "name": "whatever",
+    "permissions": []
+  }
+}
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script
new file mode 100644
index 000000000..f30782db6
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.pipelines > out.json
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml
new file mode 100644
index 000000000..5d1a023be
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml
@@ -0,0 +1,37 @@
+bundle:
+  name: volume-deps-on-schema
+
+resources:
+  schemas:
+    schema1:
+      catalog_name: catalog1
+      name: foobar
+
+    schema2:
+      catalog_name: catalog2
+      name: foobar
+
+    schema3:
+      catalog_name: catalog1
+      name: barfoo
+
+  volumes:
+    volume1:
+      catalog_name: catalog1
+      schema_name: ${resources.schemas.schema1.name}
+
+    volume2:
+      catalog_name: catalog2
+      schema_name: ${resources.schemas.schema2.name}
+
+    volume3:
+      catalog_name: catalog1
+      schema_name: ${resources.schemas.schema3.name}
+
+    volume4:
+      catalog_name: catalogX
+      schema_name: foobar
+
+    volume5:
+      catalog_name: catalog1
+      schema_name: schemaX
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json
new file mode 100644
index 000000000..0d27a70f1
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json
@@ -0,0 +1,27 @@
+{
+  "volume1": {
+    "catalog_name": "catalog1",
+    "schema_name": "${resources.schemas.schema1.name}",
+    "volume_type": "MANAGED"
+  },
+  "volume2": {
+    "catalog_name": "catalog2",
+    "schema_name": "${resources.schemas.schema2.name}",
+    "volume_type": "MANAGED"
+  },
+  "volume3": {
+    "catalog_name": "catalog1",
+    "schema_name": "${resources.schemas.schema3.name}",
+    "volume_type": "MANAGED"
+  },
+  "volume4": {
+    "catalog_name": "catalogX",
+    "schema_name": "foobar",
+    "volume_type": "MANAGED"
+  },
+  "volume5": {
+    "catalog_name": "catalog1",
+    "schema_name": "schemaX",
+    "volume_type": "MANAGED"
+  }
+}
diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script
new file mode 100644
index 000000000..f4997af19
--- /dev/null
+++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script
@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq .resources.volumes > out.json