From 6fd581d173a9f28167457b24b356c02f00be7ba9 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 25 Apr 2024 16:50:45 +0530
Subject: [PATCH 01/41] Allow variable references in non-string fields in the
 JSON schema (#1398)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Tests
Verified manually.

Before:
<img width="373" alt="Screenshot 2024-04-24 at 7 18 44 PM"
src="https://github.com/databricks/cli/assets/88374338/b4aef51f-0c16-4589-9d47-cdec9ab91158">

After:
<img width="364" alt="Screenshot 2024-04-24 at 7 18 31 PM"
src="https://github.com/databricks/cli/assets/88374338/3d8e412e-77ee-4641-943d-f99eab26ba02">
<img width="356" alt="Screenshot 2024-04-24 at 7 16 54 PM"
src="https://github.com/databricks/cli/assets/88374338/2aed369a-3c6a-4754-9c76-0969423f319e">

Manually verified the schema diff is sane. Example:
```
<                               "type": "boolean",
<                               "description": "If inference tables are enabled or not. NOTE: If you have already disabled payload logging once, you cannot enable again."
---
>                               "description": "If inference tables are enabled or not. NOTE: If you have already disabled payload logging once, you cannot enable again.",
>                               "anyOf": [
>                                 {
>                                   "type": "boolean"
>                                 },
>                                 {
>                                   "type": "string",
>                                   "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
>                                 }
>                               ]
```
---
 bundle/schema/schema.go      |  17 ++
 bundle/schema/schema_test.go | 490 +++++++++++++++++++++++++++++++----
 libs/dyn/dynvar/ref.go       |   4 +-
 3 files changed, 457 insertions(+), 54 deletions(-)

diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go
index b37f72d9b..ac0b4f2ec 100644
--- a/bundle/schema/schema.go
+++ b/bundle/schema/schema.go
@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/jsonschema"
 )
 
@@ -167,6 +168,22 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschem
 	}
 	jsonSchema := &jsonschema.Schema{Type: rootJavascriptType}
 
+	// If the type is a non-string primitive, then we allow it to be a string
+	// provided it's a pure variable reference (ie only a single variable reference).
+	if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType {
+		jsonSchema = &jsonschema.Schema{
+			AnyOf: []*jsonschema.Schema{
+				{
+					Type: rootJavascriptType,
+				},
+				{
+					Type:    jsonschema.StringType,
+					Pattern: dynvar.VariableRegex,
+				},
+			},
+		}
+	}
+
 	if docs != nil {
 		jsonSchema.Description = docs.Description
 	}
diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go
index d44a2082a..ea4fd1020 100644
--- a/bundle/schema/schema_test.go
+++ b/bundle/schema/schema_test.go
@@ -14,7 +14,15 @@ func TestIntSchema(t *testing.T) {
 
 	expected :=
 		`{
-			"type": "number"
+			"anyOf": [
+				{
+					"type": "number"
+				},
+				{
+					"type": "string",
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+				}
+			]
 		}`
 
 	schema, err := New(reflect.TypeOf(elemInt), nil)
@@ -33,7 +41,15 @@ func TestBooleanSchema(t *testing.T) {
 
 	expected :=
 		`{
-			"type": "boolean"
+			"anyOf": [
+				{
+					"type": "boolean"
+				},
+				{
+					"type": "string",
+					"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+				}
+			]
 		}`
 
 	schema, err := New(reflect.TypeOf(elem), nil)
@@ -101,46 +117,150 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"bool_val": {
-					"type": "boolean"
+					"anyOf": [
+						{
+							"type": "boolean"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"float32_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"float64_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"int16_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"int32_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"int64_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"int8_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"int_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"string_val": {
 					"type": "string"
 				},
 				"uint16_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"uint32_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"uint64_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"uint8_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"uint_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			},
 			"additionalProperties": false,
@@ -200,7 +320,15 @@ func TestStructOfStructsSchema(t *testing.T) {
 							"type": "object",
 							"properties": {
 								"a": {
-									"type": "number"
+									"anyOf": [
+										{
+											"type": "number"
+										},
+										{
+											"type": "string",
+											"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+										}
+									]
 								},
 								"b": {
 									"type": "string"
@@ -257,7 +385,15 @@ func TestStructOfMapsSchema(t *testing.T) {
 						"my_map": {
 							"type": "object",
 							"additionalProperties": {
-								"type": "number"
+								"anyOf": [
+									{
+										"type": "number"
+									},
+									{
+										"type": "string",
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									}
+								]
 							}
 						}
 					},
@@ -339,7 +475,15 @@ func TestMapOfPrimitivesSchema(t *testing.T) {
 		`{
 			"type": "object",
 			"additionalProperties": {
-				"type": "number"
+				"anyOf": [
+					{
+						"type": "number"
+					},
+					{
+						"type": "string",
+						"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+					}
+				]
 			}
 		}`
 
@@ -368,7 +512,15 @@ func TestMapOfStructSchema(t *testing.T) {
 				"type": "object",
 				"properties": {
 					"my_int": {
-						"type": "number"
+						"anyOf": [
+							{
+								"type": "number"
+							},
+							{
+								"type": "string",
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							}
+						]
 					}
 				},
 				"additionalProperties": false,
@@ -398,7 +550,15 @@ func TestMapOfMapSchema(t *testing.T) {
 			"additionalProperties": {
 				"type": "object",
 				"additionalProperties": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			}
 		}`
@@ -495,7 +655,15 @@ func TestSliceOfMapSchema(t *testing.T) {
 			"items": {
 				"type": "object",
 				"additionalProperties": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			}
 		}`
@@ -525,7 +693,15 @@ func TestSliceOfStructSchema(t *testing.T) {
 				"type": "object",
 				"properties": {
 					"my_int": {
-						"type": "number"
+						"anyOf": [
+							{
+								"type": "number"
+							},
+							{
+								"type": "string",
+								"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+							}
+						]
 					}
 				},
 				"additionalProperties": false,
@@ -575,7 +751,15 @@ func TestEmbeddedStructSchema(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"age": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"country": {
 					"type": "string"
@@ -607,7 +791,15 @@ func TestEmbeddedStructSchema(t *testing.T) {
 								"type": "object",
 								"properties": {
 									"age": {
-										"type": "number"
+										"anyOf": [
+											{
+												"type": "number"
+											},
+											{
+												"type": "string",
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+											}
+										]
 									},
 									"home": {
 										"type": "object",
@@ -694,7 +886,15 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"bar": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			},
 			"additionalProperties": false,
@@ -728,7 +928,15 @@ func TestDashFieldsAreSkipped(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"bar": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			},
 			"additionalProperties": false,
@@ -773,7 +981,15 @@ func TestPointerInStructSchema(t *testing.T) {
 					"type": "object",
 					"properties": {
 						"ptr_val2": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						}
 					},
 					"additionalProperties": false,
@@ -782,13 +998,29 @@ func TestPointerInStructSchema(t *testing.T) {
 					]
 				},
 				"float_val": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"ptr_bar": {
 					"type": "object",
 					"properties": {
 						"ptr_val2": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						}
 					},
 					"additionalProperties": false,
@@ -797,7 +1029,15 @@ func TestPointerInStructSchema(t *testing.T) {
 					]
 				},
 				"ptr_int": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"ptr_string": {
 					"type": "string"
@@ -860,7 +1100,15 @@ func TestGenericSchema(t *testing.T) {
 					"type": "object",
 					"properties": {
 						"age": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						},
 						"name": {
 							"type": "string"
@@ -875,7 +1123,15 @@ func TestGenericSchema(t *testing.T) {
 					"type": "object",
 					"properties": {
 						"age": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						},
 						"name": {
 							"type": "string"
@@ -895,7 +1151,15 @@ func TestGenericSchema(t *testing.T) {
 								"type": "object",
 								"properties": {
 									"age": {
-										"type": "number"
+										"anyOf": [
+											{
+												"type": "number"
+											},
+											{
+												"type": "string",
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+											}
+										]
 									},
 									"name": {
 										"type": "string"
@@ -910,7 +1174,15 @@ func TestGenericSchema(t *testing.T) {
 								"type": "object",
 								"properties": {
 									"age": {
-										"type": "number"
+										"anyOf": [
+											{
+												"type": "number"
+											},
+											{
+												"type": "string",
+												"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+											}
+										]
 									},
 									"name": {
 										"type": "string"
@@ -932,7 +1204,15 @@ func TestGenericSchema(t *testing.T) {
 												"type": "object",
 												"properties": {
 													"age": {
-														"type": "number"
+														"anyOf": [
+															{
+																"type": "number"
+															},
+															{
+																"type": "string",
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+															}
+														]
 													},
 													"name": {
 														"type": "string"
@@ -950,7 +1230,15 @@ func TestGenericSchema(t *testing.T) {
 												"type": "object",
 												"properties": {
 													"age": {
-														"type": "number"
+														"anyOf": [
+															{
+																"type": "number"
+															},
+															{
+																"type": "string",
+																"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+															}
+														]
 													},
 													"name": {
 														"type": "string"
@@ -1028,16 +1316,40 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"apple": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"bar": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"papaya": {
 					"type": "object",
 					"properties": {
 						"a": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						},
 						"b": {
 							"type": "string"
@@ -1111,7 +1423,15 @@ func TestDocIngestionForObject(t *testing.T) {
 							"description": "docs for a"
 						},
 						"b": {
-							"type": "number"
+							"anyOf": [
+								{
+									"type": "number"
+								},
+								{
+									"type": "string",
+									"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+								}
+							]
 						}
 					},
 					"additionalProperties": false,
@@ -1185,12 +1505,28 @@ func TestDocIngestionForSlice(t *testing.T) {
 						"type": "object",
 						"properties": {
 							"guava": {
-								"type": "number",
-								"description": "docs for guava"
+								"description": "docs for guava",
+								"anyOf": [
+									{
+										"type": "number"
+									},
+									{
+										"type": "string",
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									}
+								]
 							},
 							"pineapple": {
-								"type": "number",
-								"description": "docs for pineapple"
+								"description": "docs for pineapple",
+								"anyOf": [
+									{
+										"type": "number"
+									},
+									{
+										"type": "string",
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									}
+								]
 							}
 						},
 						"additionalProperties": false,
@@ -1268,12 +1604,28 @@ func TestDocIngestionForMap(t *testing.T) {
 						"type": "object",
 						"properties": {
 							"apple": {
-								"type": "number",
-								"description": "docs for apple"
+								"description": "docs for apple",
+								"anyOf": [
+									{
+										"type": "number"
+									},
+									{
+										"type": "string",
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									}
+								]
 							},
 							"mango": {
-								"type": "number",
-								"description": "docs for mango"
+								"description": "docs for mango",
+								"anyOf": [
+									{
+										"type": "number"
+									},
+									{
+										"type": "string",
+										"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+									}
+								]
 							}
 						},
 						"additionalProperties": false,
@@ -1324,8 +1676,16 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
 			"description": "docs for root",
 			"properties": {
 				"my_val": {
-					"type": "number",
-					"description": "docs for my val"
+					"description": "docs for my val",
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				}
 			},
 			"additionalProperties": false,
@@ -1395,7 +1755,15 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"apple": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"mango": {}
 			},
@@ -1436,7 +1804,15 @@ func TestBundleReadOnlytag(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"apple": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"pokemon": {
 					"type": "object",
@@ -1488,7 +1864,15 @@ func TestBundleInternalTag(t *testing.T) {
 			"type": "object",
 			"properties": {
 				"apple": {
-					"type": "number"
+					"anyOf": [
+						{
+							"type": "number"
+						},
+						{
+							"type": "string",
+							"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
+						}
+					]
 				},
 				"pokemon": {
 					"type": "object",
diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go
index a2047032a..e6340269f 100644
--- a/libs/dyn/dynvar/ref.go
+++ b/libs/dyn/dynvar/ref.go
@@ -6,7 +6,9 @@ import (
 	"github.com/databricks/cli/libs/dyn"
 )
 
-var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}`)
+const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}`
+
+var re = regexp.MustCompile(VariableRegex)
 
 // ref represents a variable reference.
 // It is a string [dyn.Value] contained in a larger [dyn.Value].

From e6523331037c93133828e51275b56eb71a67c10e Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 25 Apr 2024 16:51:10 +0530
Subject: [PATCH 02/41] Fix variable overrides in targets for non-string
 variables (#1397)

Before variable overrides that were not string in a target would not
work. This PR fixes that. Tested manually and via a unit test.
---
 bundle/config/root.go                         | 18 ++++---
 .../databricks.yml                            | 41 ++++++++++++++++
 bundle/tests/variables_test.go                | 49 +++++++++++++++++++
 3 files changed, 101 insertions(+), 7 deletions(-)
 create mode 100644 bundle/tests/variables/variable_overrides_in_target/databricks.yml

diff --git a/bundle/config/root.go b/bundle/config/root.go
index 17f2747ef..fda3759dd 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -408,15 +408,19 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
 
 		// For each variable, normalize its contents if it is a single string.
 		return dyn.Map(target, "variables", dyn.Foreach(func(_ dyn.Path, variable dyn.Value) (dyn.Value, error) {
-			if variable.Kind() != dyn.KindString {
+			switch variable.Kind() {
+
+			case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt:
+				// Rewrite the variable to a map with a single key called "default".
+				// This conforms to the variable type. Normalization back to the typed
+				// configuration will convert this to a string if necessary.
+				return dyn.NewValue(map[string]dyn.Value{
+					"default": variable,
+				}, variable.Location()), nil
+
+			default:
 				return variable, nil
 			}
-
-			// Rewrite the variable to a map with a single key called "default".
-			// This conforms to the variable type.
-			return dyn.NewValue(map[string]dyn.Value{
-				"default": variable,
-			}, variable.Location()), nil
 		}))
 	}))
 }
diff --git a/bundle/tests/variables/variable_overrides_in_target/databricks.yml b/bundle/tests/variables/variable_overrides_in_target/databricks.yml
new file mode 100644
index 000000000..4e52b5073
--- /dev/null
+++ b/bundle/tests/variables/variable_overrides_in_target/databricks.yml
@@ -0,0 +1,41 @@
+bundle:
+  name: foobar
+
+resources:
+  pipelines:
+    my_pipeline:
+      name: ${var.foo}
+      continuous: ${var.baz}
+      clusters:
+        - num_workers: ${var.bar}
+
+
+
+variables:
+  foo:
+    default: "a_string"
+    description: "A string variable"
+
+  bar:
+    default: 42
+    description: "An integer variable"
+
+  baz:
+    default: true
+    description: "A boolean variable"
+
+targets:
+  use-default-variable-values:
+
+  override-string-variable:
+    variables:
+      foo: "overridden_string"
+
+  override-int-variable:
+    variables:
+      bar: 43
+
+  override-both-bool-and-string-variables:
+    variables:
+      foo: "overridden_string"
+      baz: false
diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go
index fde36344f..f51802684 100644
--- a/bundle/tests/variables_test.go
+++ b/bundle/tests/variables_test.go
@@ -120,3 +120,52 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) {
 	assert.Equal(t, "cluster: some-test-cluster", b.Config.Variables["d"].Lookup.String())
 	assert.Equal(t, "instance-pool: some-test-instance-pool", b.Config.Variables["e"].Lookup.String())
 }
+
+func TestVariableTargetOverrides(t *testing.T) {
+	var tcases = []struct {
+		targetName         string
+		pipelineName       string
+		pipelineContinuous bool
+		pipelineNumWorkers int
+	}{
+		{
+			"use-default-variable-values",
+			"a_string",
+			true,
+			42,
+		},
+		{
+			"override-string-variable",
+			"overridden_string",
+			true,
+			42,
+		},
+		{
+			"override-int-variable",
+			"a_string",
+			true,
+			43,
+		},
+		{
+			"override-both-bool-and-string-variables",
+			"overridden_string",
+			false,
+			42,
+		},
+	}
+
+	for _, tcase := range tcases {
+		t.Run(tcase.targetName, func(t *testing.T) {
+			b := loadTarget(t, "./variables/variable_overrides_in_target", tcase.targetName)
+			diags := bundle.Apply(context.Background(), b, bundle.Seq(
+				mutator.SetVariables(),
+				mutator.ResolveVariableReferences("variables")),
+			)
+			require.NoError(t, diags.Error())
+
+			assert.Equal(t, tcase.pipelineName, b.Config.Resources.Pipelines["my_pipeline"].Name)
+			assert.Equal(t, tcase.pipelineContinuous, b.Config.Resources.Pipelines["my_pipeline"].Continuous)
+			assert.Equal(t, tcase.pipelineNumWorkers, b.Config.Resources.Pipelines["my_pipeline"].Clusters[0].NumWorkers)
+		})
+	}
+}

From d949f2b4f2a3cd59cd57cdefef25fefbbb29af1d Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 25 Apr 2024 16:53:50 +0530
Subject: [PATCH 03/41] Fix bundle schema for variables (#1396)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Changes
This PR fixes the variable schema to:
1. Allow non-string values in the "default" value of a variable.
2. Allow non-string overrides in a target for a variable.

## Tests
Manually. There are no longer squiggly lines.

Before:
<img width="329" alt="Screenshot 2024-04-24 at 3 26 43 PM"
src="https://github.com/databricks/cli/assets/88374338/43be02c2-80a4-4f80-bd79-0f3e1e93ee17">


After:
<img width="361" alt="Screenshot 2024-04-24 at 3 26 10 PM"
src="https://github.com/databricks/cli/assets/88374338/2c1fb892-a2a2-478b-8d2e-9bda6d844b54">
---
 bundle/config/mutator/set_variables.go |  2 -
 cmd/bundle/schema.go                   | 56 ++++++++++++++++
 libs/jsonschema/schema.go              | 38 ++++++++++-
 libs/jsonschema/schema_test.go         | 90 ++++++++++++++++++++++++++
 4 files changed, 183 insertions(+), 3 deletions(-)

diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go
index bb88379e0..eae1fe2ab 100644
--- a/bundle/config/mutator/set_variables.go
+++ b/bundle/config/mutator/set_variables.go
@@ -53,8 +53,6 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
 	}
 
 	// We should have had a value to set for the variable at this point.
-	// TODO: use cmdio to request values for unassigned variables if current
-	// terminal is a tty. Tracked in https://github.com/databricks/cli/issues/379
 	return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
 }
 
diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go
index 0f27142bd..b0d6b3dd5 100644
--- a/cmd/bundle/schema.go
+++ b/cmd/bundle/schema.go
@@ -7,9 +7,58 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/schema"
 	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/jsonschema"
 	"github.com/spf13/cobra"
 )
 
+func overrideVariables(s *jsonschema.Schema) error {
+	// Override schema for default values to allow for multiple primitive types.
+	// These are normalized to strings when converted to the typed representation.
+	err := s.SetByPath("variables.*.default", jsonschema.Schema{
+		AnyOf: []*jsonschema.Schema{
+			{
+				Type: jsonschema.StringType,
+			},
+			{
+				Type: jsonschema.BooleanType,
+			},
+			{
+				Type: jsonschema.NumberType,
+			},
+			{
+				Type: jsonschema.IntegerType,
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Override schema for variables in targets to allow just specifying the value
+	// along side overriding the variable definition if needed.
+	ns, err := s.GetByPath("variables.*")
+	if err != nil {
+		return err
+	}
+	return s.SetByPath("targets.*.variables.*", jsonschema.Schema{
+		AnyOf: []*jsonschema.Schema{
+			{
+				Type: jsonschema.StringType,
+			},
+			{
+				Type: jsonschema.BooleanType,
+			},
+			{
+				Type: jsonschema.NumberType,
+			},
+			{
+				Type: jsonschema.IntegerType,
+			},
+			&ns,
+		},
+	})
+}
+
 func newSchemaCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "schema",
@@ -30,6 +79,13 @@ func newSchemaCommand() *cobra.Command {
 			return err
 		}
 
+		// Override schema for variables to take into account normalization of default
+		// variable values and variable overrides in a target.
+		err = overrideVariables(schema)
+		if err != nil {
+			return err
+		}
+
 		// Print the JSON schema to stdout.
 		result, err := json.MarshalIndent(schema, "", "  ")
 		if err != nil {
diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go
index 967e2e9cd..f1e223ec7 100644
--- a/libs/jsonschema/schema.go
+++ b/libs/jsonschema/schema.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"regexp"
 	"slices"
+	"strings"
 
 	"github.com/databricks/cli/internal/build"
 	"golang.org/x/mod/semver"
@@ -81,6 +82,41 @@ func (s *Schema) ParseString(v string) (any, error) {
 	return fromString(v, s.Type)
 }
 
+func (s *Schema) getByPath(path string) (*Schema, error) {
+	p := strings.Split(path, ".")
+
+	res := s
+	for _, node := range p {
+		if node == "*" {
+			res = res.AdditionalProperties.(*Schema)
+			continue
+		}
+		var ok bool
+		res, ok = res.Properties[node]
+		if !ok {
+			return nil, fmt.Errorf("property %q not found in schema. Query path: %s", node, path)
+		}
+	}
+	return res, nil
+}
+
+func (s *Schema) GetByPath(path string) (Schema, error) {
+	v, err := s.getByPath(path)
+	if err != nil {
+		return Schema{}, err
+	}
+	return *v, nil
+}
+
+func (s *Schema) SetByPath(path string, v Schema) error {
+	dst, err := s.getByPath(path)
+	if err != nil {
+		return err
+	}
+	*dst = v
+	return nil
+}
+
 type Type string
 
 const (
@@ -97,7 +133,7 @@ const (
 func (schema *Schema) validateSchemaPropertyTypes() error {
 	for _, v := range schema.Properties {
 		switch v.Type {
-		case NumberType, BooleanType, StringType, IntegerType:
+		case NumberType, BooleanType, StringType, IntegerType, ObjectType, ArrayType:
 			continue
 		case "int", "int32", "int64":
 			return fmt.Errorf("type %s is not a recognized json schema type. Please use \"integer\" instead", v.Type)
diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go
index cf1f12767..c365cf235 100644
--- a/libs/jsonschema/schema_test.go
+++ b/libs/jsonschema/schema_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestSchemaValidateTypeNames(t *testing.T) {
@@ -305,3 +306,92 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) {
 	err = s.validate()
 	assert.NoError(t, err)
 }
+
+func testSchema() *Schema {
+	return &Schema{
+		Type: "object",
+		Properties: map[string]*Schema{
+			"int_val": {
+				Type:    "integer",
+				Default: int64(123),
+			},
+			"string_val": {
+				Type: "string",
+			},
+			"object_val": {
+				Type: "object",
+				Properties: map[string]*Schema{
+					"bar": {
+						Type:    "string",
+						Default: "baz",
+					},
+				},
+				AdditionalProperties: &Schema{
+					Type: "object",
+					Properties: map[string]*Schema{
+						"foo": {
+							Type:    "string",
+							Default: "zab",
+						},
+					},
+				},
+			},
+		},
+	}
+
+}
+
+func TestSchemaGetByPath(t *testing.T) {
+	s := testSchema()
+
+	ss, err := s.GetByPath("int_val")
+	require.NoError(t, err)
+	assert.Equal(t, Schema{
+		Type:    IntegerType,
+		Default: int64(123),
+	}, ss)
+
+	ss, err = s.GetByPath("string_val")
+	require.NoError(t, err)
+	assert.Equal(t, Schema{
+		Type: StringType,
+	}, ss)
+
+	ss, err = s.GetByPath("object_val.bar")
+	require.NoError(t, err)
+	assert.Equal(t, Schema{
+		Type:    StringType,
+		Default: "baz",
+	}, ss)
+
+	ss, err = s.GetByPath("object_val.*.foo")
+	require.NoError(t, err)
+	assert.Equal(t, Schema{
+		Type:    StringType,
+		Default: "zab",
+	}, ss)
+}
+
+func TestSchemaSetByPath(t *testing.T) {
+	s := testSchema()
+
+	err := s.SetByPath("int_val", Schema{
+		Type:    IntegerType,
+		Default: int64(456),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, int64(456), s.Properties["int_val"].Default)
+
+	err = s.SetByPath("object_val.*.foo", Schema{
+		Type:    StringType,
+		Default: "zooby",
+	})
+	require.NoError(t, err)
+
+	ns, err := s.GetByPath("object_val.*.foo")
+	require.NoError(t, err)
+	assert.Equal(t, Schema{
+		Type:    StringType,
+		Default: "zooby",
+	}, ns)
+}

From db84a707cd56ad0c04a14dfe21b940c8261154c1 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 25 Apr 2024 13:25:26 +0200
Subject: [PATCH 04/41] Fix bundle documentation URL (#1399)

Closes #1395.
---
 cmd/bundle/bundle.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go
index 1db60d585..0880c9c44 100644
--- a/cmd/bundle/bundle.go
+++ b/cmd/bundle/bundle.go
@@ -9,7 +9,7 @@ func New() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "bundle",
 		Short:   "Databricks Asset Bundles let you express data/AI/analytics projects as code.",
-		Long:    "Databricks Asset Bundles let you express data/AI/analytics projects as code.\n\nOnline documentation: https://docs.databricks.com/en/dev-tools/bundles",
+		Long:    "Databricks Asset Bundles let you express data/AI/analytics projects as code.\n\nOnline documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html",
 		GroupID: "development",
 	}
 

From a292eefc2edb2ae6a0b53068bfa4f07fb93b1075 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 25 Apr 2024 15:19:23 +0200
Subject: [PATCH 05/41] Release v0.218.1 (#1401)

This is a bugfix release.

CLI:
* Pass `DATABRICKS_CONFIG_FILE` for `auth profiles`
([#1394](https://github.com/databricks/cli/pull/1394)).

Bundles:
* Show a better error message for using wheel tasks with older DBR
versions ([#1373](https://github.com/databricks/cli/pull/1373)).
* Allow variable references in non-string fields in the JSON schema
([#1398](https://github.com/databricks/cli/pull/1398)).
* Fix variable overrides in targets for non-string variables
([#1397](https://github.com/databricks/cli/pull/1397)).
* Fix bundle schema for variables
([#1396](https://github.com/databricks/cli/pull/1396)).
* Fix bundle documentation URL
([#1399](https://github.com/databricks/cli/pull/1399)).

Internal:
* Removed autogenerated docs for the CLI commands
([#1392](https://github.com/databricks/cli/pull/1392)).
* Remove `JSON.parse` call from homebrew-tap action
([#1393](https://github.com/databricks/cli/pull/1393)).
* Ensure that Python dependencies are installed during upgrade
([#1390](https://github.com/databricks/cli/pull/1390)).
---
 CHANGELOG.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b74498ec..898f0df9d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,26 @@
 # Version changelog
 
+## 0.218.1
+
+This is a bugfix release.
+
+CLI:
+ * Pass `DATABRICKS_CONFIG_FILE` for `auth profiles` ([#1394](https://github.com/databricks/cli/pull/1394)).
+
+Bundles:
+ * Show a better error message for using wheel tasks with older DBR versions ([#1373](https://github.com/databricks/cli/pull/1373)).
+ * Allow variable references in non-string fields in the JSON schema ([#1398](https://github.com/databricks/cli/pull/1398)).
+ * Fix variable overrides in targets for non-string variables ([#1397](https://github.com/databricks/cli/pull/1397)).
+ * Fix bundle schema for variables ([#1396](https://github.com/databricks/cli/pull/1396)).
+ * Fix bundle documentation URL ([#1399](https://github.com/databricks/cli/pull/1399)).
+
+Internal:
+ * Removed autogenerated docs for the CLI commands ([#1392](https://github.com/databricks/cli/pull/1392)).
+ * Remove `JSON.parse` call from homebrew-tap action ([#1393](https://github.com/databricks/cli/pull/1393)).
+ * Ensure that Python dependencies are installed during upgrade ([#1390](https://github.com/databricks/cli/pull/1390)).
+
+
+
 ## 0.218.0
 
 This release marks the general availability of Databricks Asset Bundles.

From 781688c9cb7699d9c0e1977d2f77334381c65640 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 30 Apr 2024 16:41:24 +0200
Subject: [PATCH 06/41] Bump github.com/databricks/databricks-sdk-go from
 0.38.0 to 0.39.0 (#1405)

Bumps
[github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go)
from 0.38.0 to 0.39.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/releases">github.com/databricks/databricks-sdk-go's
releases</a>.</em></p>
<blockquote>
<h2>v0.39.0</h2>
<h2>0.39.0</h2>
<ul>
<li>Ignored flaky integration tests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/894">#894</a>).</li>
<li>Added retries for &quot;worker env WorkerEnvId(workerenv-XXXXX) not
found&quot; (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/890">#890</a>).</li>
<li>Updated SDK to OpenAPI spec (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/899">#899</a>).</li>
</ul>
<p>Note: This release contains breaking changes, please see the API
changes below for more details.</p>
<p>API Changes:</p>
<ul>
<li>Added <code>IngestionDefinition</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>,
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>
and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineSpec">pipelines.PipelineSpec</a>.</li>
<li>Added <code>Deployment</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>,
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>
and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineSpec">pipelines.PipelineSpec</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatus">compute.ClusterStatus</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusResponse">compute.ClusterStatusResponse</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryInstallStatus">compute.LibraryInstallStatus</a>.</li>
<li>Added <code>WarehouseId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#NotebookTask">jobs.NotebookTask</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#SubmitRun">jobs.SubmitRun</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#DeploymentKind">pipelines.DeploymentKind</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#IngestionConfig">pipelines.IngestionConfig</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#ManagedIngestionPipelineDefinition">pipelines.ManagedIngestionPipelineDefinition</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineDeployment">pipelines.PipelineDeployment</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#SchemaSpec">pipelines.SchemaSpec</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#TableSpec">pipelines.TableSpec</a>.</li>
<li>Added <code>GetOpenApi</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetOpenApiRequest">serving.GetOpenApiRequest</a>.</li>
<li>Added <code>SchemaId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#SchemaInfo">catalog.SchemaInfo</a>.</li>
<li>Added <code>Operation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultOperation">catalog.ValidationResultOperation</a>.</li>
<li>Added <code>Requirements</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#Library">compute.Library</a>.</li>
<li>Removed <code>AwsOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <code>AzureOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <code>GcpOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultAwsOperation">catalog.ValidationResultAwsOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultAzureOperation">catalog.ValidationResultAzureOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultGcpOperation">catalog.ValidationResultGcpOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusRequest">compute.ClusterStatusRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryFullStatusStatus">compute.LibraryFullStatusStatus</a>.</li>
<li>Changed <code>ClusterStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibrariesAPI">w.Libraries</a>
workspace-level service . New request type is <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatus">compute.ClusterStatus</a>.</li>
<li>Changed <code>ClusterStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibrariesAPI">w.Libraries</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusResponse">compute.ClusterStatusResponse</a>.</li>
<li>Changed <code>Status</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryFullStatus">compute.LibraryFullStatus</a>
to <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryInstallStatus">compute.LibraryInstallStatus</a>.</li>
</ul>
<p>OpenAPI SHA: 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55, Date:
2024-04-23</p>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/blob/main/CHANGELOG.md">github.com/databricks/databricks-sdk-go's
changelog</a>.</em></p>
<blockquote>
<h2>0.39.0</h2>
<ul>
<li>Ignored flaky integration tests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/894">#894</a>).</li>
<li>Added retries for &quot;worker env WorkerEnvId(workerenv-XXXXX) not
found&quot; (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/890">#890</a>).</li>
<li>Updated SDK to OpenAPI spec (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/899">#899</a>).</li>
</ul>
<p>Note: This release contains breaking changes, please see the API
changes below for more details.</p>
<p>API Changes:</p>
<ul>
<li>Added <code>IngestionDefinition</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>,
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>
and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineSpec">pipelines.PipelineSpec</a>.</li>
<li>Added <code>Deployment</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#CreatePipeline">pipelines.CreatePipeline</a>,
<a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#EditPipeline">pipelines.EditPipeline</a>
and <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineSpec">pipelines.PipelineSpec</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatus">compute.ClusterStatus</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusResponse">compute.ClusterStatusResponse</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryInstallStatus">compute.LibraryInstallStatus</a>.</li>
<li>Added <code>WarehouseId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#NotebookTask">jobs.NotebookTask</a>.</li>
<li>Added <code>RunAs</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/jobs#SubmitRun">jobs.SubmitRun</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#DeploymentKind">pipelines.DeploymentKind</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#IngestionConfig">pipelines.IngestionConfig</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#ManagedIngestionPipelineDefinition">pipelines.ManagedIngestionPipelineDefinition</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#PipelineDeployment">pipelines.PipelineDeployment</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#SchemaSpec">pipelines.SchemaSpec</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/pipelines#TableSpec">pipelines.TableSpec</a>.</li>
<li>Added <code>GetOpenApi</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ServingEndpointsAPI">w.ServingEndpoints</a>
workspace-level service.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetOpenApiRequest">serving.GetOpenApiRequest</a>.</li>
<li>Added <code>SchemaId</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#SchemaInfo">catalog.SchemaInfo</a>.</li>
<li>Added <code>Operation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultOperation">catalog.ValidationResultOperation</a>.</li>
<li>Added <code>Requirements</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#Library">compute.Library</a>.</li>
<li>Removed <code>AwsOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <code>AzureOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <code>GcpOperation</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResult">catalog.ValidationResult</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultAwsOperation">catalog.ValidationResultAwsOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultAzureOperation">catalog.ValidationResultAzureOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/catalog#ValidationResultGcpOperation">catalog.ValidationResultGcpOperation</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusRequest">compute.ClusterStatusRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryFullStatusStatus">compute.LibraryFullStatusStatus</a>.</li>
<li>Changed <code>ClusterStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibrariesAPI">w.Libraries</a>
workspace-level service . New request type is <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatus">compute.ClusterStatus</a>.</li>
<li>Changed <code>ClusterStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibrariesAPI">w.Libraries</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#ClusterStatusResponse">compute.ClusterStatusResponse</a>.</li>
<li>Changed <code>Status</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryFullStatus">compute.LibraryFullStatus</a>
to <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/compute#LibraryInstallStatus">compute.LibraryInstallStatus</a>.</li>
</ul>
<p>OpenAPI SHA: 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55, Date:
2024-04-23</p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/7672dece3899599a726b7e497d2727b7b74830d1"><code>7672dec</code></a>
Release v0.39.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/901">#901</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/2f56ab84318f28ddc284038887ee7546e1a199a3"><code>2f56ab8</code></a>
Update SDK to OpenAPI spec (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/899">#899</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/fa3a5d24eb47e9579b70530f789951db27d05ff5"><code>fa3a5d2</code></a>
Add retries for &quot;worker env WorkerEnvId(workerenv-XXXXX) not
found&quot; (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/890">#890</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/219975c53f8f1b1ca7a49c690df650e4b7116ed2"><code>219975c</code></a>
Ignore flaky integration tests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/894">#894</a>)</li>
<li>See full diff in <a
href="https://github.com/databricks/databricks-sdk-go/compare/v0.38.0...v0.39.0">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.38.0&new-version=0.39.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Andrew Nester <andrew.nester@databricks.com>
---
 .codegen/_openapi_sha                         |   2 +-
 .codegen/service.go.tmpl                      |   1 +
 bundle/schema/docs/bundle_descriptions.json   | 205 ++++++++++++++++--
 .../esm-enablement-account.go                 |   3 +
 .../automatic-cluster-update.go               |   3 +
 .../csp-enablement/csp-enablement.go          |   3 +
 .../esm-enablement/esm-enablement.go          |   3 +
 cmd/workspace/jobs/jobs.go                    |   1 +
 cmd/workspace/libraries/libraries.go          |  53 ++---
 cmd/workspace/pipelines/pipelines.go          |   2 +
 .../provider-exchanges/provider-exchanges.go  |  22 +-
 .../serving-endpoints/serving-endpoints.go    |  62 ++++++
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 14 files changed, 296 insertions(+), 70 deletions(-)

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 0aa4b1028..1f11c17bf 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-94684175b8bd65f8701f89729351f8069e8309c9
\ No newline at end of file
+21f9f1482f9d0d15228da59f2cd9f0863d2a6d55
\ No newline at end of file
diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 6aabb02c9..492b2132f 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -151,6 +151,7 @@ func new{{.PascalName}}() *cobra.Command {
 		"provider-exchanges delete"
 		"provider-exchanges delete-listing-from-exchange"
 		"provider-exchanges list-exchanges-for-listing"
+		"provider-exchanges list-listings-for-exchange"
 		-}}
 	{{- $fullCommandName := (print $serviceName " " .KebabName) -}}
 	{{- $noPrompt := or .IsCrudCreate (in $excludeFromPrompts $fullCommandName) }}
diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json
index ca889ae52..75499507d 100644
--- a/bundle/schema/docs/bundle_descriptions.json
+++ b/bundle/schema/docs/bundle_descriptions.json
@@ -46,6 +46,17 @@
           "properties": {
             "fail_on_active_runs": {
               "description": ""
+            },
+            "lock": {
+              "description": "",
+              "properties": {
+                "enabled": {
+                  "description": ""
+                },
+                "force": {
+                  "description": ""
+                }
+              }
             }
           }
         },
@@ -76,6 +87,9 @@
           "additionalproperties": {
             "description": ""
           }
+        },
+        "use_legacy_run_as": {
+          "description": ""
         }
       }
     },
@@ -242,7 +256,7 @@
                       "description": "",
                       "properties": {
                         "client": {
-                          "description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version"
+                          "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version."
                         },
                         "dependencies": {
                           "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]",
@@ -909,10 +923,10 @@
                             }
                           },
                           "egg": {
-                            "description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                            "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                           },
                           "jar": {
-                            "description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                            "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                           },
                           "maven": {
                             "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`",
@@ -942,8 +956,11 @@
                               }
                             }
                           },
+                          "requirements": {
+                            "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`"
+                          },
                           "whl": {
-                            "description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                            "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                           }
                         }
                       }
@@ -1303,6 +1320,9 @@
                         },
                         "source": {
                           "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                        },
+                        "warehouse_id": {
+                          "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail."
                         }
                       }
                     },
@@ -1526,7 +1546,7 @@
                           }
                         },
                         "file": {
-                          "description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.",
+                          "description": "If file, indicates that this job runs a SQL file in a remote Git repository.",
                           "properties": {
                             "path": {
                               "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
@@ -1562,7 +1582,7 @@
                       "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout."
                     },
                     "webhook_notifications": {
-                      "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
+                      "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
                       "properties": {
                         "on_duration_warning_threshold_exceeded": {
                           "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@@ -1679,7 +1699,7 @@
                 }
               },
               "webhook_notifications": {
-                "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
+                "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
                 "properties": {
                   "on_duration_warning_threshold_exceeded": {
                     "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@@ -2415,6 +2435,17 @@
               "continuous": {
                 "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`."
               },
+              "deployment": {
+                "description": "Deployment type of this pipeline.",
+                "properties": {
+                  "kind": {
+                    "description": "The deployment method that manages the pipeline."
+                  },
+                  "metadata_file_path": {
+                    "description": "The path to the file containing metadata about the deployment."
+                  }
+                }
+              },
               "development": {
                 "description": "Whether the pipeline is in Development mode. Defaults to false."
               },
@@ -2441,6 +2472,65 @@
               "id": {
                 "description": "Unique identifier for this pipeline."
               },
+              "ingestion_definition": {
+                "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.",
+                "properties": {
+                  "connection_name": {
+                    "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name."
+                  },
+                  "ingestion_gateway_id": {
+                    "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name."
+                  },
+                  "objects": {
+                    "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.",
+                    "items": {
+                      "description": "",
+                      "properties": {
+                        "schema": {
+                          "description": "Select tables from a specific source schema.",
+                          "properties": {
+                            "destination_catalog": {
+                              "description": "Required. Destination catalog to store tables."
+                            },
+                            "destination_schema": {
+                              "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists."
+                            },
+                            "source_catalog": {
+                              "description": "The source catalog name. Might be optional depending on the type of source."
+                            },
+                            "source_schema": {
+                              "description": "Required. Schema name in the source database."
+                            }
+                          }
+                        },
+                        "table": {
+                          "description": "Select tables from a specific source table.",
+                          "properties": {
+                            "destination_catalog": {
+                              "description": "Required. Destination catalog to store table."
+                            },
+                            "destination_schema": {
+                              "description": "Required. Destination schema to store table."
+                            },
+                            "destination_table": {
+                              "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used."
+                            },
+                            "source_catalog": {
+                              "description": "Source catalog name. Might be optional depending on the type of source."
+                            },
+                            "source_schema": {
+                              "description": "Schema name in the source database. Might be optional depending on the type of source."
+                            },
+                            "source_table": {
+                              "description": "Required. Table name in the source database."
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              },
               "libraries": {
                 "description": "Libraries or code needed by this deployment.",
                 "items": {
@@ -2682,6 +2772,17 @@
                 "properties": {
                   "fail_on_active_runs": {
                     "description": ""
+                  },
+                  "lock": {
+                    "description": "",
+                    "properties": {
+                      "enabled": {
+                        "description": ""
+                      },
+                      "force": {
+                        "description": ""
+                      }
+                    }
                   }
                 }
               },
@@ -2878,7 +2979,7 @@
                             "description": "",
                             "properties": {
                               "client": {
-                                "description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version"
+                                "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version."
                               },
                               "dependencies": {
                                 "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]",
@@ -3545,10 +3646,10 @@
                                   }
                                 },
                                 "egg": {
-                                  "description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                                  "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                                 },
                                 "jar": {
-                                  "description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                                  "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                                 },
                                 "maven": {
                                   "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`",
@@ -3578,8 +3679,11 @@
                                     }
                                   }
                                 },
+                                "requirements": {
+                                  "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`"
+                                },
                                 "whl": {
-                                  "description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
+                                  "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
                                 }
                               }
                             }
@@ -3939,6 +4043,9 @@
                               },
                               "source": {
                                 "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                              },
+                              "warehouse_id": {
+                                "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail."
                               }
                             }
                           },
@@ -4162,7 +4269,7 @@
                                 }
                               },
                               "file": {
-                                "description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.",
+                                "description": "If file, indicates that this job runs a SQL file in a remote Git repository.",
                                 "properties": {
                                   "path": {
                                     "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
@@ -4198,7 +4305,7 @@
                             "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout."
                           },
                           "webhook_notifications": {
-                            "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
+                            "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
                             "properties": {
                               "on_duration_warning_threshold_exceeded": {
                                 "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@@ -4315,7 +4422,7 @@
                       }
                     },
                     "webhook_notifications": {
-                      "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
+                      "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
                       "properties": {
                         "on_duration_warning_threshold_exceeded": {
                           "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@@ -5051,6 +5158,17 @@
                     "continuous": {
                       "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`."
                     },
+                    "deployment": {
+                      "description": "Deployment type of this pipeline.",
+                      "properties": {
+                        "kind": {
+                          "description": "The deployment method that manages the pipeline."
+                        },
+                        "metadata_file_path": {
+                          "description": "The path to the file containing metadata about the deployment."
+                        }
+                      }
+                    },
                     "development": {
                       "description": "Whether the pipeline is in Development mode. Defaults to false."
                     },
@@ -5077,6 +5195,65 @@
                     "id": {
                       "description": "Unique identifier for this pipeline."
                     },
+                    "ingestion_definition": {
+                      "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.",
+                      "properties": {
+                        "connection_name": {
+                          "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name."
+                        },
+                        "ingestion_gateway_id": {
+                          "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name."
+                        },
+                        "objects": {
+                          "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.",
+                          "items": {
+                            "description": "",
+                            "properties": {
+                              "schema": {
+                                "description": "Select tables from a specific source schema.",
+                                "properties": {
+                                  "destination_catalog": {
+                                    "description": "Required. Destination catalog to store tables."
+                                  },
+                                  "destination_schema": {
+                                    "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists."
+                                  },
+                                  "source_catalog": {
+                                    "description": "The source catalog name. Might be optional depending on the type of source."
+                                  },
+                                  "source_schema": {
+                                    "description": "Required. Schema name in the source database."
+                                  }
+                                }
+                              },
+                              "table": {
+                                "description": "Select tables from a specific source table.",
+                                "properties": {
+                                  "destination_catalog": {
+                                    "description": "Required. Destination catalog to store table."
+                                  },
+                                  "destination_schema": {
+                                    "description": "Required. Destination schema to store table."
+                                  },
+                                  "destination_table": {
+                                    "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used."
+                                  },
+                                  "source_catalog": {
+                                    "description": "Source catalog name. Might be optional depending on the type of source."
+                                  },
+                                  "source_schema": {
+                                    "description": "Schema name in the source database. Might be optional depending on the type of source."
+                                  },
+                                  "source_table": {
+                                    "description": "Required. Table name in the source database."
+                                  }
+                                }
+                              }
+                            }
+                          }
+                        }
+                      }
+                    },
                     "libraries": {
                       "description": "Libraries or code needed by this deployment.",
                       "items": {
diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go
index dd407e2e5..a2e95ffe1 100755
--- a/cmd/account/esm-enablement-account/esm-enablement-account.go
+++ b/cmd/account/esm-enablement-account/esm-enablement-account.go
@@ -25,6 +25,9 @@ func New() *cobra.Command {
   setting is disabled for new workspaces. After workspace creation, account
   admins can enable enhanced security monitoring individually for each
   workspace.`,
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
index 2385195bb..681dba7b3 100755
--- a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
+++ b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
@@ -22,6 +22,9 @@ func New() *cobra.Command {
 		Short: `Controls whether automatic cluster update is enabled for the current workspace.`,
 		Long: `Controls whether automatic cluster update is enabled for the current
   workspace. By default, it is turned off.`,
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/csp-enablement/csp-enablement.go b/cmd/workspace/csp-enablement/csp-enablement.go
index 312591564..e82fdc2a4 100755
--- a/cmd/workspace/csp-enablement/csp-enablement.go
+++ b/cmd/workspace/csp-enablement/csp-enablement.go
@@ -25,6 +25,9 @@ func New() *cobra.Command {
   off.
   
   This settings can NOT be disabled once it is enabled.`,
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/esm-enablement/esm-enablement.go b/cmd/workspace/esm-enablement/esm-enablement.go
index a65fe2f76..784c01f21 100755
--- a/cmd/workspace/esm-enablement/esm-enablement.go
+++ b/cmd/workspace/esm-enablement/esm-enablement.go
@@ -27,6 +27,9 @@ func New() *cobra.Command {
   
   If the compliance security profile is disabled, you can enable or disable this
   setting and it is not permanent.`,
+
+		// This service is being previewed; hide from help output.
+		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go
index 267dfc73b..e31c3f086 100755
--- a/cmd/workspace/jobs/jobs.go
+++ b/cmd/workspace/jobs/jobs.go
@@ -1513,6 +1513,7 @@ func newSubmit() *cobra.Command {
 	// TODO: complex arg: pipeline_task
 	// TODO: complex arg: python_wheel_task
 	// TODO: complex arg: queue
+	// TODO: complex arg: run_as
 	// TODO: complex arg: run_job_task
 	cmd.Flags().StringVar(&submitReq.RunName, "run-name", submitReq.RunName, `An optional name for the run.`)
 	// TODO: complex arg: spark_jar_task
diff --git a/cmd/workspace/libraries/libraries.go b/cmd/workspace/libraries/libraries.go
index e11e5a4c5..aed8843dc 100755
--- a/cmd/workspace/libraries/libraries.go
+++ b/cmd/workspace/libraries/libraries.go
@@ -25,18 +25,14 @@ func New() *cobra.Command {
   
   To make third-party or custom code available to notebooks and jobs running on
   your clusters, you can install a library. Libraries can be written in Python,
-  Java, Scala, and R. You can upload Java, Scala, and Python libraries and point
-  to external packages in PyPI, Maven, and CRAN repositories.
+  Java, Scala, and R. You can upload Python, Java, Scala and R libraries and
+  point to external packages in PyPI, Maven, and CRAN repositories.
   
   Cluster libraries can be used by all notebooks running on a cluster. You can
   install a cluster library directly from a public repository such as PyPI or
   Maven, using a previously installed workspace library, or using an init
   script.
   
-  When you install a library on a cluster, a notebook already attached to that
-  cluster will not immediately see the new library. You must first detach and
-  then reattach the notebook to the cluster.
-  
   When you uninstall a library from a cluster, the library is removed only when
   you restart the cluster. Until you restart the cluster, the status of the
   uninstalled library appears as Uninstall pending restart.`,
@@ -75,9 +71,8 @@ func newAllClusterStatuses() *cobra.Command {
 	cmd.Short = `Get all statuses.`
 	cmd.Long = `Get all statuses.
   
-  Get the status of all libraries on all clusters. A status will be available
-  for all libraries installed on this cluster via the API or the libraries UI as
-  well as libraries set to be installed on all clusters via the libraries UI.`
+  Get the status of all libraries on all clusters. A status is returned for all
+  libraries installed on this cluster via the API or the libraries UI.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -110,13 +105,13 @@ func newAllClusterStatuses() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var clusterStatusOverrides []func(
 	*cobra.Command,
-	*compute.ClusterStatusRequest,
+	*compute.ClusterStatus,
 )
 
 func newClusterStatus() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var clusterStatusReq compute.ClusterStatusRequest
+	var clusterStatusReq compute.ClusterStatus
 
 	// TODO: short flags
 
@@ -124,21 +119,13 @@ func newClusterStatus() *cobra.Command {
 	cmd.Short = `Get status.`
 	cmd.Long = `Get status.
   
-  Get the status of libraries on a cluster. A status will be available for all
-  libraries installed on this cluster via the API or the libraries UI as well as
-  libraries set to be installed on all clusters via the libraries UI. The order
-  of returned libraries will be as follows.
-  
-  1. Libraries set to be installed on this cluster will be returned first.
-  Within this group, the final order will be order in which the libraries were
-  added to the cluster.
-  
-  2. Libraries set to be installed on all clusters are returned next. Within
-  this group there is no order guarantee.
-  
-  3. Libraries that were previously requested on this cluster or on all
-  clusters, but now marked for removal. Within this group there is no order
-  guarantee.
+  Get the status of libraries on a cluster. A status is returned for all
+  libraries installed on this cluster via the API or the libraries UI. The order
+  of returned libraries is as follows: 1. Libraries set to be installed on this
+  cluster, in the order that the libraries were added to the cluster, are
+  returned first. 2. Libraries that were previously requested to be installed on
+  this cluster or, but are now marked for removal, in no particular order, are
+  returned last.
 
   Arguments:
     CLUSTER_ID: Unique identifier of the cluster whose status should be retrieved.`
@@ -195,12 +182,8 @@ func newInstall() *cobra.Command {
 	cmd.Short = `Add a library.`
 	cmd.Long = `Add a library.
   
-  Add libraries to be installed on a cluster. The installation is asynchronous;
-  it happens in the background after the completion of this request.
-  
-  **Note**: The actual set of libraries to be installed on a cluster is the
-  union of the libraries specified via this method and the libraries set to be
-  installed on all clusters via the libraries UI.`
+  Add libraries to install on a cluster. The installation is asynchronous; it
+  happens in the background after the completion of this request.`
 
 	cmd.Annotations = make(map[string]string)
 
@@ -259,9 +242,9 @@ func newUninstall() *cobra.Command {
 	cmd.Short = `Uninstall libraries.`
 	cmd.Long = `Uninstall libraries.
   
-  Set libraries to be uninstalled on a cluster. The libraries won't be
-  uninstalled until the cluster is restarted. Uninstalling libraries that are
-  not installed on the cluster will have no impact but is not an error.`
+  Set libraries to uninstall from a cluster. The libraries won't be uninstalled
+  until the cluster is restarted. A request to uninstall a library that is not
+  currently installed is ignored.`
 
 	cmd.Annotations = make(map[string]string)
 
diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go
index b7c3235f8..5a55fd72b 100755
--- a/cmd/workspace/pipelines/pipelines.go
+++ b/cmd/workspace/pipelines/pipelines.go
@@ -940,11 +940,13 @@ func newUpdate() *cobra.Command {
 	// TODO: array: clusters
 	// TODO: map via StringToStringVar: configuration
 	cmd.Flags().BoolVar(&updateReq.Continuous, "continuous", updateReq.Continuous, `Whether the pipeline is continuous or triggered.`)
+	// TODO: complex arg: deployment
 	cmd.Flags().BoolVar(&updateReq.Development, "development", updateReq.Development, `Whether the pipeline is in Development mode.`)
 	cmd.Flags().StringVar(&updateReq.Edition, "edition", updateReq.Edition, `Pipeline product edition.`)
 	cmd.Flags().Int64Var(&updateReq.ExpectedLastModified, "expected-last-modified", updateReq.ExpectedLastModified, `If present, the last-modified time of the pipeline settings before the edit.`)
 	// TODO: complex arg: filters
 	cmd.Flags().StringVar(&updateReq.Id, "id", updateReq.Id, `Unique identifier for this pipeline.`)
+	// TODO: complex arg: ingestion_definition
 	// TODO: array: libraries
 	cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `Friendly identifier for this pipeline.`)
 	// TODO: array: notifications
diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go
index fe1a9a3dc..c9f5818f5 100755
--- a/cmd/workspace/provider-exchanges/provider-exchanges.go
+++ b/cmd/workspace/provider-exchanges/provider-exchanges.go
@@ -508,28 +508,16 @@ func newListListingsForExchange() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		if len(args) == 0 {
-			promptSpinner := cmdio.Spinner(ctx)
-			promptSpinner <- "No EXCHANGE_ID argument specified. Loading names for Provider Exchanges drop-down."
-			names, err := w.ProviderExchanges.ExchangeListingExchangeNameToExchangeIdMap(ctx, marketplace.ListExchangesForListingRequest{})
-			close(promptSpinner)
-			if err != nil {
-				return fmt.Errorf("failed to load names for Provider Exchanges drop-down. Please manually specify required arguments. Original error: %w", err)
-			}
-			id, err := cmdio.Select(ctx, names, "")
-			if err != nil {
-				return err
-			}
-			args = append(args, id)
-		}
-		if len(args) != 1 {
-			return fmt.Errorf("expected to have ")
-		}
 		listListingsForExchangeReq.ExchangeId = args[0]
 
 		response := w.ProviderExchanges.ListListingsForExchange(ctx, listListingsForExchangeReq)
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index 6706b99ea..dee341ab4 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -46,6 +46,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(newDelete())
 	cmd.AddCommand(newExportMetrics())
 	cmd.AddCommand(newGet())
+	cmd.AddCommand(newGetOpenApi())
 	cmd.AddCommand(newGetPermissionLevels())
 	cmd.AddCommand(newGetPermissions())
 	cmd.AddCommand(newList())
@@ -379,6 +380,67 @@ func newGet() *cobra.Command {
 	return cmd
 }
 
+// start get-open-api command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getOpenApiOverrides []func(
+	*cobra.Command,
+	*serving.GetOpenApiRequest,
+)
+
+func newGetOpenApi() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getOpenApiReq serving.GetOpenApiRequest
+
+	// TODO: short flags
+
+	cmd.Use = "get-open-api NAME"
+	cmd.Short = `Get the schema for a serving endpoint.`
+	cmd.Long = `Get the schema for a serving endpoint.
+  
+  Get the query schema of the serving endpoint in OpenAPI format. The schema
+  contains information for the supported paths, input and output format and
+  datatypes.
+
+  Arguments:
+    NAME: The name of the serving endpoint that the served model belongs to. This
+      field is required.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		getOpenApiReq.Name = args[0]
+
+		err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getOpenApiOverrides {
+		fn(cmd, &getOpenApiReq)
+	}
+
+	return cmd
+}
+
 // start get-permission-levels command
 
 // Slice with functions to override default command behavior.
diff --git a/go.mod b/go.mod
index 6a991b0ec..7b2d31daa 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/semver/v3 v3.2.1 // MIT
 	github.com/briandowns/spinner v1.23.0 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.38.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.39.0 // Apache 2.0
 	github.com/fatih/color v1.16.0 // MIT
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
diff --git a/go.sum b/go.sum
index 8fe9109b5..5dc02d099 100644
--- a/go.sum
+++ b/go.sum
@@ -30,8 +30,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.38.0 h1:MQhOCWTkdKItG+n6ZwcXQv9FWBVXq9fax8VSZns2e+0=
-github.com/databricks/databricks-sdk-go v0.38.0/go.mod h1:Yjy1gREDLK65g4axpVbVNKYAHYE2Sqzj0AB9QWHCBVM=
+github.com/databricks/databricks-sdk-go v0.39.0 h1:nVnQYkk47SkEsRSXWkn6j7jBOxXgusjoo6xwbaHTGss=
+github.com/databricks/databricks-sdk-go v0.39.0/go.mod h1:Yjy1gREDLK65g4axpVbVNKYAHYE2Sqzj0AB9QWHCBVM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

From 153141d3eaab78c918a41ae950b3a4c2a24f109d Mon Sep 17 00:00:00 2001
From: Ilia Babanov <ilia.babanov@databricks.com>
Date: Wed, 1 May 2024 10:22:35 +0200
Subject: [PATCH 07/41] Don't fail while parsing outdated terraform state
 (#1404)

`terraform show -json` (`terraform.Show()`) fails if the state file
contains resources with fields that non longer conform to the provider
schemas.

This can happen when you deploy a bundle with one version of the CLI,
then updated the CLI to a version that uses different databricks
terraform provider, and try to run `bundle run` or `bundle summary`.
Those commands don't recreate local terraform state (only `terraform
apply` or `plan` do) and terraform itself fails while parsing it.
[Terraform
docs](https://developer.hashicorp.com/terraform/language/state#format)
point out that it's best to use `terraform show` after successful
`apply` or `plan`.

Here we parse the state ourselves. The state file format is internal to
terraform, but it's more stable than our resource schemas. We only parse
a subset of fields from the state, and only update ID and ModifiedStatus
of bundle resources in the `terraform.Load` mutator.
---
 bundle/deploy/check_running_resources.go      |  92 ++----
 bundle/deploy/check_running_resources_test.go |  64 +---
 bundle/deploy/terraform/convert.go            |  88 +++---
 bundle/deploy/terraform/convert_test.go       | 293 +++++++++++-------
 bundle/deploy/terraform/load.go               |  16 +-
 bundle/deploy/terraform/util.go               |  55 +++-
 bundle/deploy/terraform/util_test.go          |  99 ++++++
 bundle/phases/deploy.go                       |   1 +
 8 files changed, 423 insertions(+), 285 deletions(-)

diff --git a/bundle/deploy/check_running_resources.go b/bundle/deploy/check_running_resources.go
index 7f7a9bcac..a2305cd75 100644
--- a/bundle/deploy/check_running_resources.go
+++ b/bundle/deploy/check_running_resources.go
@@ -6,12 +6,11 @@ import (
 	"strconv"
 
 	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
-	"github.com/hashicorp/terraform-exec/tfexec"
-	tfjson "github.com/hashicorp/terraform-json"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -35,27 +34,11 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia
 	if !b.Config.Bundle.Deployment.FailOnActiveRuns {
 		return nil
 	}
-
-	tf := b.Terraform
-	if tf == nil {
-		return diag.Errorf("terraform not initialized")
-	}
-
-	err := tf.Init(ctx, tfexec.Upgrade(true))
-	if err != nil {
-		return diag.Errorf("terraform init: %v", err)
-	}
-
-	state, err := b.Terraform.Show(ctx)
+	w := b.WorkspaceClient()
+	err := checkAnyResourceRunning(ctx, w, &b.Config.Resources)
 	if err != nil {
 		return diag.FromErr(err)
 	}
-
-	err = checkAnyResourceRunning(ctx, b.WorkspaceClient(), state)
-	if err != nil {
-		return diag.Errorf("deployment aborted, err: %v", err)
-	}
-
 	return nil
 }
 
@@ -63,54 +46,43 @@ func CheckRunningResource() *checkRunningResources {
 	return &checkRunningResources{}
 }
 
-func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *tfjson.State) error {
-	if state.Values == nil || state.Values.RootModule == nil {
-		return nil
-	}
-
+func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, resources *config.Resources) error {
 	errs, errCtx := errgroup.WithContext(ctx)
 
-	for _, resource := range state.Values.RootModule.Resources {
-		// Limit to resources.
-		if resource.Mode != tfjson.ManagedResourceMode {
+	for _, job := range resources.Jobs {
+		id := job.ID
+		if id == "" {
 			continue
 		}
+		errs.Go(func() error {
+			isRunning, err := IsJobRunning(errCtx, w, id)
+			// If there's an error retrieving the job, we assume it's not running
+			if err != nil {
+				return err
+			}
+			if isRunning {
+				return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
+			}
+			return nil
+		})
+	}
 
-		value, ok := resource.AttributeValues["id"]
-		if !ok {
+	for _, pipeline := range resources.Pipelines {
+		id := pipeline.ID
+		if id == "" {
 			continue
 		}
-		id, ok := value.(string)
-		if !ok {
-			continue
-		}
-
-		switch resource.Type {
-		case "databricks_job":
-			errs.Go(func() error {
-				isRunning, err := IsJobRunning(errCtx, w, id)
-				// If there's an error retrieving the job, we assume it's not running
-				if err != nil {
-					return err
-				}
-				if isRunning {
-					return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
-				}
+		errs.Go(func() error {
+			isRunning, err := IsPipelineRunning(errCtx, w, id)
+			// If there's an error retrieving the pipeline, we assume it's not running
+			if err != nil {
 				return nil
-			})
-		case "databricks_pipeline":
-			errs.Go(func() error {
-				isRunning, err := IsPipelineRunning(errCtx, w, id)
-				// If there's an error retrieving the pipeline, we assume it's not running
-				if err != nil {
-					return nil
-				}
-				if isRunning {
-					return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
-				}
-				return nil
-			})
-		}
+			}
+			if isRunning {
+				return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
+			}
+			return nil
+		})
 	}
 
 	return errs.Wait()
diff --git a/bundle/deploy/check_running_resources_test.go b/bundle/deploy/check_running_resources_test.go
index 7dc1fb865..d61c80fc4 100644
--- a/bundle/deploy/check_running_resources_test.go
+++ b/bundle/deploy/check_running_resources_test.go
@@ -5,36 +5,26 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
-	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
 func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
 	mock := mocks.NewMockWorkspaceClient(t)
-	state := &tfjson.State{}
-	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, state)
+	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &config.Resources{})
 	require.NoError(t, err)
 }
 
 func TestIsAnyResourceRunningWithJob(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	state := &tfjson.State{
-		Values: &tfjson.StateValues{
-			RootModule: &tfjson.StateModule{
-				Resources: []*tfjson.StateResource{
-					{
-						Type: "databricks_job",
-						AttributeValues: map[string]interface{}{
-							"id": "123",
-						},
-						Mode: tfjson.ManagedResourceMode,
-					},
-				},
-			},
+	resources := &config.Resources{
+		Jobs: map[string]*resources.Job{
+			"job1": {ID: "123"},
 		},
 	}
 
@@ -46,7 +36,7 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
 		{RunId: 1234},
 	}, nil).Once()
 
-	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
+	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
 	require.ErrorContains(t, err, "job 123 is running")
 
 	jobsApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{
@@ -54,25 +44,15 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
 		ActiveOnly: true,
 	}).Return([]jobs.BaseRun{}, nil).Once()
 
-	err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
+	err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
 	require.NoError(t, err)
 }
 
 func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	state := &tfjson.State{
-		Values: &tfjson.StateValues{
-			RootModule: &tfjson.StateModule{
-				Resources: []*tfjson.StateResource{
-					{
-						Type: "databricks_pipeline",
-						AttributeValues: map[string]interface{}{
-							"id": "123",
-						},
-						Mode: tfjson.ManagedResourceMode,
-					},
-				},
-			},
+	resources := &config.Resources{
+		Pipelines: map[string]*resources.Pipeline{
+			"pipeline1": {ID: "123"},
 		},
 	}
 
@@ -84,7 +64,7 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 		State:      pipelines.PipelineStateRunning,
 	}, nil).Once()
 
-	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
+	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
 	require.ErrorContains(t, err, "pipeline 123 is running")
 
 	pipelineApi.EXPECT().Get(mock.Anything, pipelines.GetPipelineRequest{
@@ -93,25 +73,15 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 		PipelineId: "123",
 		State:      pipelines.PipelineStateIdle,
 	}, nil).Once()
-	err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
+	err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
 	require.NoError(t, err)
 }
 
 func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	state := &tfjson.State{
-		Values: &tfjson.StateValues{
-			RootModule: &tfjson.StateModule{
-				Resources: []*tfjson.StateResource{
-					{
-						Type: "databricks_pipeline",
-						AttributeValues: map[string]interface{}{
-							"id": "123",
-						},
-						Mode: tfjson.ManagedResourceMode,
-					},
-				},
-			},
+	resources := &config.Resources{
+		Pipelines: map[string]*resources.Pipeline{
+			"pipeline1": {ID: "123"},
 		},
 	}
 
@@ -120,6 +90,6 @@ func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
 		PipelineId: "123",
 	}).Return(nil, errors.New("API failure")).Once()
 
-	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
+	err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
 	require.NoError(t, err)
 }
diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go
index 0ae6751d0..d0b633582 100644
--- a/bundle/deploy/terraform/convert.go
+++ b/bundle/deploy/terraform/convert.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"reflect"
 
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
@@ -19,15 +18,6 @@ func conv(from any, to any) {
 	json.Unmarshal(buf, &to)
 }
 
-func convRemoteToLocal(remote any, local any) resources.ModifiedStatus {
-	var modifiedStatus resources.ModifiedStatus
-	if reflect.ValueOf(local).Elem().IsNil() {
-		modifiedStatus = resources.ModifiedStatusDeleted
-	}
-	conv(remote, local)
-	return modifiedStatus
-}
-
 func convPermissions(acl []resources.Permission) *schema.ResourcePermissions {
 	if len(acl) == 0 {
 		return nil
@@ -248,7 +238,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
 	tfroot.Provider = schema.NewProviders()
 
 	// Convert each resource in the bundle to the equivalent Terraform representation.
-	resources, err := dyn.Get(root, "resources")
+	dynResources, err := dyn.Get(root, "resources")
 	if err != nil {
 		// If the resources key is missing, return an empty root.
 		if dyn.IsNoSuchKeyError(err) {
@@ -260,11 +250,20 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
 	tfroot.Resource = schema.NewResources()
 
 	numResources := 0
-	_, err = dyn.Walk(resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+	_, err = dyn.Walk(dynResources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
 		if len(p) < 2 {
 			return v, nil
 		}
 
+		// Skip resources that have been deleted locally.
+		modifiedStatus, err := dyn.Get(v, "modified_status")
+		if err == nil {
+			modifiedStatusStr, ok := modifiedStatus.AsString()
+			if ok && modifiedStatusStr == resources.ModifiedStatusDeleted {
+				return v, dyn.ErrSkip
+			}
+		}
+
 		typ := p[0].Key()
 		key := p[1].Key()
 
@@ -275,7 +274,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
 		}
 
 		// Convert resource to Terraform representation.
-		err := c.Convert(ctx, key, v, tfroot.Resource)
+		err = c.Convert(ctx, key, v, tfroot.Resource)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
@@ -299,75 +298,72 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
 	return tfroot, nil
 }
 
-func TerraformToBundle(state *tfjson.State, config *config.Root) error {
-	if state.Values != nil && state.Values.RootModule != nil {
-		for _, resource := range state.Values.RootModule.Resources {
-			// Limit to resources.
-			if resource.Mode != tfjson.ManagedResourceMode {
-				continue
-			}
-
+func TerraformToBundle(state *resourcesState, config *config.Root) error {
+	for _, resource := range state.Resources {
+		if resource.Mode != tfjson.ManagedResourceMode {
+			continue
+		}
+		for _, instance := range resource.Instances {
 			switch resource.Type {
 			case "databricks_job":
-				var tmp schema.ResourceJob
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.Jobs == nil {
 					config.Resources.Jobs = make(map[string]*resources.Job)
 				}
 				cur := config.Resources.Jobs[resource.Name]
-				// TODO: make sure we can unmarshall tf state properly and don't swallow errors
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.Job{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.Jobs[resource.Name] = cur
 			case "databricks_pipeline":
-				var tmp schema.ResourcePipeline
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.Pipelines == nil {
 					config.Resources.Pipelines = make(map[string]*resources.Pipeline)
 				}
 				cur := config.Resources.Pipelines[resource.Name]
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.Pipeline{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.Pipelines[resource.Name] = cur
 			case "databricks_mlflow_model":
-				var tmp schema.ResourceMlflowModel
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.Models == nil {
 					config.Resources.Models = make(map[string]*resources.MlflowModel)
 				}
 				cur := config.Resources.Models[resource.Name]
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.MlflowModel{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.Models[resource.Name] = cur
 			case "databricks_mlflow_experiment":
-				var tmp schema.ResourceMlflowExperiment
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.Experiments == nil {
 					config.Resources.Experiments = make(map[string]*resources.MlflowExperiment)
 				}
 				cur := config.Resources.Experiments[resource.Name]
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.MlflowExperiment{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.Experiments[resource.Name] = cur
 			case "databricks_model_serving":
-				var tmp schema.ResourceModelServing
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.ModelServingEndpoints == nil {
 					config.Resources.ModelServingEndpoints = make(map[string]*resources.ModelServingEndpoint)
 				}
 				cur := config.Resources.ModelServingEndpoints[resource.Name]
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.ModelServingEndpoint{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.ModelServingEndpoints[resource.Name] = cur
 			case "databricks_registered_model":
-				var tmp schema.ResourceRegisteredModel
-				conv(resource.AttributeValues, &tmp)
 				if config.Resources.RegisteredModels == nil {
 					config.Resources.RegisteredModels = make(map[string]*resources.RegisteredModel)
 				}
 				cur := config.Resources.RegisteredModels[resource.Name]
-				modifiedStatus := convRemoteToLocal(tmp, &cur)
-				cur.ModifiedStatus = modifiedStatus
+				if cur == nil {
+					cur = &resources.RegisteredModel{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
 				config.Resources.RegisteredModels[resource.Name] = cur
 			case "databricks_permissions":
 			case "databricks_grants":
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index 986599a79..58523bb49 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -17,7 +17,6 @@ import (
 	"github.com/databricks/databricks-sdk-go/service/ml"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/databricks/databricks-sdk-go/service/serving"
-	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -548,50 +547,86 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) {
 	bundleToTerraformEquivalenceTest(t, &config)
 }
 
+func TestBundleToTerraformDeletedResources(t *testing.T) {
+	var job1 = resources.Job{
+		JobSettings: &jobs.JobSettings{},
+	}
+	var job2 = resources.Job{
+		ModifiedStatus: resources.ModifiedStatusDeleted,
+		JobSettings:    &jobs.JobSettings{},
+	}
+	var config = config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*resources.Job{
+				"my_job1": &job1,
+				"my_job2": &job2,
+			},
+		},
+	}
+
+	vin, err := convert.FromTyped(config, dyn.NilValue)
+	require.NoError(t, err)
+	out, err := BundleToTerraformWithDynValue(context.Background(), vin)
+	require.NoError(t, err)
+
+	_, ok := out.Resource.Job["my_job1"]
+	assert.True(t, ok)
+	_, ok = out.Resource.Job["my_job2"]
+	assert.False(t, ok)
+}
+
 func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 	var config = config.Root{
 		Resources: config.Resources{},
 	}
-	var tfState = tfjson.State{
-		Values: &tfjson.StateValues{
-			RootModule: &tfjson.StateModule{
-				Resources: []*tfjson.StateResource{
-					{
-						Type:            "databricks_job",
-						Mode:            "managed",
-						Name:            "test_job",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_pipeline",
-						Mode:            "managed",
-						Name:            "test_pipeline",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_mlflow_model",
-						Mode:            "managed",
-						Name:            "test_mlflow_model",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_mlflow_experiment",
-						Mode:            "managed",
-						Name:            "test_mlflow_experiment",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_model_serving",
-						Mode:            "managed",
-						Name:            "test_model_serving",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_registered_model",
-						Mode:            "managed",
-						Name:            "test_registered_model",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
+	var tfState = resourcesState{
+		Resources: []stateResource{
+			{
+				Type: "databricks_job",
+				Mode: "managed",
+				Name: "test_job",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_pipeline",
+				Mode: "managed",
+				Name: "test_pipeline",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_model",
+				Mode: "managed",
+				Name: "test_mlflow_model",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_experiment",
+				Mode: "managed",
+				Name: "test_mlflow_experiment",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_model_serving",
+				Mode: "managed",
+				Name: "test_model_serving",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_registered_model",
+				Mode: "managed",
+				Name: "test_registered_model",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
 				},
 			},
 		},
@@ -667,8 +702,8 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 			},
 		},
 	}
-	var tfState = tfjson.State{
-		Values: nil,
+	var tfState = resourcesState{
+		Resources: nil,
 	}
 	err := TerraformToBundle(&tfState, &config)
 	assert.NoError(t, err)
@@ -771,82 +806,102 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 			},
 		},
 	}
-	var tfState = tfjson.State{
-		Values: &tfjson.StateValues{
-			RootModule: &tfjson.StateModule{
-				Resources: []*tfjson.StateResource{
-					{
-						Type:            "databricks_job",
-						Mode:            "managed",
-						Name:            "test_job",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_job",
-						Mode:            "managed",
-						Name:            "test_job_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
-					{
-						Type:            "databricks_pipeline",
-						Mode:            "managed",
-						Name:            "test_pipeline",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_pipeline",
-						Mode:            "managed",
-						Name:            "test_pipeline_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
-					{
-						Type:            "databricks_mlflow_model",
-						Mode:            "managed",
-						Name:            "test_mlflow_model",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_mlflow_model",
-						Mode:            "managed",
-						Name:            "test_mlflow_model_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
-					{
-						Type:            "databricks_mlflow_experiment",
-						Mode:            "managed",
-						Name:            "test_mlflow_experiment",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_mlflow_experiment",
-						Mode:            "managed",
-						Name:            "test_mlflow_experiment_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
-					{
-						Type:            "databricks_model_serving",
-						Mode:            "managed",
-						Name:            "test_model_serving",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_model_serving",
-						Mode:            "managed",
-						Name:            "test_model_serving_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
-					{
-						Type:            "databricks_registered_model",
-						Mode:            "managed",
-						Name:            "test_registered_model",
-						AttributeValues: map[string]interface{}{"id": "1"},
-					},
-					{
-						Type:            "databricks_registered_model",
-						Mode:            "managed",
-						Name:            "test_registered_model_old",
-						AttributeValues: map[string]interface{}{"id": "2"},
-					},
+	var tfState = resourcesState{
+		Resources: []stateResource{
+			{
+				Type: "databricks_job",
+				Mode: "managed",
+				Name: "test_job",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_job",
+				Mode: "managed",
+				Name: "test_job_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
+			{
+				Type: "databricks_pipeline",
+				Mode: "managed",
+				Name: "test_pipeline",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_pipeline",
+				Mode: "managed",
+				Name: "test_pipeline_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_model",
+				Mode: "managed",
+				Name: "test_mlflow_model",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_model",
+				Mode: "managed",
+				Name: "test_mlflow_model_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_experiment",
+				Mode: "managed",
+				Name: "test_mlflow_experiment",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_mlflow_experiment",
+				Mode: "managed",
+				Name: "test_mlflow_experiment_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
+			{
+				Type: "databricks_model_serving",
+				Mode: "managed",
+				Name: "test_model_serving",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_model_serving",
+				Mode: "managed",
+				Name: "test_model_serving_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
+				},
+			},
+			{
+				Type: "databricks_registered_model",
+				Mode: "managed",
+				Name: "test_registered_model",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
+			{
+				Type: "databricks_registered_model",
+				Mode: "managed",
+				Name: "test_registered_model_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "2"}},
 				},
 			},
 		},
diff --git a/bundle/deploy/terraform/load.go b/bundle/deploy/terraform/load.go
index fa0cd5b4f..3fb76855e 100644
--- a/bundle/deploy/terraform/load.go
+++ b/bundle/deploy/terraform/load.go
@@ -8,7 +8,6 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/hashicorp/terraform-exec/tfexec"
-	tfjson "github.com/hashicorp/terraform-json"
 )
 
 type loadMode int
@@ -34,7 +33,7 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diag.Errorf("terraform init: %v", err)
 	}
 
-	state, err := b.Terraform.Show(ctx)
+	state, err := ParseResourcesState(ctx, b)
 	if err != nil {
 		return diag.FromErr(err)
 	}
@@ -53,16 +52,13 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	return nil
 }
 
-func (l *load) validateState(state *tfjson.State) error {
-	if state.Values == nil {
-		if slices.Contains(l.modes, ErrorOnEmptyState) {
-			return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
-		}
-		return nil
+func (l *load) validateState(state *resourcesState) error {
+	if state.Version != SupportedStateVersion {
+		return fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version)
 	}
 
-	if state.Values.RootModule == nil {
-		return fmt.Errorf("malformed terraform state: RootModule not set")
+	if len(state.Resources) == 0 && slices.Contains(l.modes, ErrorOnEmptyState) {
+		return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
 	}
 
 	return nil
diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go
index a5978b397..1a8a83ac7 100644
--- a/bundle/deploy/terraform/util.go
+++ b/bundle/deploy/terraform/util.go
@@ -1,14 +1,46 @@
 package terraform
 
 import (
+	"context"
 	"encoding/json"
+	"errors"
 	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	tfjson "github.com/hashicorp/terraform-json"
 )
 
-type state struct {
+// Partial representation of the Terraform state file format.
+// We are only interested global version and serial numbers,
+// plus resource types, names, modes, and ids.
+type resourcesState struct {
+	Version   int             `json:"version"`
+	Resources []stateResource `json:"resources"`
+}
+
+const SupportedStateVersion = 4
+
+type serialState struct {
 	Serial int `json:"serial"`
 }
 
+type stateResource struct {
+	Type      string                  `json:"type"`
+	Name      string                  `json:"name"`
+	Mode      tfjson.ResourceMode     `json:"mode"`
+	Instances []stateResourceInstance `json:"instances"`
+}
+
+type stateResourceInstance struct {
+	Attributes stateInstanceAttributes `json:"attributes"`
+}
+
+type stateInstanceAttributes struct {
+	ID string `json:"id"`
+}
+
 func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
 	localState, err := loadState(local)
 	if err != nil {
@@ -23,12 +55,12 @@ func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
 	return localState.Serial < remoteState.Serial
 }
 
-func loadState(input io.Reader) (*state, error) {
+func loadState(input io.Reader) (*serialState, error) {
 	content, err := io.ReadAll(input)
 	if err != nil {
 		return nil, err
 	}
-	var s state
+	var s serialState
 	err = json.Unmarshal(content, &s)
 	if err != nil {
 		return nil, err
@@ -36,3 +68,20 @@ func loadState(input io.Reader) (*state, error) {
 
 	return &s, nil
 }
+
+func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) {
+	cacheDir, err := Dir(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+	rawState, err := os.ReadFile(filepath.Join(cacheDir, TerraformStateFileName))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return &resourcesState{Version: SupportedStateVersion}, nil
+		}
+		return nil, err
+	}
+	var state resourcesState
+	err = json.Unmarshal(rawState, &state)
+	return &state, err
+}
diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go
index 4f2cf2918..8949ebca8 100644
--- a/bundle/deploy/terraform/util_test.go
+++ b/bundle/deploy/terraform/util_test.go
@@ -1,11 +1,16 @@
 package terraform
 
 import (
+	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"testing/iotest"
 
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -38,3 +43,97 @@ func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) {
 	remote := iotest.ErrReader(fmt.Errorf("Random error"))
 	assert.False(t, IsLocalStateStale(local, remote))
 }
+
+func TestParseResourcesStateWithNoFile(t *testing.T) {
+	b := &bundle.Bundle{
+		RootPath: t.TempDir(),
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Target: "whatever",
+				Terraform: &config.Terraform{
+					ExecPath: "terraform",
+				},
+			},
+		},
+	}
+	state, err := ParseResourcesState(context.Background(), b)
+	assert.NoError(t, err)
+	assert.Equal(t, &resourcesState{Version: SupportedStateVersion}, state)
+}
+
+func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
+	ctx := context.Background()
+	b := &bundle.Bundle{
+		RootPath: t.TempDir(),
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Target: "whatever",
+				Terraform: &config.Terraform{
+					ExecPath: "terraform",
+				},
+			},
+		},
+	}
+	cacheDir, err := Dir(ctx, b)
+	assert.NoError(t, err)
+	data := []byte(`{
+		"version": 4,
+		"unknown_field": "hello",
+		"resources": [
+		{
+			"mode": "managed",
+			"type": "databricks_pipeline",
+			"name": "test_pipeline",
+			"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+			"instances": [
+			  {
+				"schema_version": 0,
+				"attributes": {
+				  "allow_duplicate_names": false,
+				  "catalog": null,
+				  "channel": "CURRENT",
+				  "cluster": [],
+				  "random_field": "random_value",
+				  "configuration": {
+					"bundle.sourcePath": "/Workspace//Users/user/.bundle/test/dev/files/src"
+				  },
+				  "continuous": false,
+				  "development": true,
+				  "edition": "ADVANCED",
+				  "filters": [],
+				  "id": "123",
+				  "library": [],
+				  "name": "test_pipeline",
+				  "notification": [],
+				  "photon": false,
+				  "serverless": false,
+				  "storage": "dbfs:/123456",
+				  "target": "test_dev",
+				  "timeouts": null,
+				  "url": "https://test.com"
+				},
+				"sensitive_attributes": []
+			  }
+			]
+		  }
+		]
+	}`)
+	err = os.WriteFile(filepath.Join(cacheDir, TerraformStateFileName), data, os.ModePerm)
+	assert.NoError(t, err)
+	state, err := ParseResourcesState(ctx, b)
+	assert.NoError(t, err)
+	expected := &resourcesState{
+		Version: 4,
+		Resources: []stateResource{
+			{
+				Mode: "managed",
+				Type: "databricks_pipeline",
+				Name: "test_pipeline",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "123"}},
+				},
+			},
+		},
+	}
+	assert.Equal(t, expected, state)
+}
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index fce98b038..4fc4f6300 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -36,6 +36,7 @@ func Deploy() bundle.Mutator {
 				permissions.ApplyWorkspaceRootPermissions(),
 				terraform.Interpolate(),
 				terraform.Write(),
+				terraform.Load(),
 				deploy.CheckRunningResource(),
 				bundle.Defer(
 					terraform.Apply(),

From 507053ee50563bae098557246b4415b1b272d6ec Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 1 May 2024 14:07:03 +0530
Subject: [PATCH 08/41] Annotate DLT pipelines when deployed using DABs (#1410)

## Changes
This PR annotates any pipelines that were deployed using DABs to have
`deployment.kind` set to "BUNDLE", mirroring the annotation for Jobs
(similar PR for jobs FYI: https://github.com/databricks/cli/pull/880).

Breakglass UI is not yet available for pipelines, so this annotation
will just be used for revenue attribution ATM.

Note: The API field has been deployed in all regions including GovCloud.

## Tests
Unit tests and manually.

Manually verified that the kind and metadata_file_path are being set by
DABs, and are returned by a GET API to a pipeline deployed using a DAB.
Example:
```
    "deployment": {
      "kind":"BUNDLE",
      "metadata_file_path":"/Users/shreyas.goenka@databricks.com/.bundle/bundle-playground/default/state/metadata.json"
    },
```
---
 bundle/deploy/metadata/annotate_jobs.go       |  3 +-
 bundle/deploy/metadata/annotate_pipelines.go  | 34 +++++++++
 .../metadata/annotate_pipelines_test.go       | 72 +++++++++++++++++++
 bundle/deploy/metadata/upload.go              |  9 ++-
 bundle/phases/initialize.go                   |  1 +
 5 files changed, 115 insertions(+), 4 deletions(-)
 create mode 100644 bundle/deploy/metadata/annotate_pipelines.go
 create mode 100644 bundle/deploy/metadata/annotate_pipelines_test.go

diff --git a/bundle/deploy/metadata/annotate_jobs.go b/bundle/deploy/metadata/annotate_jobs.go
index 2b03a59b7..f42d46931 100644
--- a/bundle/deploy/metadata/annotate_jobs.go
+++ b/bundle/deploy/metadata/annotate_jobs.go
@@ -2,7 +2,6 @@ package metadata
 
 import (
 	"context"
-	"path"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
@@ -27,7 +26,7 @@ func (m *annotateJobs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
 
 		job.JobSettings.Deployment = &jobs.JobDeployment{
 			Kind:             jobs.JobDeploymentKindBundle,
-			MetadataFilePath: path.Join(b.Config.Workspace.StatePath, MetadataFileName),
+			MetadataFilePath: metadataFilePath(b),
 		}
 		job.JobSettings.EditMode = jobs.JobEditModeUiLocked
 		job.JobSettings.Format = jobs.FormatMultiTask
diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go
new file mode 100644
index 000000000..990f48907
--- /dev/null
+++ b/bundle/deploy/metadata/annotate_pipelines.go
@@ -0,0 +1,34 @@
+package metadata
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+)
+
+type annotatePipelines struct{}
+
+func AnnotatePipelines() bundle.Mutator {
+	return &annotatePipelines{}
+}
+
+func (m *annotatePipelines) Name() string {
+	return "metadata.AnnotatePipelines"
+}
+
+func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
+	for _, pipeline := range b.Config.Resources.Pipelines {
+		if pipeline.PipelineSpec == nil {
+			continue
+		}
+
+		pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{
+			Kind:             pipelines.DeploymentKindBundle,
+			MetadataFilePath: metadataFilePath(b),
+		}
+	}
+
+	return nil
+}
diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go
new file mode 100644
index 000000000..448a022d0
--- /dev/null
+++ b/bundle/deploy/metadata/annotate_pipelines_test.go
@@ -0,0 +1,72 @@
+package metadata
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAnnotatePipelinesMutator(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				StatePath: "/a/b/c",
+			},
+			Resources: config.Resources{
+				Pipelines: map[string]*resources.Pipeline{
+					"my-pipeline-1": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Name: "My Pipeline One",
+						},
+					},
+					"my-pipeline-2": {
+						PipelineSpec: &pipelines.PipelineSpec{
+							Name: "My Pipeline Two",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, AnnotatePipelines())
+	require.NoError(t, diags.Error())
+
+	assert.Equal(t,
+		&pipelines.PipelineDeployment{
+			Kind:             pipelines.DeploymentKindBundle,
+			MetadataFilePath: "/a/b/c/metadata.json",
+		},
+		b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment)
+
+	assert.Equal(t,
+		&pipelines.PipelineDeployment{
+			Kind:             pipelines.DeploymentKindBundle,
+			MetadataFilePath: "/a/b/c/metadata.json",
+		},
+		b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment)
+}
+
+func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				StatePath: "/a/b/c",
+			},
+			Resources: config.Resources{
+				Pipelines: map[string]*resources.Pipeline{
+					"my-pipeline-1": {},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, AnnotatePipelines())
+	require.NoError(t, diags.Error())
+}
diff --git a/bundle/deploy/metadata/upload.go b/bundle/deploy/metadata/upload.go
index a040a0ae8..ee87816de 100644
--- a/bundle/deploy/metadata/upload.go
+++ b/bundle/deploy/metadata/upload.go
@@ -4,13 +4,18 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"path"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/filer"
 )
 
-const MetadataFileName = "metadata.json"
+const metadataFileName = "metadata.json"
+
+func metadataFilePath(b *bundle.Bundle) string {
+	return path.Join(b.Config.Workspace.StatePath, metadataFileName)
+}
 
 type upload struct{}
 
@@ -33,5 +38,5 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diag.FromErr(err)
 	}
 
-	return diag.FromErr(f.Write(ctx, MetadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists))
+	return diag.FromErr(f.Write(ctx, metadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists))
 }
diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go
index 2f5eab302..ded2e1980 100644
--- a/bundle/phases/initialize.go
+++ b/bundle/phases/initialize.go
@@ -45,6 +45,7 @@ func Initialize() bundle.Mutator {
 			permissions.ApplyBundlePermissions(),
 			permissions.FilterCurrentUser(),
 			metadata.AnnotateJobs(),
+			metadata.AnnotatePipelines(),
 			terraform.Initialize(),
 			scripts.Execute(config.ScriptPostInit),
 		},

From 30215860e797c5c154862bff95ed2092a7e92d19 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Wed, 1 May 2024 16:34:37 +0530
Subject: [PATCH 09/41] Fix description memoization in bundle schema (#1409)

## Changes
Fixes https://github.com/databricks/cli/issues/559

The CLI generation is now stable and does not produce a diff for the
`bundle_descriptions.json` file.

Before a pointer to the schema was stored in the memo, which would be
mutated later to include the description. This lead to duplicate
documentation for schema components that were used in multiple places.
This PR fixes this issue.

Eg: Before all references of `pause_status` would have the same
description.

## Tests
Added regression test.
---
 bundle/schema/docs.go                       |  2 +-
 bundle/schema/docs/bundle_descriptions.json | 28 ++++----
 bundle/schema/openapi.go                    | 43 +++++++-----
 bundle/schema/openapi_test.go               | 74 ++++++++++++++++++---
 4 files changed, 108 insertions(+), 39 deletions(-)

diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go
index fe63e4328..5b960ea55 100644
--- a/bundle/schema/docs.go
+++ b/bundle/schema/docs.go
@@ -70,7 +70,7 @@ func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) {
 	}
 	openapiReader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 
 	// Generate descriptions for the "resources" field
diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json
index 75499507d..01d37dd71 100644
--- a/bundle/schema/docs/bundle_descriptions.json
+++ b/bundle/schema/docs/bundle_descriptions.json
@@ -756,7 +756,7 @@
                 "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.",
                 "properties": {
                   "pause_status": {
-                    "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
+                    "description": "Indicate whether this schedule is paused or not."
                   },
                   "quartz_cron_expression": {
                     "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required."
@@ -813,7 +813,7 @@
                           "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used."
                         },
                         "source": {
-                          "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                          "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider."
                         },
                         "warehouse_id": {
                           "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument."
@@ -972,7 +972,7 @@
                       "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried."
                     },
                     "new_cluster": {
-                      "description": "If new_cluster, a description of a cluster that is created for each task.",
+                      "description": "If new_cluster, a description of a new cluster that is created for each run.",
                       "properties": {
                         "apply_policy_default_values": {
                           "description": ""
@@ -1474,7 +1474,7 @@
                           "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required."
                         },
                         "source": {
-                          "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                          "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository."
                         }
                       }
                     },
@@ -1552,7 +1552,7 @@
                               "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
                             },
                             "source": {
-                              "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                              "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider."
                             }
                           }
                         },
@@ -1654,10 +1654,10 @@
                     }
                   },
                   "pause_status": {
-                    "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
+                    "description": "Whether this trigger is paused or not."
                   },
                   "table": {
-                    "description": "",
+                    "description": "Old table trigger settings name. Deprecated in favor of `table_update`.",
                     "properties": {
                       "condition": {
                         "description": "The table(s) condition based on which to trigger a job run."
@@ -3479,7 +3479,7 @@
                       "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.",
                       "properties": {
                         "pause_status": {
-                          "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
+                          "description": "Indicate whether this schedule is paused or not."
                         },
                         "quartz_cron_expression": {
                           "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required."
@@ -3536,7 +3536,7 @@
                                 "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used."
                               },
                               "source": {
-                                "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                                "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider."
                               },
                               "warehouse_id": {
                                 "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument."
@@ -3695,7 +3695,7 @@
                             "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried."
                           },
                           "new_cluster": {
-                            "description": "If new_cluster, a description of a cluster that is created for each task.",
+                            "description": "If new_cluster, a description of a new cluster that is created for each run.",
                             "properties": {
                               "apply_policy_default_values": {
                                 "description": ""
@@ -4197,7 +4197,7 @@
                                 "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required."
                               },
                               "source": {
-                                "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                                "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository."
                               }
                             }
                           },
@@ -4275,7 +4275,7 @@
                                     "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
                                   },
                                   "source": {
-                                    "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
+                                    "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider."
                                   }
                                 }
                               },
@@ -4377,10 +4377,10 @@
                           }
                         },
                         "pause_status": {
-                          "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
+                          "description": "Whether this trigger is paused or not."
                         },
                         "table": {
-                          "description": "",
+                          "description": "Old table trigger settings name. Deprecated in favor of `table_update`.",
                           "properties": {
                             "condition": {
                               "description": "The table(s) condition based on which to trigger a job run."
diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go
index fe329e7ac..1756d5165 100644
--- a/bundle/schema/openapi.go
+++ b/bundle/schema/openapi.go
@@ -10,17 +10,21 @@ import (
 )
 
 type OpenapiReader struct {
+	// OpenAPI spec to read schemas from.
 	OpenapiSpec *openapi.Specification
-	Memo        map[string]*jsonschema.Schema
+
+	// In-memory cache of schemas read from the OpenAPI spec.
+	memo map[string]jsonschema.Schema
 }
 
 const SchemaPathPrefix = "#/components/schemas/"
 
-func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, error) {
+// Read a schema directly from the OpenAPI spec.
+func (reader *OpenapiReader) readOpenapiSchema(path string) (jsonschema.Schema, error) {
 	schemaKey := strings.TrimPrefix(path, SchemaPathPrefix)
 
 	// return early if we already have a computed schema
-	memoSchema, ok := reader.Memo[schemaKey]
+	memoSchema, ok := reader.memo[schemaKey]
 	if ok {
 		return memoSchema, nil
 	}
@@ -28,18 +32,18 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema,
 	// check path is present in openapi spec
 	openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey]
 	if !ok {
-		return nil, fmt.Errorf("schema with path %s not found in openapi spec", path)
+		return jsonschema.Schema{}, fmt.Errorf("schema with path %s not found in openapi spec", path)
 	}
 
 	// convert openapi schema to the native schema struct
 	bytes, err := json.Marshal(*openapiSchema)
 	if err != nil {
-		return nil, err
+		return jsonschema.Schema{}, err
 	}
-	jsonSchema := &jsonschema.Schema{}
-	err = json.Unmarshal(bytes, jsonSchema)
+	jsonSchema := jsonschema.Schema{}
+	err = json.Unmarshal(bytes, &jsonSchema)
 	if err != nil {
-		return nil, err
+		return jsonschema.Schema{}, err
 	}
 
 	// A hack to convert a map[string]interface{} to *Schema
@@ -49,23 +53,28 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema,
 	if ok {
 		b, err := json.Marshal(jsonSchema.AdditionalProperties)
 		if err != nil {
-			return nil, err
+			return jsonschema.Schema{}, err
 		}
 		additionalProperties := &jsonschema.Schema{}
 		err = json.Unmarshal(b, additionalProperties)
 		if err != nil {
-			return nil, err
+			return jsonschema.Schema{}, err
 		}
 		jsonSchema.AdditionalProperties = additionalProperties
 	}
 
 	// store read schema into memo
-	reader.Memo[schemaKey] = jsonSchema
+	reader.memo[schemaKey] = jsonSchema
 
 	return jsonSchema, nil
 }
 
-// safe againt loops in refs
+// Resolve all nested "$ref" references in the schema. This function unrolls a single
+// level of "$ref" in the schema and calls into traverseSchema to resolve nested references.
+// Thus this function and traverseSchema are mutually recursive.
+//
+// This function is safe against reference loops. If a reference loop is detected, an error
+// is returned.
 func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) {
 	if root.Reference == nil {
 		return reader.traverseSchema(root, tracker)
@@ -91,12 +100,12 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t
 	// in the memo
 	root.Reference = nil
 
-	// unroll one level of reference
+	// unroll one level of reference.
 	selfRef, err := reader.readOpenapiSchema(ref)
 	if err != nil {
 		return nil, err
 	}
-	root = selfRef
+	root = &selfRef
 	root.Description = description
 
 	// traverse again to find new references
@@ -108,6 +117,8 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t
 	return root, err
 }
 
+// Traverse the nested properties of the schema to resolve "$ref" references. This function
+// and safeResolveRefs are mutually recursive.
 func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) {
 	// case primitive (or invalid)
 	if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType {
@@ -154,11 +165,11 @@ func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema
 	}
 	tracker := newTracker()
 	tracker.push(path, path)
-	root, err = reader.safeResolveRefs(root, tracker)
+	resolvedRoot, err := reader.safeResolveRefs(&root, tracker)
 	if err != nil {
 		return nil, tracker.errWithTrace(err.Error(), "")
 	}
-	return root, nil
+	return resolvedRoot, nil
 }
 
 func (reader *OpenapiReader) jobsDocs() (*Docs, error) {
diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go
index 0d71fa440..359b1e58a 100644
--- a/bundle/schema/openapi_test.go
+++ b/bundle/schema/openapi_test.go
@@ -48,7 +48,7 @@ func TestReadSchemaForObject(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -106,7 +106,7 @@ func TestReadSchemaForArray(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -152,7 +152,7 @@ func TestReadSchemaForMap(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -201,7 +201,7 @@ func TestRootReferenceIsResolved(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -251,7 +251,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -285,7 +285,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -330,7 +330,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -400,7 +400,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) {
 	spec := &openapi.Specification{}
 	reader := &OpenapiReader{
 		OpenapiSpec: spec,
-		Memo:        make(map[string]*jsonschema.Schema),
+		memo:        make(map[string]jsonschema.Schema),
 	}
 	err := json.Unmarshal([]byte(specString), spec)
 	require.NoError(t, err)
@@ -434,3 +434,61 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) {
 	t.Log("[DEBUG] expected: ", expected)
 	assert.Equal(t, expected, string(fruitsSchemaJson))
 }
+
+func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "number"
+				},
+				"fruits": {
+					"type": "object",
+					"properties": {
+						"guava": {
+							"type": "object",
+							"description": "Guava is a fruit",
+							"$ref": "#/components/schemas/foo"
+						},
+						"mango": {
+							"type": "object",
+							"description": "What is a mango?",
+							"$ref": "#/components/schemas/foo"
+						}
+					}
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		memo:        make(map[string]jsonschema.Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"properties": {
+				"guava": {
+					"type": "number",
+					"description": "Guava is a fruit"
+				},
+				"mango": {
+					"type": "number",
+					"description": "What is a mango?"
+				}
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}

From 4724ecb324cf0286f6fd756f74e26689b516d924 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 1 May 2024 14:09:06 +0200
Subject: [PATCH 10/41] Release v0.219.0 (#1412)

Bundles:
* Don't fail while parsing outdated terraform state
([#1404](https://github.com/databricks/cli/pull/1404)).
* Annotate DLT pipelines when deployed using DABs
([#1410](https://github.com/databricks/cli/pull/1410)).


API Changes:
* Changed `databricks libraries cluster-status` command. New request
type is compute.ClusterStatus.
 * Changed `databricks libraries cluster-status` command to return .
 * Added `databricks serving-endpoints get-open-api` command.

OpenAPI commit 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55 (2024-04-23)
Dependency updates:
* Bump github.com/databricks/databricks-sdk-go from 0.38.0 to 0.39.0
([#1405](https://github.com/databricks/cli/pull/1405)).
---
 CHANGELOG.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 898f0df9d..1bd824daf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,21 @@
 # Version changelog
 
+## 0.219.0
+
+Bundles:
+ * Don't fail while parsing outdated terraform state ([#1404](https://github.com/databricks/cli/pull/1404)).
+ * Annotate DLT pipelines when deployed using DABs ([#1410](https://github.com/databricks/cli/pull/1410)).
+
+
+API Changes:
+ * Changed `databricks libraries cluster-status` command. New request type is compute.ClusterStatus.
+ * Changed `databricks libraries cluster-status` command to return .
+ * Added `databricks serving-endpoints get-open-api` command.
+
+OpenAPI commit 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55 (2024-04-23)
+Dependency updates:
+ * Bump github.com/databricks/databricks-sdk-go from 0.38.0 to 0.39.0 ([#1405](https://github.com/databricks/cli/pull/1405)).
+
 ## 0.218.1
 
 This is a bugfix release.

From a393c87ed931f449b4a1b0d86024399ea1febfb9 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Mon, 6 May 2024 13:41:37 +0200
Subject: [PATCH 11/41] Upgrade TF provider to 1.42.0 (#1418)

## Changes
Upgrade TF provider to 1.42.0

Also fixes #1258
---
 bundle/internal/tf/codegen/schema/version.go  |  2 +-
 bundle/internal/tf/schema/data_source_job.go  | 39 +++++++-------
 bundle/internal/tf/schema/resource_cluster.go | 13 ++---
 .../tf/schema/resource_cluster_policy.go      | 13 ++---
 bundle/internal/tf/schema/resource_job.go     | 39 +++++++-------
 bundle/internal/tf/schema/resource_library.go | 17 ++++---
 .../tf/schema/resource_mws_ncc_binding.go     |  9 ++++
 .../resource_mws_ncc_private_endpoint_rule.go | 17 +++++++
 ...esource_mws_network_connectivity_config.go | 51 +++++++++++++++++++
 bundle/internal/tf/schema/resources.go        |  6 +++
 bundle/internal/tf/schema/root.go             |  2 +-
 11 files changed, 150 insertions(+), 58 deletions(-)
 create mode 100644 bundle/internal/tf/schema/resource_mws_ncc_binding.go
 create mode 100644 bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go
 create mode 100644 bundle/internal/tf/schema/resource_mws_network_connectivity_config.go

diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index 4fb4bf2c5..30885d961 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.40.0"
+const ProviderVersion = "1.42.0"
diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go
index dbd29f4ba..e5ec5afb7 100644
--- a/bundle/internal/tf/schema/data_source_job.go
+++ b/bundle/internal/tf/schema/data_source_job.go
@@ -243,12 +243,13 @@ type DataSourceJobJobSettingsSettingsLibraryPypi struct {
 }
 
 type DataSourceJobJobSettingsSettingsLibrary struct {
-	Egg   string                                        `json:"egg,omitempty"`
-	Jar   string                                        `json:"jar,omitempty"`
-	Whl   string                                        `json:"whl,omitempty"`
-	Cran  *DataSourceJobJobSettingsSettingsLibraryCran  `json:"cran,omitempty"`
-	Maven *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"`
-	Pypi  *DataSourceJobJobSettingsSettingsLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                                        `json:"egg,omitempty"`
+	Jar          string                                        `json:"jar,omitempty"`
+	Requirements string                                        `json:"requirements,omitempty"`
+	Whl          string                                        `json:"whl,omitempty"`
+	Cran         *DataSourceJobJobSettingsSettingsLibraryCran  `json:"cran,omitempty"`
+	Maven        *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"`
+	Pypi         *DataSourceJobJobSettingsSettingsLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type DataSourceJobJobSettingsSettingsNewClusterAutoscale struct {
@@ -558,12 +559,13 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi struct {
 }
 
 type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibrary struct {
-	Egg   string                                                           `json:"egg,omitempty"`
-	Jar   string                                                           `json:"jar,omitempty"`
-	Whl   string                                                           `json:"whl,omitempty"`
-	Cran  *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran  `json:"cran,omitempty"`
-	Maven *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
-	Pypi  *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                                                           `json:"egg,omitempty"`
+	Jar          string                                                           `json:"jar,omitempty"`
+	Requirements string                                                           `json:"requirements,omitempty"`
+	Whl          string                                                           `json:"whl,omitempty"`
+	Cran         *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran  `json:"cran,omitempty"`
+	Maven        *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
+	Pypi         *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskNewClusterAutoscale struct {
@@ -896,12 +898,13 @@ type DataSourceJobJobSettingsSettingsTaskLibraryPypi struct {
 }
 
 type DataSourceJobJobSettingsSettingsTaskLibrary struct {
-	Egg   string                                            `json:"egg,omitempty"`
-	Jar   string                                            `json:"jar,omitempty"`
-	Whl   string                                            `json:"whl,omitempty"`
-	Cran  *DataSourceJobJobSettingsSettingsTaskLibraryCran  `json:"cran,omitempty"`
-	Maven *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"`
-	Pypi  *DataSourceJobJobSettingsSettingsTaskLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                                            `json:"egg,omitempty"`
+	Jar          string                                            `json:"jar,omitempty"`
+	Requirements string                                            `json:"requirements,omitempty"`
+	Whl          string                                            `json:"whl,omitempty"`
+	Cran         *DataSourceJobJobSettingsSettingsTaskLibraryCran  `json:"cran,omitempty"`
+	Maven        *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"`
+	Pypi         *DataSourceJobJobSettingsSettingsTaskLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type DataSourceJobJobSettingsSettingsTaskNewClusterAutoscale struct {
diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go
index 6f866ba87..046e0bb43 100644
--- a/bundle/internal/tf/schema/resource_cluster.go
+++ b/bundle/internal/tf/schema/resource_cluster.go
@@ -146,12 +146,13 @@ type ResourceClusterLibraryPypi struct {
 }
 
 type ResourceClusterLibrary struct {
-	Egg   string                       `json:"egg,omitempty"`
-	Jar   string                       `json:"jar,omitempty"`
-	Whl   string                       `json:"whl,omitempty"`
-	Cran  *ResourceClusterLibraryCran  `json:"cran,omitempty"`
-	Maven *ResourceClusterLibraryMaven `json:"maven,omitempty"`
-	Pypi  *ResourceClusterLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                       `json:"egg,omitempty"`
+	Jar          string                       `json:"jar,omitempty"`
+	Requirements string                       `json:"requirements,omitempty"`
+	Whl          string                       `json:"whl,omitempty"`
+	Cran         *ResourceClusterLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceClusterLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceClusterLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type ResourceClusterWorkloadTypeClients struct {
diff --git a/bundle/internal/tf/schema/resource_cluster_policy.go b/bundle/internal/tf/schema/resource_cluster_policy.go
index 637fe6455..d8111fef2 100644
--- a/bundle/internal/tf/schema/resource_cluster_policy.go
+++ b/bundle/internal/tf/schema/resource_cluster_policy.go
@@ -19,12 +19,13 @@ type ResourceClusterPolicyLibrariesPypi struct {
 }
 
 type ResourceClusterPolicyLibraries struct {
-	Egg   string                               `json:"egg,omitempty"`
-	Jar   string                               `json:"jar,omitempty"`
-	Whl   string                               `json:"whl,omitempty"`
-	Cran  *ResourceClusterPolicyLibrariesCran  `json:"cran,omitempty"`
-	Maven *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"`
-	Pypi  *ResourceClusterPolicyLibrariesPypi  `json:"pypi,omitempty"`
+	Egg          string                               `json:"egg,omitempty"`
+	Jar          string                               `json:"jar,omitempty"`
+	Requirements string                               `json:"requirements,omitempty"`
+	Whl          string                               `json:"whl,omitempty"`
+	Cran         *ResourceClusterPolicyLibrariesCran  `json:"cran,omitempty"`
+	Maven        *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"`
+	Pypi         *ResourceClusterPolicyLibrariesPypi  `json:"pypi,omitempty"`
 }
 
 type ResourceClusterPolicy struct {
diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go
index 2431262c1..6958face8 100644
--- a/bundle/internal/tf/schema/resource_job.go
+++ b/bundle/internal/tf/schema/resource_job.go
@@ -243,12 +243,13 @@ type ResourceJobLibraryPypi struct {
 }
 
 type ResourceJobLibrary struct {
-	Egg   string                   `json:"egg,omitempty"`
-	Jar   string                   `json:"jar,omitempty"`
-	Whl   string                   `json:"whl,omitempty"`
-	Cran  *ResourceJobLibraryCran  `json:"cran,omitempty"`
-	Maven *ResourceJobLibraryMaven `json:"maven,omitempty"`
-	Pypi  *ResourceJobLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                   `json:"egg,omitempty"`
+	Jar          string                   `json:"jar,omitempty"`
+	Requirements string                   `json:"requirements,omitempty"`
+	Whl          string                   `json:"whl,omitempty"`
+	Cran         *ResourceJobLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type ResourceJobNewClusterAutoscale struct {
@@ -558,12 +559,13 @@ type ResourceJobTaskForEachTaskTaskLibraryPypi struct {
 }
 
 type ResourceJobTaskForEachTaskTaskLibrary struct {
-	Egg   string                                      `json:"egg,omitempty"`
-	Jar   string                                      `json:"jar,omitempty"`
-	Whl   string                                      `json:"whl,omitempty"`
-	Cran  *ResourceJobTaskForEachTaskTaskLibraryCran  `json:"cran,omitempty"`
-	Maven *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
-	Pypi  *ResourceJobTaskForEachTaskTaskLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                                      `json:"egg,omitempty"`
+	Jar          string                                      `json:"jar,omitempty"`
+	Requirements string                                      `json:"requirements,omitempty"`
+	Whl          string                                      `json:"whl,omitempty"`
+	Cran         *ResourceJobTaskForEachTaskTaskLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobTaskForEachTaskTaskLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct {
@@ -896,12 +898,13 @@ type ResourceJobTaskLibraryPypi struct {
 }
 
 type ResourceJobTaskLibrary struct {
-	Egg   string                       `json:"egg,omitempty"`
-	Jar   string                       `json:"jar,omitempty"`
-	Whl   string                       `json:"whl,omitempty"`
-	Cran  *ResourceJobTaskLibraryCran  `json:"cran,omitempty"`
-	Maven *ResourceJobTaskLibraryMaven `json:"maven,omitempty"`
-	Pypi  *ResourceJobTaskLibraryPypi  `json:"pypi,omitempty"`
+	Egg          string                       `json:"egg,omitempty"`
+	Jar          string                       `json:"jar,omitempty"`
+	Requirements string                       `json:"requirements,omitempty"`
+	Whl          string                       `json:"whl,omitempty"`
+	Cran         *ResourceJobTaskLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobTaskLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobTaskLibraryPypi  `json:"pypi,omitempty"`
 }
 
 type ResourceJobTaskNewClusterAutoscale struct {
diff --git a/bundle/internal/tf/schema/resource_library.go b/bundle/internal/tf/schema/resource_library.go
index e2e83fb4f..385d992df 100644
--- a/bundle/internal/tf/schema/resource_library.go
+++ b/bundle/internal/tf/schema/resource_library.go
@@ -19,12 +19,13 @@ type ResourceLibraryPypi struct {
 }
 
 type ResourceLibrary struct {
-	ClusterId string                `json:"cluster_id"`
-	Egg       string                `json:"egg,omitempty"`
-	Id        string                `json:"id,omitempty"`
-	Jar       string                `json:"jar,omitempty"`
-	Whl       string                `json:"whl,omitempty"`
-	Cran      *ResourceLibraryCran  `json:"cran,omitempty"`
-	Maven     *ResourceLibraryMaven `json:"maven,omitempty"`
-	Pypi      *ResourceLibraryPypi  `json:"pypi,omitempty"`
+	ClusterId    string                `json:"cluster_id"`
+	Egg          string                `json:"egg,omitempty"`
+	Id           string                `json:"id,omitempty"`
+	Jar          string                `json:"jar,omitempty"`
+	Requirements string                `json:"requirements,omitempty"`
+	Whl          string                `json:"whl,omitempty"`
+	Cran         *ResourceLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceLibraryPypi  `json:"pypi,omitempty"`
 }
diff --git a/bundle/internal/tf/schema/resource_mws_ncc_binding.go b/bundle/internal/tf/schema/resource_mws_ncc_binding.go
new file mode 100644
index 000000000..8beafb6f5
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_mws_ncc_binding.go
@@ -0,0 +1,9 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceMwsNccBinding struct {
+	Id                          string `json:"id,omitempty"`
+	NetworkConnectivityConfigId string `json:"network_connectivity_config_id"`
+	WorkspaceId                 int    `json:"workspace_id"`
+}
diff --git a/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go b/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go
new file mode 100644
index 000000000..2acb374bc
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go
@@ -0,0 +1,17 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceMwsNccPrivateEndpointRule struct {
+	ConnectionState             string `json:"connection_state,omitempty"`
+	CreationTime                int    `json:"creation_time,omitempty"`
+	Deactivated                 bool   `json:"deactivated,omitempty"`
+	DeactivatedAt               int    `json:"deactivated_at,omitempty"`
+	EndpointName                string `json:"endpoint_name,omitempty"`
+	GroupId                     string `json:"group_id"`
+	Id                          string `json:"id,omitempty"`
+	NetworkConnectivityConfigId string `json:"network_connectivity_config_id"`
+	ResourceId                  string `json:"resource_id"`
+	RuleId                      string `json:"rule_id,omitempty"`
+	UpdatedTime                 int    `json:"updated_time,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go b/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go
new file mode 100644
index 000000000..64ebab224
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go
@@ -0,0 +1,51 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule struct {
+	CidrBlocks []string `json:"cidr_blocks,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule struct {
+	Subnets        []string `json:"subnets,omitempty"`
+	TargetRegion   string   `json:"target_region,omitempty"`
+	TargetServices []string `json:"target_services,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules struct {
+	AwsStableIpRule          *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule          `json:"aws_stable_ip_rule,omitempty"`
+	AzureServiceEndpointRule *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule `json:"azure_service_endpoint_rule,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules struct {
+	ConnectionState             string `json:"connection_state,omitempty"`
+	CreationTime                int    `json:"creation_time,omitempty"`
+	Deactivated                 bool   `json:"deactivated,omitempty"`
+	DeactivatedAt               int    `json:"deactivated_at,omitempty"`
+	EndpointName                string `json:"endpoint_name,omitempty"`
+	GroupId                     string `json:"group_id,omitempty"`
+	NetworkConnectivityConfigId string `json:"network_connectivity_config_id,omitempty"`
+	ResourceId                  string `json:"resource_id,omitempty"`
+	RuleId                      string `json:"rule_id,omitempty"`
+	UpdatedTime                 int    `json:"updated_time,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules struct {
+	AzurePrivateEndpointRules []ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules `json:"azure_private_endpoint_rules,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfigEgressConfig struct {
+	DefaultRules *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules `json:"default_rules,omitempty"`
+	TargetRules  *ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules  `json:"target_rules,omitempty"`
+}
+
+type ResourceMwsNetworkConnectivityConfig struct {
+	AccountId                   string                                            `json:"account_id,omitempty"`
+	CreationTime                int                                               `json:"creation_time,omitempty"`
+	Id                          string                                            `json:"id,omitempty"`
+	Name                        string                                            `json:"name"`
+	NetworkConnectivityConfigId string                                            `json:"network_connectivity_config_id,omitempty"`
+	Region                      string                                            `json:"region"`
+	UpdatedTime                 int                                               `json:"updated_time,omitempty"`
+	EgressConfig                *ResourceMwsNetworkConnectivityConfigEgressConfig `json:"egress_config,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go
index b1b1841d6..e5eacc867 100644
--- a/bundle/internal/tf/schema/resources.go
+++ b/bundle/internal/tf/schema/resources.go
@@ -45,6 +45,9 @@ type Resources struct {
 	MwsCredentials                 map[string]any `json:"databricks_mws_credentials,omitempty"`
 	MwsCustomerManagedKeys         map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
 	MwsLogDelivery                 map[string]any `json:"databricks_mws_log_delivery,omitempty"`
+	MwsNccBinding                  map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
+	MwsNccPrivateEndpointRule      map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
+	MwsNetworkConnectivityConfig   map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
 	MwsNetworks                    map[string]any `json:"databricks_mws_networks,omitempty"`
 	MwsPermissionAssignment        map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
 	MwsPrivateAccessSettings       map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
@@ -137,6 +140,9 @@ func NewResources() *Resources {
 		MwsCredentials:                 make(map[string]any),
 		MwsCustomerManagedKeys:         make(map[string]any),
 		MwsLogDelivery:                 make(map[string]any),
+		MwsNccBinding:                  make(map[string]any),
+		MwsNccPrivateEndpointRule:      make(map[string]any),
+		MwsNetworkConnectivityConfig:   make(map[string]any),
 		MwsNetworks:                    make(map[string]any),
 		MwsPermissionAssignment:        make(map[string]any),
 		MwsPrivateAccessSettings:       make(map[string]any),
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index be6852bc0..50d05daab 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.40.0"
+const ProviderVersion = "1.42.0"
 
 func NewRoot() *Root {
 	return &Root{

From 648309d939be8eee360b9d337cc817ab4bf733a9 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 13:32:35 +0200
Subject: [PATCH 12/41] Bump golang.org/x/text from 0.14.0 to 0.15.0 (#1419)

Bumps [golang.org/x/text](https://github.com/golang/text) from 0.14.0 to
0.15.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/text/commit/8d533a0c40adec778a7d09ac6c8aa640d3c883f4"><code>8d533a0</code></a>
encoding/charmap: update UCM spec file URL prefix</li>
<li>See full diff in <a
href="https://github.com/golang/text/compare/v0.14.0...v0.15.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.14.0&new-version=0.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 7b2d31daa..5ba534106 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	golang.org/x/oauth2 v0.19.0
 	golang.org/x/sync v0.7.0
 	golang.org/x/term v0.19.0
-	golang.org/x/text v0.14.0
+	golang.org/x/text v0.15.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
 )
diff --git a/go.sum b/go.sum
index 5dc02d099..226969a87 100644
--- a/go.sum
+++ b/go.sum
@@ -214,8 +214,8 @@ golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
 golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 649016d50ddabd7b7210325057f0509660c102b0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 13:32:55 +0200
Subject: [PATCH 13/41] Bump golang.org/x/oauth2 from 0.19.0 to 0.20.0 (#1421)

Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from
0.19.0 to 0.20.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/oauth2/commit/84cb9f7f5c5a639955cd501bfdd54f0e63997e61"><code>84cb9f7</code></a>
oauth2: fix typo in comment</li>
<li><a
href="https://github.com/golang/oauth2/commit/4b7f0bdbc7c30d05383d2597847300fbb870608b"><code>4b7f0bd</code></a>
go.mod: update cloud.google.com/go/compute/metadata dependency</li>
<li><a
href="https://github.com/golang/oauth2/commit/e11eea88a89331c1fae3d38a37f484363a8d597d"><code>e11eea8</code></a>
microsoft: added DeviceAuthURL to AzureADEndpoint</li>
<li>See full diff in <a
href="https://github.com/golang/oauth2/compare/v0.19.0...v0.20.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.19.0&new-version=0.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  5 ++---
 go.sum | 10 ++++------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index 5ba534106..475f66f38 100644
--- a/go.mod
+++ b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/stretchr/testify v1.9.0 // MIT
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
 	golang.org/x/mod v0.17.0
-	golang.org/x/oauth2 v0.19.0
+	golang.org/x/oauth2 v0.20.0
 	golang.org/x/sync v0.7.0
 	golang.org/x/term v0.19.0
 	golang.org/x/text v0.15.0
@@ -32,8 +32,7 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.23.4 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
diff --git a/go.sum b/go.sum
index 226969a87..78f7bbd91 100644
--- a/go.sum
+++ b/go.sum
@@ -1,8 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw=
-cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI=
-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -191,8 +189,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
-golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
+golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
+golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

From 95bbe2ece191c0446a881b6637a682c4c3f5034e Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Mon, 13 May 2024 17:46:43 +0530
Subject: [PATCH 14/41] Fix flaky tests for the parallel mutator (#1426)

## Changes
Around 0.5% to 1% of the time, the tests would fail due to concurrent
access to the underlying slice in the mutator. This PR makes the test
thread safe preventing race conditions.

Example of failed run:
https://github.com/databricks/cli/actions/runs/9004657555/job/24738145829
---
 bundle/parallel_test.go | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/bundle/parallel_test.go b/bundle/parallel_test.go
index be1e33637..dfc7ddac9 100644
--- a/bundle/parallel_test.go
+++ b/bundle/parallel_test.go
@@ -2,6 +2,7 @@ package bundle
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/databricks/cli/bundle/config"
@@ -10,9 +11,14 @@ import (
 )
 
 type addToContainer struct {
+	t         *testing.T
 	container *[]int
 	value     int
 	err       bool
+
+	// mu is a mutex that protects container. It is used to ensure that the
+	// container slice is only modified by one goroutine at a time.
+	mu *sync.Mutex
 }
 
 func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagnostics {
@@ -20,9 +26,10 @@ func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagn
 		return diag.Errorf("error")
 	}
 
-	c := *m.container
-	c = append(c, m.value)
-	*m.container = c
+	m.mu.Lock()
+	*m.container = append(*m.container, m.value)
+	m.mu.Unlock()
+
 	return nil
 }
 
@@ -36,9 +43,10 @@ func TestParallelMutatorWork(t *testing.T) {
 	}
 
 	container := []int{}
-	m1 := &addToContainer{container: &container, value: 1}
-	m2 := &addToContainer{container: &container, value: 2}
-	m3 := &addToContainer{container: &container, value: 3}
+	var mu sync.Mutex
+	m1 := &addToContainer{t: t, container: &container, value: 1, mu: &mu}
+	m2 := &addToContainer{t: t, container: &container, value: 2, mu: &mu}
+	m3 := &addToContainer{t: t, container: &container, value: 3, mu: &mu}
 
 	m := Parallel(m1, m2, m3)
 
@@ -57,9 +65,10 @@ func TestParallelMutatorWorkWithErrors(t *testing.T) {
 	}
 
 	container := []int{}
-	m1 := &addToContainer{container: &container, value: 1}
-	m2 := &addToContainer{container: &container, err: true, value: 2}
-	m3 := &addToContainer{container: &container, value: 3}
+	var mu sync.Mutex
+	m1 := &addToContainer{container: &container, value: 1, mu: &mu}
+	m2 := &addToContainer{container: &container, err: true, value: 2, mu: &mu}
+	m3 := &addToContainer{container: &container, value: 3, mu: &mu}
 
 	m := Parallel(m1, m2, m3)
 

From 63617253bdd81f3250faa2d8bff2bd37384f0982 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 14 May 2024 16:00:48 +0530
Subject: [PATCH 15/41] Assert customer marshalling is implemented for
 resources (#1425)

## Changes
This PR ensures every resource implements a custom marshaller /
unmarshaller. This is required because we directly embed Go SDK structs.
which implement custom marshalling overrides. Since the struct is
embedded, the [customer marshalling
overrides](https://pkg.go.dev/encoding/json#example-package-CustomMarshalJSON)
are promoted to the top level. If the embedded struct itself is nil,
then JSON marshal / unmarshal will panic because it tries to call
`MarshalJSON` / `UnmarshalJSON` on a nil object.

Fixing this issue at the Go SDK level does not seem possible. Discussed
with @hectorcast-db.
---
 bundle/config/resources_test.go | 56 +++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go
index 9c4104e4d..7415029b1 100644
--- a/bundle/config/resources_test.go
+++ b/bundle/config/resources_test.go
@@ -1,6 +1,8 @@
 package config
 
 import (
+	"encoding/json"
+	"reflect"
 	"testing"
 
 	"github.com/databricks/cli/bundle/config/paths"
@@ -125,3 +127,57 @@ func TestVerifySafeMergeForRegisteredModels(t *testing.T) {
 	err := r.VerifySafeMerge(&other)
 	assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)")
 }
+
+// This test ensures that all resources have a custom marshaller and unmarshaller.
+// This is required because DABs resources map to Databricks APIs, and they do so
+// by embedding the corresponding Go SDK structs.
+//
+// Go SDK structs often implement custom marshalling and unmarshalling methods (based on the API specifics).
+// If the Go SDK struct implements custom marshalling and unmarshalling and we do not
+// for the resources at the top level, marshalling and unmarshalling operations will panic.
+// Thus we will be overly cautious and ensure that all resources need a custom marshaller and unmarshaller.
+//
+// Why do we not assert this using an interface to assert MarshalJSON and UnmarshalJSON
+// are implemented at the top level?
+// If a method is implemented for an embedded struct, the top level struct will
+// also have that method and satisfy the interface. This is why we cannot assert
+// that the methods are implemented at the top level using an interface.
+//
+// Why don't we use reflection to assert that the methods are implemented at the
+// top level?
+// Same problem as above, the golang reflection package does not seem to provide
+// a way to directly assert that MarshalJSON and UnmarshalJSON are implemented
+// at the top level.
+func TestCustomMarshallerIsImplemented(t *testing.T) {
+	r := Resources{}
+	rt := reflect.TypeOf(r)
+
+	for i := 0; i < rt.NumField(); i++ {
+		field := rt.Field(i)
+
+		// Fields in Resources are expected be of the form map[string]*resourceStruct
+		assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name)
+		kt := field.Type.Key()
+		assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name)
+		vt := field.Type.Elem()
+		assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name)
+
+		// Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller
+		// This is because resourceStruct embeds a Go SDK struct that implements
+		// a custom marshaller.
+		// Eg: resource.Job implements MarshalJSON
+		v := reflect.Zero(vt.Elem()).Interface()
+		assert.NotPanics(t, func() {
+			json.Marshal(v)
+		}, "Resource %s does not have a custom marshaller", field.Name)
+
+		// Unmarshalling a *resourceStruct will panic if the resource does not have a custom unmarshaller
+		// This is because resourceStruct embeds a Go SDK struct that implements
+		// a custom unmarshaller.
+		// Eg: *resource.Job implements UnmarshalJSON
+		v = reflect.New(vt.Elem()).Interface()
+		assert.NotPanics(t, func() {
+			json.Unmarshal([]byte("{}"), v)
+		}, "Resource %s does not have a custom unmarshaller", field.Name)
+	}
+}

From 5920da432007edb1d2c0109bf9bc4808215282f6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 14 May 2024 12:53:59 +0200
Subject: [PATCH 16/41] Bump golang.org/x/term from 0.19.0 to 0.20.0 (#1422)

Bumps [golang.org/x/term](https://github.com/golang/term) from 0.19.0 to
0.20.0.
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/golang/term/commit/46c790f81f1f50148a57f7ddf0c637b84ff2f0e6"><code>46c790f</code></a>
go.mod: update golang.org/x dependencies</li>
<li>See full diff in <a
href="https://github.com/golang/term/compare/v0.19.0...v0.20.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/term&package-manager=go_modules&previous-version=0.19.0&new-version=0.20.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 475f66f38..636fbf44a 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	golang.org/x/mod v0.17.0
 	golang.org/x/oauth2 v0.20.0
 	golang.org/x/sync v0.7.0
-	golang.org/x/term v0.19.0
+	golang.org/x/term v0.20.0
 	golang.org/x/text v0.15.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -59,7 +59,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.169.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
diff --git a/go.sum b/go.sum
index 78f7bbd91..3dd6b0cb7 100644
--- a/go.sum
+++ b/go.sum
@@ -206,10 +206,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=

From a71929b94399fe36ed56beeeb0bd12b71f186196 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 14 May 2024 16:28:55 +0530
Subject: [PATCH 17/41] Add line about Docker installation to README.md (#1363)

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 README.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.md b/README.md
index 83051ccf7..5f3b78b79 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,18 @@ See https://github.com/databricks/cli/releases for releases and
 [the docs pages](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) for
 installation instructions.
 
+------
+You can use the CLI via a Docker image by pulling the image from `ghcr.io`. You can find all available versions
+at: https://github.com/databricks/cli/pkgs/container/cli.
+```
+docker pull ghcr.io/databricks/cli:latest
+```
+
+Example of how to run the CLI using the Docker image. More documentation is available at https://docs.databricks.com/dev-tools/bundles/airgapped-environment.html.
+```
+docker run -e DATABRICKS_HOST=$YOUR_HOST_URL -e DATABRICKS_TOKEN=$YOUR_TOKEN ghcr.io/databricks/cli:latest current-user me
+```
+
 ## Authentication
 
 This CLI follows the Databricks Unified Authentication principles.

From 0a21428a4827a5dc2a26929b29f052dd6bbcb64c Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 14 May 2024 14:19:34 +0200
Subject: [PATCH 18/41] Upgrade to 1.43 terraform provider (#1429)

## Changes
Upgrade to 1.43 terraform provider
---
 bundle/internal/tf/codegen/schema/version.go | 2 +-
 bundle/internal/tf/schema/root.go            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index 30885d961..cf98e16e8 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.42.0"
+const ProviderVersion = "1.43.0"
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index 50d05daab..b1fed9424 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.42.0"
+const ProviderVersion = "1.43.0"
 
 func NewRoot() *Root {
 	return &Root{

From 2035516fde7f55c1cd424b9df5a76bfab5b7da4a Mon Sep 17 00:00:00 2001
From: Ilia Babanov <ilia.babanov@databricks.com>
Date: Wed, 15 May 2024 14:41:44 +0200
Subject: [PATCH 19/41] Don't merge-in remote resources during depolyments
 (#1432)

## Changes
`check_running_resources` now pulls the remote state without modifying
the bundle state, similar to how it was doing before. This avoids a
problem when we fail to compute deployment metadata for a deleted job
(which we shouldn't do in the first place)

`deploy_then_remove_resources_test` now also deploys and deletes a job
(in addition to a pipeline), which catches the error that this PR fixes.

## Tests
Unit and integ tests
---
 .../check_running_resources.go                | 81 +++++++++++--------
 .../check_running_resources_test.go           | 45 ++++++++---
 bundle/phases/deploy.go                       |  3 +-
 .../databricks_template_schema.json           |  8 ++
 .../template/bar.py                           |  2 +
 .../template/resources.yml.tmpl               | 11 +++
 .../deploy_then_remove_resources_test.go      | 17 +++-
 7 files changed, 117 insertions(+), 50 deletions(-)
 rename bundle/deploy/{ => terraform}/check_running_resources.go (60%)
 rename bundle/deploy/{ => terraform}/check_running_resources_test.go (75%)
 create mode 100644 internal/bundle/bundles/deploy_then_remove_resources/template/bar.py

diff --git a/bundle/deploy/check_running_resources.go b/bundle/deploy/terraform/check_running_resources.go
similarity index 60%
rename from bundle/deploy/check_running_resources.go
rename to bundle/deploy/terraform/check_running_resources.go
index a2305cd75..737f773e5 100644
--- a/bundle/deploy/check_running_resources.go
+++ b/bundle/deploy/terraform/check_running_resources.go
@@ -1,4 +1,4 @@
-package deploy
+package terraform
 
 import (
 	"context"
@@ -6,11 +6,11 @@ import (
 	"strconv"
 
 	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	tfjson "github.com/hashicorp/terraform-json"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -34,8 +34,14 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia
 	if !b.Config.Bundle.Deployment.FailOnActiveRuns {
 		return nil
 	}
+
+	state, err := ParseResourcesState(ctx, b)
+	if err != nil && state == nil {
+		return diag.FromErr(err)
+	}
+
 	w := b.WorkspaceClient()
-	err := checkAnyResourceRunning(ctx, w, &b.Config.Resources)
+	err = checkAnyResourceRunning(ctx, w, state)
 	if err != nil {
 		return diag.FromErr(err)
 	}
@@ -46,43 +52,50 @@ func CheckRunningResource() *checkRunningResources {
 	return &checkRunningResources{}
 }
 
-func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, resources *config.Resources) error {
-	errs, errCtx := errgroup.WithContext(ctx)
-
-	for _, job := range resources.Jobs {
-		id := job.ID
-		if id == "" {
-			continue
-		}
-		errs.Go(func() error {
-			isRunning, err := IsJobRunning(errCtx, w, id)
-			// If there's an error retrieving the job, we assume it's not running
-			if err != nil {
-				return err
-			}
-			if isRunning {
-				return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
-			}
-			return nil
-		})
+func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *resourcesState) error {
+	if state == nil {
+		return nil
 	}
 
-	for _, pipeline := range resources.Pipelines {
-		id := pipeline.ID
-		if id == "" {
+	errs, errCtx := errgroup.WithContext(ctx)
+
+	for _, resource := range state.Resources {
+		if resource.Mode != tfjson.ManagedResourceMode {
 			continue
 		}
-		errs.Go(func() error {
-			isRunning, err := IsPipelineRunning(errCtx, w, id)
-			// If there's an error retrieving the pipeline, we assume it's not running
-			if err != nil {
-				return nil
+		for _, instance := range resource.Instances {
+			id := instance.Attributes.ID
+			if id == "" {
+				continue
 			}
-			if isRunning {
-				return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
+
+			switch resource.Type {
+			case "databricks_job":
+				errs.Go(func() error {
+					isRunning, err := IsJobRunning(errCtx, w, id)
+					// If there's an error retrieving the job, we assume it's not running
+					if err != nil {
+						return err
+					}
+					if isRunning {
+						return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
+					}
+					return nil
+				})
+			case "databricks_pipeline":
+				errs.Go(func() error {
+					isRunning, err := IsPipelineRunning(errCtx, w, id)
+					// If there's an error retrieving the pipeline, we assume it's not running
+					if err != nil {
+						return nil
+					}
+					if isRunning {
+						return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
+					}
+					return nil
+				})
 			}
-			return nil
-		})
+		}
 	}
 
 	return errs.Wait()
diff --git a/bundle/deploy/check_running_resources_test.go b/bundle/deploy/terraform/check_running_resources_test.go
similarity index 75%
rename from bundle/deploy/check_running_resources_test.go
rename to bundle/deploy/terraform/check_running_resources_test.go
index d61c80fc4..a1bbbd37b 100644
--- a/bundle/deploy/check_running_resources_test.go
+++ b/bundle/deploy/terraform/check_running_resources_test.go
@@ -1,12 +1,10 @@
-package deploy
+package terraform
 
 import (
 	"context"
 	"errors"
 	"testing"
 
-	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
@@ -16,15 +14,22 @@ import (
 
 func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
 	mock := mocks.NewMockWorkspaceClient(t)
-	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &config.Resources{})
+	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &resourcesState{})
 	require.NoError(t, err)
 }
 
 func TestIsAnyResourceRunningWithJob(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &config.Resources{
-		Jobs: map[string]*resources.Job{
-			"job1": {ID: "123"},
+	resources := &resourcesState{
+		Resources: []stateResource{
+			{
+				Type: "databricks_job",
+				Mode: "managed",
+				Name: "job1",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "123"}},
+				},
+			},
 		},
 	}
 
@@ -50,9 +55,16 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
 
 func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &config.Resources{
-		Pipelines: map[string]*resources.Pipeline{
-			"pipeline1": {ID: "123"},
+	resources := &resourcesState{
+		Resources: []stateResource{
+			{
+				Type: "databricks_pipeline",
+				Mode: "managed",
+				Name: "pipeline1",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "123"}},
+				},
+			},
 		},
 	}
 
@@ -79,9 +91,16 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 
 func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &config.Resources{
-		Pipelines: map[string]*resources.Pipeline{
-			"pipeline1": {ID: "123"},
+	resources := &resourcesState{
+		Resources: []stateResource{
+			{
+				Type: "databricks_pipeline",
+				Mode: "managed",
+				Name: "pipeline1",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "123"}},
+				},
+			},
 		},
 	}
 
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 4fc4f6300..46c389189 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -36,8 +36,7 @@ func Deploy() bundle.Mutator {
 				permissions.ApplyWorkspaceRootPermissions(),
 				terraform.Interpolate(),
 				terraform.Write(),
-				terraform.Load(),
-				deploy.CheckRunningResource(),
+				terraform.CheckRunningResource(),
 				bundle.Defer(
 					terraform.Apply(),
 					bundle.Seq(
diff --git a/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json b/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json
index 8fca7a7c4..f03ad1c2b 100644
--- a/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json
+++ b/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json
@@ -3,6 +3,14 @@
         "unique_id": {
             "type": "string",
             "description": "Unique ID for pipeline name"
+        },
+        "spark_version": {
+            "type": "string",
+            "description": "Spark version used for job cluster"
+        },
+        "node_type_id": {
+            "type": "string",
+            "description": "Node type id for job cluster"
         }
     }
 }
diff --git a/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py b/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py
new file mode 100644
index 000000000..4914a7436
--- /dev/null
+++ b/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py
@@ -0,0 +1,2 @@
+# Databricks notebook source
+print("hello")
diff --git a/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl b/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl
index e3a676770..f3be9aafd 100644
--- a/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl
+++ b/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl
@@ -1,4 +1,15 @@
 resources:
+  jobs:
+    foo:
+      name: test-bundle-job-{{.unique_id}}
+      tasks:
+        - task_key: my_notebook_task
+          new_cluster:
+            num_workers: 1
+            spark_version: "{{.spark_version}}"
+            node_type_id: "{{.node_type_id}}"
+          notebook_task:
+            notebook_path: "./bar.py"
   pipelines:
     bar:
       name: test-bundle-pipeline-{{.unique_id}}
diff --git a/internal/bundle/deploy_then_remove_resources_test.go b/internal/bundle/deploy_then_remove_resources_test.go
index 72baf798c..66ec5c16a 100644
--- a/internal/bundle/deploy_then_remove_resources_test.go
+++ b/internal/bundle/deploy_then_remove_resources_test.go
@@ -5,7 +5,9 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/libs/env"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -15,9 +17,12 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) {
 	ctx, wt := acc.WorkspaceTest(t)
 	w := wt.W
 
+	nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
 	uniqueId := uuid.New().String()
 	bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{
-		"unique_id": uniqueId,
+		"unique_id":     uniqueId,
+		"node_type_id":  nodeTypeId,
+		"spark_version": defaultSparkVersion,
 	})
 	require.NoError(t, err)
 
@@ -31,6 +36,12 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, pipeline.Name, pipelineName)
 
+	// assert job is created
+	jobName := "test-bundle-job-" + uniqueId
+	job, err := w.Jobs.GetBySettingsName(ctx, jobName)
+	require.NoError(t, err)
+	assert.Equal(t, job.Settings.Name, jobName)
+
 	// delete resources.yml
 	err = os.Remove(filepath.Join(bundleRoot, "resources.yml"))
 	require.NoError(t, err)
@@ -43,6 +54,10 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) {
 	_, err = w.Pipelines.GetByName(ctx, pipelineName)
 	assert.ErrorContains(t, err, "does not exist")
 
+	// assert job is deleted
+	_, err = w.Jobs.GetBySettingsName(ctx, jobName)
+	assert.ErrorContains(t, err, "does not exist")
+
 	t.Cleanup(func() {
 		err = destroyBundle(t, ctx, bundleRoot)
 		require.NoError(t, err)

From 216d2b058aa519b31592861db7af3b69cf993193 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 16 May 2024 11:04:58 +0200
Subject: [PATCH 20/41] Bump github.com/databricks/databricks-sdk-go from
 0.39.0 to 0.40.1 (#1431)

Bumps
[github.com/databricks/databricks-sdk-go](https://github.com/databricks/databricks-sdk-go)
from 0.39.0 to 0.40.1.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/releases">github.com/databricks/databricks-sdk-go's
releases</a>.</em></p>
<blockquote>
<h2>v0.40.1</h2>
<ul>
<li>Fixed codecov for repository (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/909">#909</a>).</li>
<li>Add traceparent header to enable distributed tracing. (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/914">#914</a>).</li>
<li>Log cancelled and failed requests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/919">#919</a>).</li>
</ul>
<p>Dependency updates:</p>
<ul>
<li>Bump golang.org/x/net from 0.22.0 to 0.24.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/884">#884</a>).</li>
<li>Bump golang.org/x/net from 0.17.0 to 0.23.0 in /examples/zerolog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/896">#896</a>).</li>
<li>Bump golang.org/x/net from 0.21.0 to 0.23.0 in /examples/slog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/897">#897</a>).</li>
</ul>
<h2>v0.40.0</h2>
<h2>0.40.0</h2>
<ul>
<li>Allow unlimited timeouts in retries (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/904">#904</a>).
By setting RETRY_TIMEOUT_SECONDS to a negative value, WorkspaceClient
and AccountClient will retry retriable failures indefinitely. As a
reminder, without setting this parameter, the default retry timeout is 5
minutes.</li>
</ul>
<p>API Changes:</p>
<ul>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service . New request type is <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#CreateAppRequest">serving.CreateAppRequest</a>.</li>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#App">serving.App</a>.</li>
<li>Removed <code>DeleteApp</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetApp</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetAppDeploymentStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetApps</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetEvents</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>CreateDeployment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Delete</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Get</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>GetDeployment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>GetEnvironment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>List</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>ListDeployments</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Stop</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppEvents">serving.AppEvents</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppManifest">serving.AppManifest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppServiceStatus">serving.AppServiceStatus</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeleteAppResponse">serving.DeleteAppResponse</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeployAppRequest">serving.DeployAppRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeploymentStatus">serving.DeploymentStatus</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeploymentStatusState">serving.DeploymentStatusState</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetAppDeploymentStatusRequest">serving.GetAppDeploymentStatusRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetAppResponse">serving.GetAppResponse</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetEventsRequest">serving.GetEventsRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ListAppEventsResponse">serving.ListAppEventsResponse</a>.</li>
<li>Changed <code>Apps</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ListAppsResponse">serving.ListAppsResponse</a>
to <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppList">serving.AppList</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#App">serving.App</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppDeployment">serving.AppDeployment</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppDeploymentState">serving.AppDeploymentState</a>.</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/databricks/databricks-sdk-go/blob/main/CHANGELOG.md">github.com/databricks/databricks-sdk-go's
changelog</a>.</em></p>
<blockquote>
<h2>0.40.1</h2>
<ul>
<li>Fixed codecov for repository (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/909">#909</a>).</li>
<li>Add traceparent header to enable distributed tracing. (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/914">#914</a>).</li>
<li>Log cancelled and failed requests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/919">#919</a>).</li>
</ul>
<p>Dependency updates:</p>
<ul>
<li>Bump golang.org/x/net from 0.22.0 to 0.24.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/884">#884</a>).</li>
<li>Bump golang.org/x/net from 0.17.0 to 0.23.0 in /examples/zerolog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/896">#896</a>).</li>
<li>Bump golang.org/x/net from 0.21.0 to 0.23.0 in /examples/slog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/897">#897</a>).</li>
</ul>
<h2>0.40.0</h2>
<ul>
<li>Allow unlimited timeouts in retries (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/pull/904">#904</a>).
By setting RETRY_TIMEOUT_SECONDS to a negative value, WorkspaceClient
and AccountClient will retry retriable failures indefinitely. As a
reminder, without setting this parameter, the default retry timeout is 5
minutes.</li>
</ul>
<p>API Changes:</p>
<ul>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service . New request type is <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#CreateAppRequest">serving.CreateAppRequest</a>.</li>
<li>Changed <code>Create</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service to return <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#App">serving.App</a>.</li>
<li>Removed <code>DeleteApp</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetApp</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetAppDeploymentStatus</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetApps</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <code>GetEvents</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>CreateDeployment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Delete</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Get</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>GetDeployment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>GetEnvironment</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>List</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>ListDeployments</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Stop</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Added <code>Update</code> method for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppsAPI">w.Apps</a>
workspace-level service.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppEvents">serving.AppEvents</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppManifest">serving.AppManifest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppServiceStatus">serving.AppServiceStatus</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeleteAppResponse">serving.DeleteAppResponse</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeployAppRequest">serving.DeployAppRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeploymentStatus">serving.DeploymentStatus</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#DeploymentStatusState">serving.DeploymentStatusState</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetAppDeploymentStatusRequest">serving.GetAppDeploymentStatusRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetAppResponse">serving.GetAppResponse</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#GetEventsRequest">serving.GetEventsRequest</a>.</li>
<li>Removed <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ListAppEventsResponse">serving.ListAppEventsResponse</a>.</li>
<li>Changed <code>Apps</code> field for <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#ListAppsResponse">serving.ListAppsResponse</a>
to <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppList">serving.AppList</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#App">serving.App</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppDeployment">serving.AppDeployment</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppDeploymentState">serving.AppDeploymentState</a>.</li>
<li>Added <a
href="https://pkg.go.dev/github.com/databricks/databricks-sdk-go/service/serving#AppDeploymentStatus">serving.AppDeploymentStatus</a>.</li>
</ul>
<!-- raw HTML omitted -->
</blockquote>
<p>... (truncated)</p>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/72334ef44955eb985fa1d7d0de7e26378f140030"><code>72334ef</code></a>
Release v0.40.1 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/920">#920</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/63343c8c66dcfb97fac2f4e874a4c99cc879d6c9"><code>63343c8</code></a>
Log cancelled and failed requests (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/919">#919</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/27d08a67df5b0d35544c663f9d47577f85ffc6e4"><code>27d08a6</code></a>
Add traceparent header to enable distributed tracing. (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/914">#914</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/b5322db1a7c99a919013a1d90e0a6878b8e0ba38"><code>b5322db</code></a>
Bump golang.org/x/net from 0.21.0 to 0.23.0 in /examples/slog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/897">#897</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/5edb2dfc92bc636b1a94520c599eea5c932255df"><code>5edb2df</code></a>
Bump golang.org/x/net from 0.17.0 to 0.23.0 in /examples/zerolog (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/896">#896</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/0e8dba0f70283f6a02b69b45ac63ed90cc2fe68d"><code>0e8dba0</code></a>
Bump golang.org/x/net from 0.22.0 to 0.24.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/884">#884</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/b6e76fc44b81bf9e043d1e7eb160c4d92126079b"><code>b6e76fc</code></a>
Fixed codecov for repository (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/909">#909</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/70aa8eea19c495d984363382c3db5dc9d8234a75"><code>70aa8ee</code></a>
Release v0.40.0 (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/908">#908</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/dae1642c5d03911e98cb4af4ca3cf1d126e6accc"><code>dae1642</code></a>
Add ToLibraryList() for ClusterStatusResponse (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/906">#906</a>)</li>
<li><a
href="https://github.com/databricks/databricks-sdk-go/commit/c6516aa754967d7f49146b5a10de8c71b59f126b"><code>c6516aa</code></a>
Bump version of mockery (<a
href="https://redirect.github.com/databricks/databricks-sdk-go/issues/907">#907</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/databricks/databricks-sdk-go/compare/v0.39.0...v0.40.1">compare
view</a></li>
</ul>
</details>
<br />

<details>
<summary>Most Recent Ignore Conditions Applied to This Pull
Request</summary>

| Dependency Name | Ignore Conditions |
| --- | --- |
| github.com/databricks/databricks-sdk-go | [>= 0.28.a, < 0.29] |
</details>


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/databricks/databricks-sdk-go&package-manager=go_modules&previous-version=0.39.0&new-version=0.40.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 .codegen/_openapi_sha                         |   2 +-
 .gitattributes                                |   4 +-
 bundle/schema/docs/bundle_descriptions.json   |   4 +-
 .../csp-enablement-account.go                 |   2 +-
 .../esm-enablement-account.go                 |   2 +-
 cmd/workspace/apps/apps.go                    | 558 +++++++++++++++---
 cmd/workspace/apps/overrides.go               |  58 --
 .../compliance-security-profile.go}           |  18 +-
 cmd/workspace/dashboards/dashboards.go        |   1 +
 .../enhanced-security-monitoring.go}          |  18 +-
 .../model-versions/model-versions.go          |   1 +
 cmd/workspace/queries/queries.go              |   1 +
 cmd/workspace/settings/settings.go            |   8 +-
 go.mod                                        |   6 +-
 go.sum                                        |  12 +-
 15 files changed, 508 insertions(+), 187 deletions(-)
 delete mode 100644 cmd/workspace/apps/overrides.go
 rename cmd/workspace/{csp-enablement/csp-enablement.go => compliance-security-profile/compliance-security-profile.go} (89%)
 rename cmd/workspace/{esm-enablement/esm-enablement.go => enhanced-security-monitoring/enhanced-security-monitoring.go} (89%)

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 1f11c17bf..f07cf44e5 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-21f9f1482f9d0d15228da59f2cd9f0863d2a6d55
\ No newline at end of file
+9bb7950fa3390afb97abaa552934bc0a2e069de5
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
index f9aa02d18..fb42588a7 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -37,6 +37,7 @@ cmd/workspace/clean-rooms/clean-rooms.go linguist-generated=true
 cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true
 cmd/workspace/clusters/clusters.go linguist-generated=true
 cmd/workspace/cmd.go linguist-generated=true
+cmd/workspace/compliance-security-profile/compliance-security-profile.go linguist-generated=true
 cmd/workspace/connections/connections.go linguist-generated=true
 cmd/workspace/consumer-fulfillments/consumer-fulfillments.go linguist-generated=true
 cmd/workspace/consumer-installations/consumer-installations.go linguist-generated=true
@@ -44,13 +45,12 @@ cmd/workspace/consumer-listings/consumer-listings.go linguist-generated=true
 cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go linguist-generated=true
 cmd/workspace/consumer-providers/consumer-providers.go linguist-generated=true
 cmd/workspace/credentials-manager/credentials-manager.go linguist-generated=true
-cmd/workspace/csp-enablement/csp-enablement.go linguist-generated=true
 cmd/workspace/current-user/current-user.go linguist-generated=true
 cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true
 cmd/workspace/dashboards/dashboards.go linguist-generated=true
 cmd/workspace/data-sources/data-sources.go linguist-generated=true
 cmd/workspace/default-namespace/default-namespace.go linguist-generated=true
-cmd/workspace/esm-enablement/esm-enablement.go linguist-generated=true
+cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true
 cmd/workspace/experiments/experiments.go linguist-generated=true
 cmd/workspace/external-locations/external-locations.go linguist-generated=true
 cmd/workspace/functions/functions.go linguist-generated=true
diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json
index 01d37dd71..ba6fe8ce2 100644
--- a/bundle/schema/docs/bundle_descriptions.json
+++ b/bundle/schema/docs/bundle_descriptions.json
@@ -1582,7 +1582,7 @@
                       "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout."
                     },
                     "webhook_notifications": {
-                      "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
+                      "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
                       "properties": {
                         "on_duration_warning_threshold_exceeded": {
                           "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@@ -4305,7 +4305,7 @@
                             "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout."
                           },
                           "webhook_notifications": {
-                            "description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
+                            "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
                             "properties": {
                               "on_duration_warning_threshold_exceeded": {
                                 "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
diff --git a/cmd/account/csp-enablement-account/csp-enablement-account.go b/cmd/account/csp-enablement-account/csp-enablement-account.go
index 79819003b..d6fce9537 100755
--- a/cmd/account/csp-enablement-account/csp-enablement-account.go
+++ b/cmd/account/csp-enablement-account/csp-enablement-account.go
@@ -156,4 +156,4 @@ func newUpdate() *cobra.Command {
 	return cmd
 }
 
-// end service CSPEnablementAccount
+// end service CspEnablementAccount
diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go
index a2e95ffe1..49c21eb48 100755
--- a/cmd/account/esm-enablement-account/esm-enablement-account.go
+++ b/cmd/account/esm-enablement-account/esm-enablement-account.go
@@ -157,4 +157,4 @@ func newUpdate() *cobra.Command {
 	return cmd
 }
 
-// end service ESMEnablementAccount
+// end service EsmEnablementAccount
diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go
index 1ea50e830..2ccd16c0c 100755
--- a/cmd/workspace/apps/apps.go
+++ b/cmd/workspace/apps/apps.go
@@ -4,6 +4,7 @@ package apps
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
@@ -19,10 +20,10 @@ var cmdOverrides []func(*cobra.Command)
 func New() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "apps",
-		Short: `Lakehouse Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`,
-		Long: `Lakehouse Apps run directly on a customer’s Databricks instance, integrate
-  with their data, use and extend Databricks services, and enable users to
-  interact through single sign-on.`,
+		Short: `Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`,
+		Long: `Apps run directly on a customer’s Databricks instance, integrate with their
+  data, use and extend Databricks services, and enable users to interact through
+  single sign-on.`,
 		GroupID: "serving",
 		Annotations: map[string]string{
 			"package": "serving",
@@ -34,11 +35,15 @@ func New() *cobra.Command {
 
 	// Add methods
 	cmd.AddCommand(newCreate())
-	cmd.AddCommand(newDeleteApp())
-	cmd.AddCommand(newGetApp())
-	cmd.AddCommand(newGetAppDeploymentStatus())
-	cmd.AddCommand(newGetApps())
-	cmd.AddCommand(newGetEvents())
+	cmd.AddCommand(newCreateDeployment())
+	cmd.AddCommand(newDelete())
+	cmd.AddCommand(newGet())
+	cmd.AddCommand(newGetDeployment())
+	cmd.AddCommand(newGetEnvironment())
+	cmd.AddCommand(newList())
+	cmd.AddCommand(newListDeployments())
+	cmd.AddCommand(newStop())
+	cmd.AddCommand(newUpdate())
 
 	// Apply optional overrides to this command.
 	for _, fn := range cmdOverrides {
@@ -54,28 +59,53 @@ func New() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var createOverrides []func(
 	*cobra.Command,
-	*serving.DeployAppRequest,
+	*serving.CreateAppRequest,
 )
 
 func newCreate() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var createReq serving.DeployAppRequest
+	var createReq serving.CreateAppRequest
 	var createJson flags.JsonFlag
 
+	var createSkipWait bool
+	var createTimeout time.Duration
+
+	cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach IDLE state`)
+	cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach IDLE state`)
 	// TODO: short flags
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
-	// TODO: any: resources
+	cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`)
 
-	cmd.Use = "create"
-	cmd.Short = `Create and deploy an application.`
-	cmd.Long = `Create and deploy an application.
+	cmd.Use = "create NAME"
+	cmd.Short = `Create an App.`
+	cmd.Long = `Create an App.
   
-  Creates and deploys an application.`
+  Creates a new app.
+
+  Arguments:
+    NAME: The name of the app. The name must contain only lowercase alphanumeric
+      characters and hyphens and be between 2 and 30 characters long. It must be
+      unique within the workspace.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(0)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
@@ -86,15 +116,35 @@ func newCreate() *cobra.Command {
 			if err != nil {
 				return err
 			}
-		} else {
-			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
+		}
+		if !cmd.Flags().Changed("json") {
+			createReq.Name = args[0]
 		}
 
-		response, err := w.Apps.Create(ctx, createReq)
+		wait, err := w.Apps.Create(ctx, createReq)
 		if err != nil {
 			return err
 		}
-		return cmdio.Render(ctx, response)
+		if createSkipWait {
+			return cmdio.Render(ctx, wait.Response)
+		}
+		spinner := cmdio.Spinner(ctx)
+		info, err := wait.OnProgress(func(i *serving.App) {
+			if i.Status == nil {
+				return
+			}
+			status := i.Status.State
+			statusMessage := fmt.Sprintf("current status: %s", status)
+			if i.Status != nil {
+				statusMessage = i.Status.Message
+			}
+			spinner <- statusMessage
+		}).GetWithTimeout(createTimeout)
+		close(spinner)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, info)
 	}
 
 	// Disable completions since they are not applicable.
@@ -109,30 +159,137 @@ func newCreate() *cobra.Command {
 	return cmd
 }
 
-// start delete-app command
+// start create-deployment command
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
-var deleteAppOverrides []func(
+var createDeploymentOverrides []func(
+	*cobra.Command,
+	*serving.CreateAppDeploymentRequest,
+)
+
+func newCreateDeployment() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var createDeploymentReq serving.CreateAppDeploymentRequest
+	var createDeploymentJson flags.JsonFlag
+
+	var createDeploymentSkipWait bool
+	var createDeploymentTimeout time.Duration
+
+	cmd.Flags().BoolVar(&createDeploymentSkipWait, "no-wait", createDeploymentSkipWait, `do not wait to reach SUCCEEDED state`)
+	cmd.Flags().DurationVar(&createDeploymentTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`)
+	// TODO: short flags
+	cmd.Flags().Var(&createDeploymentJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	cmd.Use = "create-deployment APP_NAME SOURCE_CODE_PATH"
+	cmd.Short = `Create an App Deployment.`
+	cmd.Long = `Create an App Deployment.
+  
+  Creates an app deployment for the app with the supplied name.
+
+  Arguments:
+    APP_NAME: The name of the app.
+    SOURCE_CODE_PATH: The source code path of the deployment.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		if cmd.Flags().Changed("json") {
+			err := root.ExactArgs(1)(cmd, args)
+			if err != nil {
+				return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input")
+			}
+			return nil
+		}
+		check := root.ExactArgs(2)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			err = createDeploymentJson.Unmarshal(&createDeploymentReq)
+			if err != nil {
+				return err
+			}
+		}
+		createDeploymentReq.AppName = args[0]
+		if !cmd.Flags().Changed("json") {
+			createDeploymentReq.SourceCodePath = args[1]
+		}
+
+		wait, err := w.Apps.CreateDeployment(ctx, createDeploymentReq)
+		if err != nil {
+			return err
+		}
+		if createDeploymentSkipWait {
+			return cmdio.Render(ctx, wait.Response)
+		}
+		spinner := cmdio.Spinner(ctx)
+		info, err := wait.OnProgress(func(i *serving.AppDeployment) {
+			if i.Status == nil {
+				return
+			}
+			status := i.Status.State
+			statusMessage := fmt.Sprintf("current status: %s", status)
+			if i.Status != nil {
+				statusMessage = i.Status.Message
+			}
+			spinner <- statusMessage
+		}).GetWithTimeout(createDeploymentTimeout)
+		close(spinner)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, info)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range createDeploymentOverrides {
+		fn(cmd, &createDeploymentReq)
+	}
+
+	return cmd
+}
+
+// start delete command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var deleteOverrides []func(
 	*cobra.Command,
 	*serving.DeleteAppRequest,
 )
 
-func newDeleteApp() *cobra.Command {
+func newDelete() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var deleteAppReq serving.DeleteAppRequest
+	var deleteReq serving.DeleteAppRequest
 
 	// TODO: short flags
 
-	cmd.Use = "delete-app NAME"
-	cmd.Short = `Delete an application.`
-	cmd.Long = `Delete an application.
+	cmd.Use = "delete NAME"
+	cmd.Short = `Delete an App.`
+	cmd.Long = `Delete an App.
   
-  Delete an application definition
+  Deletes an app.
 
   Arguments:
-    NAME: The name of an application. This field is required.`
+    NAME: The name of the app.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
@@ -146,13 +303,13 @@ func newDeleteApp() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		deleteAppReq.Name = args[0]
+		deleteReq.Name = args[0]
 
-		response, err := w.Apps.DeleteApp(ctx, deleteAppReq)
+		err = w.Apps.Delete(ctx, deleteReq)
 		if err != nil {
 			return err
 		}
-		return cmdio.Render(ctx, response)
+		return nil
 	}
 
 	// Disable completions since they are not applicable.
@@ -160,37 +317,40 @@ func newDeleteApp() *cobra.Command {
 	cmd.ValidArgsFunction = cobra.NoFileCompletions
 
 	// Apply optional overrides to this command.
-	for _, fn := range deleteAppOverrides {
-		fn(cmd, &deleteAppReq)
+	for _, fn := range deleteOverrides {
+		fn(cmd, &deleteReq)
 	}
 
 	return cmd
 }
 
-// start get-app command
+// start get command
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
-var getAppOverrides []func(
+var getOverrides []func(
 	*cobra.Command,
 	*serving.GetAppRequest,
 )
 
-func newGetApp() *cobra.Command {
+func newGet() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getAppReq serving.GetAppRequest
+	var getReq serving.GetAppRequest
 
 	// TODO: short flags
 
-	cmd.Use = "get-app NAME"
-	cmd.Short = `Get definition for an application.`
-	cmd.Long = `Get definition for an application.
+	cmd.Use = "get NAME"
+	cmd.Short = `Get an App.`
+	cmd.Long = `Get an App.
   
-  Get an application definition
+  Retrieves information for the app with the supplied name.
 
   Arguments:
-    NAME: The name of an application. This field is required.`
+    NAME: The name of the app.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
@@ -204,9 +364,9 @@ func newGetApp() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		getAppReq.Name = args[0]
+		getReq.Name = args[0]
 
-		response, err := w.Apps.GetApp(ctx, getAppReq)
+		response, err := w.Apps.Get(ctx, getReq)
 		if err != nil {
 			return err
 		}
@@ -218,39 +378,104 @@ func newGetApp() *cobra.Command {
 	cmd.ValidArgsFunction = cobra.NoFileCompletions
 
 	// Apply optional overrides to this command.
-	for _, fn := range getAppOverrides {
-		fn(cmd, &getAppReq)
+	for _, fn := range getOverrides {
+		fn(cmd, &getReq)
 	}
 
 	return cmd
 }
 
-// start get-app-deployment-status command
+// start get-deployment command
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
-var getAppDeploymentStatusOverrides []func(
+var getDeploymentOverrides []func(
 	*cobra.Command,
-	*serving.GetAppDeploymentStatusRequest,
+	*serving.GetAppDeploymentRequest,
 )
 
-func newGetAppDeploymentStatus() *cobra.Command {
+func newGetDeployment() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getAppDeploymentStatusReq serving.GetAppDeploymentStatusRequest
+	var getDeploymentReq serving.GetAppDeploymentRequest
 
 	// TODO: short flags
 
-	cmd.Flags().StringVar(&getAppDeploymentStatusReq.IncludeAppLog, "include-app-log", getAppDeploymentStatusReq.IncludeAppLog, `Boolean flag to include application logs.`)
-
-	cmd.Use = "get-app-deployment-status DEPLOYMENT_ID"
-	cmd.Short = `Get deployment status for an application.`
-	cmd.Long = `Get deployment status for an application.
+	cmd.Use = "get-deployment APP_NAME DEPLOYMENT_ID"
+	cmd.Short = `Get an App Deployment.`
+	cmd.Long = `Get an App Deployment.
   
-  Get deployment status for an application
+  Retrieves information for the app deployment with the supplied name and
+  deployment id.
 
   Arguments:
-    DEPLOYMENT_ID: The deployment id for an application. This field is required.`
+    APP_NAME: The name of the app.
+    DEPLOYMENT_ID: The unique id of the deployment.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(2)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		getDeploymentReq.AppName = args[0]
+		getDeploymentReq.DeploymentId = args[1]
+
+		response, err := w.Apps.GetDeployment(ctx, getDeploymentReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getDeploymentOverrides {
+		fn(cmd, &getDeploymentReq)
+	}
+
+	return cmd
+}
+
+// start get-environment command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getEnvironmentOverrides []func(
+	*cobra.Command,
+	*serving.GetAppEnvironmentRequest,
+)
+
+func newGetEnvironment() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getEnvironmentReq serving.GetAppEnvironmentRequest
+
+	// TODO: short flags
+
+	cmd.Use = "get-environment NAME"
+	cmd.Short = `Get App Environment.`
+	cmd.Long = `Get App Environment.
+  
+  Retrieves app environment.
+
+  Arguments:
+    NAME: The name of the app.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
@@ -264,9 +489,9 @@ func newGetAppDeploymentStatus() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		getAppDeploymentStatusReq.DeploymentId = args[0]
+		getEnvironmentReq.Name = args[0]
 
-		response, err := w.Apps.GetAppDeploymentStatus(ctx, getAppDeploymentStatusReq)
+		response, err := w.Apps.GetEnvironment(ctx, getEnvironmentReq)
 		if err != nil {
 			return err
 		}
@@ -278,41 +503,55 @@ func newGetAppDeploymentStatus() *cobra.Command {
 	cmd.ValidArgsFunction = cobra.NoFileCompletions
 
 	// Apply optional overrides to this command.
-	for _, fn := range getAppDeploymentStatusOverrides {
-		fn(cmd, &getAppDeploymentStatusReq)
+	for _, fn := range getEnvironmentOverrides {
+		fn(cmd, &getEnvironmentReq)
 	}
 
 	return cmd
 }
 
-// start get-apps command
+// start list command
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
-var getAppsOverrides []func(
+var listOverrides []func(
 	*cobra.Command,
+	*serving.ListAppsRequest,
 )
 
-func newGetApps() *cobra.Command {
+func newList() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	cmd.Use = "get-apps"
-	cmd.Short = `List all applications.`
-	cmd.Long = `List all applications.
+	var listReq serving.ListAppsRequest
+
+	// TODO: short flags
+
+	cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Upper bound for items returned.`)
+	cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Pagination token to go to the next page of apps.`)
+
+	cmd.Use = "list"
+	cmd.Short = `List Apps.`
+	cmd.Long = `List Apps.
   
-  List all available applications`
+  Lists all apps in the workspace.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
-		response, err := w.Apps.GetApps(ctx)
-		if err != nil {
-			return err
-		}
-		return cmdio.Render(ctx, response)
+
+		response := w.Apps.List(ctx, listReq)
+		return cmdio.RenderIterator(ctx, response)
 	}
 
 	// Disable completions since they are not applicable.
@@ -320,37 +559,43 @@ func newGetApps() *cobra.Command {
 	cmd.ValidArgsFunction = cobra.NoFileCompletions
 
 	// Apply optional overrides to this command.
-	for _, fn := range getAppsOverrides {
-		fn(cmd)
+	for _, fn := range listOverrides {
+		fn(cmd, &listReq)
 	}
 
 	return cmd
 }
 
-// start get-events command
+// start list-deployments command
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
-var getEventsOverrides []func(
+var listDeploymentsOverrides []func(
 	*cobra.Command,
-	*serving.GetEventsRequest,
+	*serving.ListAppDeploymentsRequest,
 )
 
-func newGetEvents() *cobra.Command {
+func newListDeployments() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getEventsReq serving.GetEventsRequest
+	var listDeploymentsReq serving.ListAppDeploymentsRequest
 
 	// TODO: short flags
 
-	cmd.Use = "get-events NAME"
-	cmd.Short = `Get deployment events for an application.`
-	cmd.Long = `Get deployment events for an application.
+	cmd.Flags().IntVar(&listDeploymentsReq.PageSize, "page-size", listDeploymentsReq.PageSize, `Upper bound for items returned.`)
+	cmd.Flags().StringVar(&listDeploymentsReq.PageToken, "page-token", listDeploymentsReq.PageToken, `Pagination token to go to the next page of apps.`)
+
+	cmd.Use = "list-deployments APP_NAME"
+	cmd.Short = `List App Deployments.`
+	cmd.Long = `List App Deployments.
   
-  Get deployment events for an application
+  Lists all app deployments for the app with the supplied name.
 
   Arguments:
-    NAME: The name of an application. This field is required.`
+    APP_NAME: The name of the app.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
 
 	cmd.Annotations = make(map[string]string)
 
@@ -364,9 +609,140 @@ func newGetEvents() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		getEventsReq.Name = args[0]
+		listDeploymentsReq.AppName = args[0]
 
-		response, err := w.Apps.GetEvents(ctx, getEventsReq)
+		response := w.Apps.ListDeployments(ctx, listDeploymentsReq)
+		return cmdio.RenderIterator(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range listDeploymentsOverrides {
+		fn(cmd, &listDeploymentsReq)
+	}
+
+	return cmd
+}
+
+// start stop command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var stopOverrides []func(
+	*cobra.Command,
+	*serving.StopAppRequest,
+)
+
+func newStop() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var stopReq serving.StopAppRequest
+
+	// TODO: short flags
+
+	cmd.Use = "stop NAME"
+	cmd.Short = `Stop an App.`
+	cmd.Long = `Stop an App.
+  
+  Stops the active deployment of the app in the workspace.
+
+  Arguments:
+    NAME: The name of the app.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		stopReq.Name = args[0]
+
+		err = w.Apps.Stop(ctx, stopReq)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range stopOverrides {
+		fn(cmd, &stopReq)
+	}
+
+	return cmd
+}
+
+// start update command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var updateOverrides []func(
+	*cobra.Command,
+	*serving.UpdateAppRequest,
+)
+
+func newUpdate() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var updateReq serving.UpdateAppRequest
+	var updateJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`)
+
+	cmd.Use = "update NAME"
+	cmd.Short = `Update an App.`
+	cmd.Long = `Update an App.
+  
+  Updates the app with the supplied name.
+
+  Arguments:
+    NAME: The name of the app. The name must contain only lowercase alphanumeric
+      characters and hyphens and be between 2 and 30 characters long. It must be
+      unique within the workspace.`
+
+	// This command is being previewed; hide from help output.
+	cmd.Hidden = true
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			err = updateJson.Unmarshal(&updateReq)
+			if err != nil {
+				return err
+			}
+		}
+		updateReq.Name = args[0]
+
+		response, err := w.Apps.Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
@@ -378,8 +754,8 @@ func newGetEvents() *cobra.Command {
 	cmd.ValidArgsFunction = cobra.NoFileCompletions
 
 	// Apply optional overrides to this command.
-	for _, fn := range getEventsOverrides {
-		fn(cmd, &getEventsReq)
+	for _, fn := range updateOverrides {
+		fn(cmd, &updateReq)
 	}
 
 	return cmd
diff --git a/cmd/workspace/apps/overrides.go b/cmd/workspace/apps/overrides.go
deleted file mode 100644
index e38e139b5..000000000
--- a/cmd/workspace/apps/overrides.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package apps
-
-import (
-	"fmt"
-
-	"github.com/databricks/cli/cmd/root"
-	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/flags"
-	"github.com/databricks/databricks-sdk-go/service/serving"
-	"github.com/spf13/cobra"
-)
-
-func createOverride(cmd *cobra.Command, deployReq *serving.DeployAppRequest) {
-	var manifestYaml flags.YamlFlag
-	var resourcesYaml flags.YamlFlag
-	createJson := cmd.Flag("json").Value.(*flags.JsonFlag)
-
-	// TODO: short flags
-	cmd.Flags().Var(&manifestYaml, "manifest", `either inline YAML string or @path/to/manifest.yaml`)
-	cmd.Flags().Var(&resourcesYaml, "resources", `either inline YAML string or @path/to/resources.yaml`)
-
-	cmd.Annotations = make(map[string]string)
-
-	cmd.PreRunE = root.MustWorkspaceClient
-	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
-		ctx := cmd.Context()
-		w := root.WorkspaceClient(ctx)
-		if cmd.Flags().Changed("json") {
-			err = createJson.Unmarshal(&deployReq)
-			if err != nil {
-				return err
-			}
-		} else if cmd.Flags().Changed("manifest") {
-			err = manifestYaml.Unmarshal(&deployReq.Manifest)
-			if err != nil {
-				return err
-			}
-			if cmd.Flags().Changed("resources") {
-				err = resourcesYaml.Unmarshal(&deployReq.Resources)
-				if err != nil {
-					return err
-				}
-			}
-		} else {
-			return fmt.Errorf("please provide command input in YAML format by specifying the --manifest flag or provide a json payload using the --json flag")
-		}
-		response, err := w.Apps.Create(ctx, *deployReq)
-		if err != nil {
-			return err
-		}
-
-		return cmdio.Render(ctx, response)
-	}
-}
-
-func init() {
-	createOverrides = append(createOverrides, createOverride)
-}
diff --git a/cmd/workspace/csp-enablement/csp-enablement.go b/cmd/workspace/compliance-security-profile/compliance-security-profile.go
similarity index 89%
rename from cmd/workspace/csp-enablement/csp-enablement.go
rename to cmd/workspace/compliance-security-profile/compliance-security-profile.go
index e82fdc2a4..efafb4627 100755
--- a/cmd/workspace/csp-enablement/csp-enablement.go
+++ b/cmd/workspace/compliance-security-profile/compliance-security-profile.go
@@ -1,6 +1,6 @@
 // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
 
-package csp_enablement
+package compliance_security_profile
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command)
 
 func New() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "csp-enablement",
+		Use:   "compliance-security-profile",
 		Short: `Controls whether to enable the compliance security profile for the current workspace.`,
 		Long: `Controls whether to enable the compliance security profile for the current
   workspace. Enabling it on a workspace is permanent. By default, it is turned
@@ -48,13 +48,13 @@ func New() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var getOverrides []func(
 	*cobra.Command,
-	*settings.GetCspEnablementSettingRequest,
+	*settings.GetComplianceSecurityProfileSettingRequest,
 )
 
 func newGet() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getReq settings.GetCspEnablementSettingRequest
+	var getReq settings.GetComplianceSecurityProfileSettingRequest
 
 	// TODO: short flags
 
@@ -78,7 +78,7 @@ func newGet() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		response, err := w.Settings.CspEnablement().Get(ctx, getReq)
+		response, err := w.Settings.ComplianceSecurityProfile().Get(ctx, getReq)
 		if err != nil {
 			return err
 		}
@@ -103,13 +103,13 @@ func newGet() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var updateOverrides []func(
 	*cobra.Command,
-	*settings.UpdateCspEnablementSettingRequest,
+	*settings.UpdateComplianceSecurityProfileSettingRequest,
 )
 
 func newUpdate() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var updateReq settings.UpdateCspEnablementSettingRequest
+	var updateReq settings.UpdateComplianceSecurityProfileSettingRequest
 	var updateJson flags.JsonFlag
 
 	// TODO: short flags
@@ -141,7 +141,7 @@ func newUpdate() *cobra.Command {
 			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 		}
 
-		response, err := w.Settings.CspEnablement().Update(ctx, updateReq)
+		response, err := w.Settings.ComplianceSecurityProfile().Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
@@ -160,4 +160,4 @@ func newUpdate() *cobra.Command {
 	return cmd
 }
 
-// end service CSPEnablement
+// end service ComplianceSecurityProfile
diff --git a/cmd/workspace/dashboards/dashboards.go b/cmd/workspace/dashboards/dashboards.go
index 0500ebecf..1a143538b 100755
--- a/cmd/workspace/dashboards/dashboards.go
+++ b/cmd/workspace/dashboards/dashboards.go
@@ -386,6 +386,7 @@ func newUpdate() *cobra.Command {
 
 	cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this dashboard that appears in list views and at the top of the dashboard page.`)
 	cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`)
+	// TODO: array: tags
 
 	cmd.Use = "update DASHBOARD_ID"
 	cmd.Short = `Change a dashboard definition.`
diff --git a/cmd/workspace/esm-enablement/esm-enablement.go b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
similarity index 89%
rename from cmd/workspace/esm-enablement/esm-enablement.go
rename to cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
index 784c01f21..86b4244d5 100755
--- a/cmd/workspace/esm-enablement/esm-enablement.go
+++ b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
@@ -1,6 +1,6 @@
 // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
 
-package esm_enablement
+package enhanced_security_monitoring
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command)
 
 func New() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "esm-enablement",
+		Use:   "enhanced-security-monitoring",
 		Short: `Controls whether enhanced security monitoring is enabled for the current workspace.`,
 		Long: `Controls whether enhanced security monitoring is enabled for the current
   workspace. If the compliance security profile is enabled, this is
@@ -50,13 +50,13 @@ func New() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var getOverrides []func(
 	*cobra.Command,
-	*settings.GetEsmEnablementSettingRequest,
+	*settings.GetEnhancedSecurityMonitoringSettingRequest,
 )
 
 func newGet() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getReq settings.GetEsmEnablementSettingRequest
+	var getReq settings.GetEnhancedSecurityMonitoringSettingRequest
 
 	// TODO: short flags
 
@@ -80,7 +80,7 @@ func newGet() *cobra.Command {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
 
-		response, err := w.Settings.EsmEnablement().Get(ctx, getReq)
+		response, err := w.Settings.EnhancedSecurityMonitoring().Get(ctx, getReq)
 		if err != nil {
 			return err
 		}
@@ -105,13 +105,13 @@ func newGet() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var updateOverrides []func(
 	*cobra.Command,
-	*settings.UpdateEsmEnablementSettingRequest,
+	*settings.UpdateEnhancedSecurityMonitoringSettingRequest,
 )
 
 func newUpdate() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var updateReq settings.UpdateEsmEnablementSettingRequest
+	var updateReq settings.UpdateEnhancedSecurityMonitoringSettingRequest
 	var updateJson flags.JsonFlag
 
 	// TODO: short flags
@@ -143,7 +143,7 @@ func newUpdate() *cobra.Command {
 			return fmt.Errorf("please provide command input in JSON format by specifying the --json flag")
 		}
 
-		response, err := w.Settings.EsmEnablement().Update(ctx, updateReq)
+		response, err := w.Settings.EnhancedSecurityMonitoring().Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
@@ -162,4 +162,4 @@ func newUpdate() *cobra.Command {
 	return cmd
 }
 
-// end service ESMEnablement
+// end service EnhancedSecurityMonitoring
diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go
index 7b556c724..034cea2df 100755
--- a/cmd/workspace/model-versions/model-versions.go
+++ b/cmd/workspace/model-versions/model-versions.go
@@ -288,6 +288,7 @@ func newList() *cobra.Command {
   schema.
   
   There is no guarantee of a specific ordering of the elements in the response.
+  The elements in the response will not contain any aliases or tags.
 
   Arguments:
     FULL_NAME: The full three-level name of the registered model under which to list
diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go
index 0126097fc..b96eb7154 100755
--- a/cmd/workspace/queries/queries.go
+++ b/cmd/workspace/queries/queries.go
@@ -401,6 +401,7 @@ func newUpdate() *cobra.Command {
 	// TODO: any: options
 	cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`)
 	cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`)
+	// TODO: array: tags
 
 	cmd.Use = "update QUERY_ID"
 	cmd.Short = `Change a query definition.`
diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go
index 38e19e839..214986c76 100755
--- a/cmd/workspace/settings/settings.go
+++ b/cmd/workspace/settings/settings.go
@@ -6,9 +6,9 @@ import (
 	"github.com/spf13/cobra"
 
 	automatic_cluster_update "github.com/databricks/cli/cmd/workspace/automatic-cluster-update"
-	csp_enablement "github.com/databricks/cli/cmd/workspace/csp-enablement"
+	compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile"
 	default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace"
-	esm_enablement "github.com/databricks/cli/cmd/workspace/esm-enablement"
+	enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring"
 	restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins"
 )
 
@@ -29,9 +29,9 @@ func New() *cobra.Command {
 
 	// Add subservices
 	cmd.AddCommand(automatic_cluster_update.New())
-	cmd.AddCommand(csp_enablement.New())
+	cmd.AddCommand(compliance_security_profile.New())
 	cmd.AddCommand(default_namespace.New())
-	cmd.AddCommand(esm_enablement.New())
+	cmd.AddCommand(enhanced_security_monitoring.New())
 	cmd.AddCommand(restrict_workspace_admins.New())
 
 	// Apply optional overrides to this command.
diff --git a/go.mod b/go.mod
index 636fbf44a..6c8e845a5 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/semver/v3 v3.2.1 // MIT
 	github.com/briandowns/spinner v1.23.0 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.39.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.40.1 // Apache 2.0
 	github.com/fatih/color v1.16.0 // MIT
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
@@ -57,8 +57,8 @@ require (
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/metric v1.24.0 // indirect
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/api v0.169.0 // indirect
diff --git a/go.sum b/go.sum
index 3dd6b0cb7..222ce1e4c 100644
--- a/go.sum
+++ b/go.sum
@@ -28,8 +28,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.39.0 h1:nVnQYkk47SkEsRSXWkn6j7jBOxXgusjoo6xwbaHTGss=
-github.com/databricks/databricks-sdk-go v0.39.0/go.mod h1:Yjy1gREDLK65g4axpVbVNKYAHYE2Sqzj0AB9QWHCBVM=
+github.com/databricks/databricks-sdk-go v0.40.1 h1:rE5yP9gIW2oap+6CnumixnZSDIsXwVojAuDBuKUl5GU=
+github.com/databricks/databricks-sdk-go v0.40.1/go.mod h1:rLIhh7DvifVLmf2QxMr/vMRGqdrTZazn8VYo4LilfCo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -170,8 +170,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y
 go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
@@ -186,8 +186,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
 golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=

From 157877a152cf157d4a40244b5ecadbc2485e9539 Mon Sep 17 00:00:00 2001
From: Ilia Babanov <ilia.babanov@databricks.com>
Date: Thu, 16 May 2024 11:32:55 +0200
Subject: [PATCH 21/41] Fix bundle destroy integration test (#1435)

I've updated the `deploy_then_remove_resources` test template in the
previous PR, but didn't notice that it was used in the destroy test too.
Now destroy test also checks deletion of jobs
---
 internal/bundle/destroy_test.go | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/internal/bundle/destroy_test.go b/internal/bundle/destroy_test.go
index 43c05fbae..baccf4e6f 100644
--- a/internal/bundle/destroy_test.go
+++ b/internal/bundle/destroy_test.go
@@ -6,7 +6,9 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/internal"
 	"github.com/databricks/cli/internal/acc"
+	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
@@ -17,9 +19,12 @@ func TestAccBundleDestroy(t *testing.T) {
 	ctx, wt := acc.WorkspaceTest(t)
 	w := wt.W
 
+	nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV"))
 	uniqueId := uuid.New().String()
 	bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{
-		"unique_id": uniqueId,
+		"unique_id":     uniqueId,
+		"node_type_id":  nodeTypeId,
+		"spark_version": defaultSparkVersion,
 	})
 	require.NoError(t, err)
 
@@ -29,7 +34,7 @@ func TestAccBundleDestroy(t *testing.T) {
 	_, err = os.ReadDir(snapshotsDir)
 	assert.ErrorIs(t, err, os.ErrNotExist)
 
-	// deploy pipeline
+	// deploy resources
 	err = deployBundle(t, ctx, bundleRoot)
 	require.NoError(t, err)
 
@@ -49,6 +54,12 @@ func TestAccBundleDestroy(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, pipeline.Name, pipelineName)
 
+	// assert job is created
+	jobName := "test-bundle-job-" + uniqueId
+	job, err := w.Jobs.GetBySettingsName(ctx, jobName)
+	require.NoError(t, err)
+	assert.Equal(t, job.Settings.Name, jobName)
+
 	// destroy bundle
 	err = destroyBundle(t, ctx, bundleRoot)
 	require.NoError(t, err)
@@ -57,6 +68,10 @@ func TestAccBundleDestroy(t *testing.T) {
 	_, err = w.Pipelines.GetByName(ctx, pipelineName)
 	assert.ErrorContains(t, err, "does not exist")
 
+	// assert job is deleted
+	_, err = w.Jobs.GetBySettingsName(ctx, jobName)
+	assert.ErrorContains(t, err, "does not exist")
+
 	// Assert snapshot file is deleted
 	entries, err = os.ReadDir(snapshotsDir)
 	require.NoError(t, err)

From f7d4b272f40b384061cd2a52bab2ef943e3f9578 Mon Sep 17 00:00:00 2001
From: Miles Yucht <miles@databricks.com>
Date: Thu, 16 May 2024 12:22:09 +0200
Subject: [PATCH 22/41] Improve token refresh flow (#1434)

## Changes
Currently, there are a number of issues with the non-happy-path flows
for token refresh in the CLI.

If the token refresh fails, the raw error message is presented to the
user, as seen below. This message is very difficult for users to
interpret and doesn't give any clear direction on how to resolve this
issue.
```
Error: token refresh: Post "https://adb-<WSID>.azuredatabricks.net/oidc/v1/token": http 400: {"error":"invalid_request","error_description":"Refresh token is invalid"}
```

When logging in again, I've noticed that the timeout for logging in is
very short, only 45 seconds. If a user is using a password manager and
needs to login to that first, or needs to do MFA, 45 seconds may not be
enough time. to an account-level profile, it is quite frustrating for
users to need to re-enter account ID information when that information
is already stored in the user's `.databrickscfg` file.

This PR tackles these two issues. First, the presentation of error
messages from `databricks auth token` is improved substantially by
converting the `error` into a human-readable message. When the refresh
token is invalid, it will present a command for the user to run to
reauthenticate. If the token fetching failed for some other reason, that
reason will be presented in a nice way, providing front-line debugging
steps and ultimately redirecting users to file a ticket at this repo if
they can't resolve the issue themselves. After this PR, the new error
message is:
```
Error: a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run `.databricks/databricks auth login --host https://adb-<WSID>.azuredatabricks.net`
```

To improve the login flow, this PR modifies `databricks auth login` to
auto-complete the account ID from the profile when present.
Additionally, it increases the login timeout from 45 seconds to 1 hour
to give the user sufficient time to login as needed.

To test this change, I needed to refactor some components of the CLI
around profile management, the token cache, and the API client used to
fetch OAuth tokens. These are now settable in the context, and a
demonstration of how they can be set and used is found in
`auth_test.go`.

Separately, this also demonstrates a sort-of integration test of the CLI
by executing the Cobra command for `databricks auth token` from tests,
which may be useful for testing other end-to-end functionality in the
CLI. In particular, I believe this is necessary in order to set flag
values (like the `--profile` flag in this case) for use in testing.

## Tests
Unit tests cover the unhappy and happy paths using the mocked API
client, token cache, and profiler.

Manually tested

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 bundle/tests/environment_git_test.go          |   8 +-
 bundle/tests/git_test.go                      |   8 +-
 cmd/auth/env.go                               |   4 +-
 cmd/auth/login.go                             |  41 +++--
 cmd/auth/login_test.go                        |   2 +-
 cmd/auth/profiles.go                          |   4 +-
 cmd/auth/token.go                             |  55 +++++-
 cmd/auth/token_test.go                        | 168 ++++++++++++++++++
 cmd/labs/project/installer.go                 |   4 +-
 cmd/root/auth.go                              |  28 +--
 libs/auth/cache/cache.go                      | 104 ++---------
 libs/auth/cache/file.go                       | 108 +++++++++++
 .../cache/{cache_test.go => file_test.go}     |  14 +-
 libs/auth/cache/in_memory.go                  |  26 +++
 libs/auth/cache/in_memory_test.go             |  44 +++++
 libs/auth/oauth.go                            |  31 ++--
 libs/databrickscfg/loader_test.go             |  10 +-
 libs/databrickscfg/ops_test.go                |  14 +-
 libs/databrickscfg/profile/context.go         |  17 ++
 libs/databrickscfg/profile/file.go            | 100 +++++++++++
 .../file_test.go}                             |  20 ++-
 libs/databrickscfg/profile/in_memory.go       |  25 +++
 libs/databrickscfg/profile/profile.go         |  49 +++++
 libs/databrickscfg/profile/profiler.go        |  32 ++++
 .../{ => profile}/testdata/badcfg             |   0
 .../{ => profile}/testdata/databrickscfg      |   0
 .../testdata/sample-home/.databrickscfg       |   0
 libs/databrickscfg/profiles.go                | 150 ----------------
 28 files changed, 743 insertions(+), 323 deletions(-)
 create mode 100644 cmd/auth/token_test.go
 create mode 100644 libs/auth/cache/file.go
 rename libs/auth/cache/{cache_test.go => file_test.go} (93%)
 create mode 100644 libs/auth/cache/in_memory.go
 create mode 100644 libs/auth/cache/in_memory_test.go
 create mode 100644 libs/databrickscfg/profile/context.go
 create mode 100644 libs/databrickscfg/profile/file.go
 rename libs/databrickscfg/{profiles_test.go => profile/file_test.go} (82%)
 create mode 100644 libs/databrickscfg/profile/in_memory.go
 create mode 100644 libs/databrickscfg/profile/profile.go
 create mode 100644 libs/databrickscfg/profile/profiler.go
 rename libs/databrickscfg/{ => profile}/testdata/badcfg (100%)
 rename libs/databrickscfg/{ => profile}/testdata/databrickscfg (100%)
 rename libs/databrickscfg/{ => profile}/testdata/sample-home/.databrickscfg (100%)
 delete mode 100644 libs/databrickscfg/profiles.go

diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go
index bb10825e4..ad4aec2e6 100644
--- a/bundle/tests/environment_git_test.go
+++ b/bundle/tests/environment_git_test.go
@@ -1,6 +1,8 @@
 package config_tests
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -9,12 +11,14 @@ import (
 func TestGitAutoLoadWithEnvironment(t *testing.T) {
 	b := load(t, "./environments_autoload_git")
 	assert.True(t, b.Config.Bundle.Git.Inferred)
-	assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
+	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
+	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
 }
 
 func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
 	b := loadTarget(t, "./environments_autoload_git", "production")
 	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
-	assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
+	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
+	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
 }
diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go
index b33ffc211..21eaaedd2 100644
--- a/bundle/tests/git_test.go
+++ b/bundle/tests/git_test.go
@@ -2,6 +2,8 @@ package config_tests
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/databricks/cli/bundle"
@@ -13,14 +15,16 @@ import (
 func TestGitAutoLoad(t *testing.T) {
 	b := load(t, "./autoload_git")
 	assert.True(t, b.Config.Bundle.Git.Inferred)
-	assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
+	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
+	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
 }
 
 func TestGitManuallySetBranch(t *testing.T) {
 	b := loadTarget(t, "./autoload_git", "production")
 	assert.False(t, b.Config.Bundle.Git.Inferred)
 	assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
-	assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
+	validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
+	assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
 }
 
 func TestGitBundleBranchValidation(t *testing.T) {
diff --git a/cmd/auth/env.go b/cmd/auth/env.go
index 04aef36a8..e72d15399 100644
--- a/cmd/auth/env.go
+++ b/cmd/auth/env.go
@@ -10,7 +10,7 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/databricks/cli/libs/databrickscfg"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/spf13/cobra"
 	"gopkg.in/ini.v1"
@@ -70,7 +70,7 @@ func resolveSection(cfg *config.Config, iniFile *config.File) (*ini.Section, err
 }
 
 func loadFromDatabricksCfg(ctx context.Context, cfg *config.Config) error {
-	iniFile, err := databrickscfg.Get(ctx)
+	iniFile, err := profile.DefaultProfiler.Get(ctx)
 	if errors.Is(err, fs.ErrNotExist) {
 		// it's fine not to have ~/.databrickscfg
 		return nil
diff --git a/cmd/auth/login.go b/cmd/auth/login.go
index c033054b8..11cba8e5f 100644
--- a/cmd/auth/login.go
+++ b/cmd/auth/login.go
@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/databrickscfg"
 	"github.com/databricks/cli/libs/databrickscfg/cfgpickers"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/spf13/cobra"
@@ -31,6 +32,7 @@ func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, arg
 }
 
 const minimalDbConnectVersion = "13.1"
+const defaultTimeout = 1 * time.Hour
 
 func newLoginCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 	defaultConfigPath := "~/.databrickscfg"
@@ -84,7 +86,7 @@ depends on the existing profiles you have set in your configuration file
 
 	var loginTimeout time.Duration
 	var configureCluster bool
-	cmd.Flags().DurationVar(&loginTimeout, "timeout", auth.DefaultTimeout,
+	cmd.Flags().DurationVar(&loginTimeout, "timeout", defaultTimeout,
 		"Timeout for completing login challenge in the browser")
 	cmd.Flags().BoolVar(&configureCluster, "configure-cluster", false,
 		"Prompts to configure cluster")
@@ -108,7 +110,7 @@ depends on the existing profiles you have set in your configuration file
 			profileName = profile
 		}
 
-		err := setHost(ctx, profileName, persistentAuth, args)
+		err := setHostAndAccountId(ctx, profileName, persistentAuth, args)
 		if err != nil {
 			return err
 		}
@@ -117,17 +119,10 @@ depends on the existing profiles you have set in your configuration file
 		// We need the config without the profile before it's used to initialise new workspace client below.
 		// Otherwise it will complain about non existing profile because it was not yet saved.
 		cfg := config.Config{
-			Host:     persistentAuth.Host,
-			AuthType: "databricks-cli",
+			Host:      persistentAuth.Host,
+			AccountID: persistentAuth.AccountID,
+			AuthType:  "databricks-cli",
 		}
-		if cfg.IsAccountClient() && persistentAuth.AccountID == "" {
-			accountId, err := promptForAccountID(ctx)
-			if err != nil {
-				return err
-			}
-			persistentAuth.AccountID = accountId
-		}
-		cfg.AccountID = persistentAuth.AccountID
 
 		ctx, cancel := context.WithTimeout(ctx, loginTimeout)
 		defer cancel()
@@ -172,15 +167,15 @@ depends on the existing profiles you have set in your configuration file
 	return cmd
 }
 
-func setHost(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error {
+func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error {
+	profiler := profile.GetProfiler(ctx)
 	// If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile.
-	_, profiles, err := databrickscfg.LoadProfiles(ctx, func(p databrickscfg.Profile) bool {
-		return p.Name == profileName
-	})
+	profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName))
 	// Tolerate ErrNoConfiguration here, as we will write out a configuration as part of the login flow.
-	if err != nil && !errors.Is(err, databrickscfg.ErrNoConfiguration) {
+	if err != nil && !errors.Is(err, profile.ErrNoConfiguration) {
 		return err
 	}
+
 	if persistentAuth.Host == "" {
 		if len(profiles) > 0 && profiles[0].Host != "" {
 			persistentAuth.Host = profiles[0].Host
@@ -188,5 +183,17 @@ func setHost(ctx context.Context, profileName string, persistentAuth *auth.Persi
 			configureHost(ctx, persistentAuth, args, 0)
 		}
 	}
+	isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient()
+	if isAccountClient && persistentAuth.AccountID == "" {
+		if len(profiles) > 0 && profiles[0].AccountID != "" {
+			persistentAuth.AccountID = profiles[0].AccountID
+		} else {
+			accountId, err := promptForAccountID(ctx)
+			if err != nil {
+				return err
+			}
+			persistentAuth.AccountID = accountId
+		}
+	}
 	return nil
 }
diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go
index 9b834bd0a..ce3ca5ae5 100644
--- a/cmd/auth/login_test.go
+++ b/cmd/auth/login_test.go
@@ -12,6 +12,6 @@ import (
 func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./imaginary-file/databrickscfg")
-	err := setHost(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{})
+	err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{})
 	assert.NoError(t, err)
 }
diff --git a/cmd/auth/profiles.go b/cmd/auth/profiles.go
index 797eb3b5f..61a6c1f33 100644
--- a/cmd/auth/profiles.go
+++ b/cmd/auth/profiles.go
@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/databrickscfg"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/config"
@@ -94,7 +94,7 @@ func newProfilesCommand() *cobra.Command {
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		var profiles []*profileMetadata
-		iniFile, err := databrickscfg.Get(cmd.Context())
+		iniFile, err := profile.DefaultProfiler.Get(cmd.Context())
 		if os.IsNotExist(err) {
 			// return empty list for non-configured machines
 			iniFile = &config.File{
diff --git a/cmd/auth/token.go b/cmd/auth/token.go
index d763b9564..3f9af43fa 100644
--- a/cmd/auth/token.go
+++ b/cmd/auth/token.go
@@ -4,12 +4,44 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/databricks/cli/libs/auth"
+	"github.com/databricks/databricks-sdk-go/httpclient"
 	"github.com/spf13/cobra"
 )
 
+type tokenErrorResponse struct {
+	Error            string `json:"error"`
+	ErrorDescription string `json:"error_description"`
+}
+
+func buildLoginCommand(profile string, persistentAuth *auth.PersistentAuth) string {
+	executable := os.Args[0]
+	cmd := []string{
+		executable,
+		"auth",
+		"login",
+	}
+	if profile != "" {
+		cmd = append(cmd, "--profile", profile)
+	} else {
+		cmd = append(cmd, "--host", persistentAuth.Host)
+		if persistentAuth.AccountID != "" {
+			cmd = append(cmd, "--account-id", persistentAuth.AccountID)
+		}
+	}
+	return strings.Join(cmd, " ")
+}
+
+func helpfulError(profile string, persistentAuth *auth.PersistentAuth) string {
+	loginMsg := buildLoginCommand(profile, persistentAuth)
+	return fmt.Sprintf("Try logging in again with `%s` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new", loginMsg)
+}
+
 func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "token [HOST]",
@@ -17,7 +49,7 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 	}
 
 	var tokenTimeout time.Duration
-	cmd.Flags().DurationVar(&tokenTimeout, "timeout", auth.DefaultTimeout,
+	cmd.Flags().DurationVar(&tokenTimeout, "timeout", defaultTimeout,
 		"Timeout for acquiring a token.")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
@@ -29,11 +61,11 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 			profileName = profileFlag.Value.String()
 			// If a profile is provided we read the host from the .databrickscfg file
 			if profileName != "" && len(args) > 0 {
-				return errors.New("providing both a profile and a host parameters is not supported")
+				return errors.New("providing both a profile and host is not supported")
 			}
 		}
 
-		err := setHost(ctx, profileName, persistentAuth, args)
+		err := setHostAndAccountId(ctx, profileName, persistentAuth, args)
 		if err != nil {
 			return err
 		}
@@ -42,8 +74,21 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
 		ctx, cancel := context.WithTimeout(ctx, tokenTimeout)
 		defer cancel()
 		t, err := persistentAuth.Load(ctx)
-		if err != nil {
-			return err
+		var httpErr *httpclient.HttpError
+		if errors.As(err, &httpErr) {
+			helpMsg := helpfulError(profileName, persistentAuth)
+			t := &tokenErrorResponse{}
+			err = json.Unmarshal([]byte(httpErr.Message), t)
+			if err != nil {
+				return fmt.Errorf("unexpected parsing token response: %w. %s", err, helpMsg)
+			}
+			if t.ErrorDescription == "Refresh token is invalid" {
+				return fmt.Errorf("a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run `%s`", buildLoginCommand(profileName, persistentAuth))
+			} else {
+				return fmt.Errorf("unexpected error refreshing token: %s. %s", t.ErrorDescription, helpMsg)
+			}
+		} else if err != nil {
+			return fmt.Errorf("unexpected error refreshing token: %w. %s", err, helpfulError(profileName, persistentAuth))
 		}
 		raw, err := json.MarshalIndent(t, "", "  ")
 		if err != nil {
diff --git a/cmd/auth/token_test.go b/cmd/auth/token_test.go
new file mode 100644
index 000000000..df98cc151
--- /dev/null
+++ b/cmd/auth/token_test.go
@@ -0,0 +1,168 @@
+package auth_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/cmd"
+	"github.com/databricks/cli/libs/auth"
+	"github.com/databricks/cli/libs/auth/cache"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
+	"github.com/databricks/databricks-sdk-go/httpclient"
+	"github.com/databricks/databricks-sdk-go/httpclient/fixtures"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/oauth2"
+)
+
+var refreshFailureTokenResponse = fixtures.HTTPFixture{
+	MatchAny: true,
+	Status:   401,
+	Response: map[string]string{
+		"error":             "invalid_request",
+		"error_description": "Refresh token is invalid",
+	},
+}
+
+var refreshFailureInvalidResponse = fixtures.HTTPFixture{
+	MatchAny: true,
+	Status:   401,
+	Response: "Not json",
+}
+
+var refreshFailureOtherError = fixtures.HTTPFixture{
+	MatchAny: true,
+	Status:   401,
+	Response: map[string]string{
+		"error":             "other_error",
+		"error_description": "Databricks is down",
+	},
+}
+
+var refreshSuccessTokenResponse = fixtures.HTTPFixture{
+	MatchAny: true,
+	Status:   200,
+	Response: map[string]string{
+		"access_token": "new-access-token",
+		"token_type":   "Bearer",
+		"expires_in":   "3600",
+	},
+}
+
+func validateToken(t *testing.T, resp string) {
+	res := map[string]string{}
+	err := json.Unmarshal([]byte(resp), &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "new-access-token", res["access_token"])
+	assert.Equal(t, "Bearer", res["token_type"])
+}
+
+func getContextForTest(f fixtures.HTTPFixture) context.Context {
+	profiler := profile.InMemoryProfiler{
+		Profiles: profile.Profiles{
+			{
+				Name:      "expired",
+				Host:      "https://accounts.cloud.databricks.com",
+				AccountID: "expired",
+			},
+			{
+				Name:      "active",
+				Host:      "https://accounts.cloud.databricks.com",
+				AccountID: "active",
+			},
+		},
+	}
+	tokenCache := &cache.InMemoryTokenCache{
+		Tokens: map[string]*oauth2.Token{
+			"https://accounts.cloud.databricks.com/oidc/accounts/expired": {
+				RefreshToken: "expired",
+			},
+			"https://accounts.cloud.databricks.com/oidc/accounts/active": {
+				RefreshToken: "active",
+				Expiry:       time.Now().Add(1 * time.Hour), // Hopefully unit tests don't take an hour to run
+			},
+		},
+	}
+	client := httpclient.NewApiClient(httpclient.ClientConfig{
+		Transport: fixtures.SliceTransport{f},
+	})
+	ctx := profile.WithProfiler(context.Background(), profiler)
+	ctx = cache.WithTokenCache(ctx, tokenCache)
+	ctx = auth.WithApiClientForOAuth(ctx, client)
+	return ctx
+}
+
+func getCobraCmdForTest(f fixtures.HTTPFixture) (*cobra.Command, *bytes.Buffer) {
+	ctx := getContextForTest(f)
+	c := cmd.New(ctx)
+	output := &bytes.Buffer{}
+	c.SetOut(output)
+	return c, output
+}
+
+func TestTokenCmdWithProfilePrintsHelpfulLoginMessageOnRefreshFailure(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshFailureTokenResponse)
+	cmd.SetArgs([]string{"auth", "token", "--profile", "expired"})
+	err := cmd.Execute()
+
+	out := output.String()
+	assert.Empty(t, out)
+	assert.ErrorContains(t, err, "a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run ")
+	assert.ErrorContains(t, err, "auth login --profile expired")
+}
+
+func TestTokenCmdWithHostPrintsHelpfulLoginMessageOnRefreshFailure(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshFailureTokenResponse)
+	cmd.SetArgs([]string{"auth", "token", "--host", "https://accounts.cloud.databricks.com", "--account-id", "expired"})
+	err := cmd.Execute()
+
+	out := output.String()
+	assert.Empty(t, out)
+	assert.ErrorContains(t, err, "a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run ")
+	assert.ErrorContains(t, err, "auth login --host https://accounts.cloud.databricks.com --account-id expired")
+}
+
+func TestTokenCmdInvalidResponse(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshFailureInvalidResponse)
+	cmd.SetArgs([]string{"auth", "token", "--profile", "active"})
+	err := cmd.Execute()
+
+	out := output.String()
+	assert.Empty(t, out)
+	assert.ErrorContains(t, err, "unexpected parsing token response: invalid character 'N' looking for beginning of value. Try logging in again with ")
+	assert.ErrorContains(t, err, "auth login --profile active` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new")
+}
+
+func TestTokenCmdOtherErrorResponse(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshFailureOtherError)
+	cmd.SetArgs([]string{"auth", "token", "--profile", "active"})
+	err := cmd.Execute()
+
+	out := output.String()
+	assert.Empty(t, out)
+	assert.ErrorContains(t, err, "unexpected error refreshing token: Databricks is down. Try logging in again with ")
+	assert.ErrorContains(t, err, "auth login --profile active` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new")
+}
+
+func TestTokenCmdWithProfileSuccess(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshSuccessTokenResponse)
+	cmd.SetArgs([]string{"auth", "token", "--profile", "active"})
+	err := cmd.Execute()
+
+	out := output.String()
+	validateToken(t, out)
+	assert.NoError(t, err)
+}
+
+func TestTokenCmdWithHostSuccess(t *testing.T) {
+	cmd, output := getCobraCmdForTest(refreshSuccessTokenResponse)
+	cmd.SetArgs([]string{"auth", "token", "--host", "https://accounts.cloud.databricks.com", "--account-id", "expired"})
+	err := cmd.Execute()
+
+	out := output.String()
+	validateToken(t, out)
+	assert.NoError(t, err)
+}
diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go
index 42c4a8496..92dfe9e7c 100644
--- a/cmd/labs/project/installer.go
+++ b/cmd/labs/project/installer.go
@@ -11,8 +11,8 @@ import (
 	"github.com/databricks/cli/cmd/labs/github"
 	"github.com/databricks/cli/cmd/labs/unpack"
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/databrickscfg"
 	"github.com/databricks/cli/libs/databrickscfg/cfgpickers"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/process"
 	"github.com/databricks/cli/libs/python"
@@ -89,7 +89,7 @@ func (i *installer) Install(ctx context.Context) error {
 		return err
 	}
 	w, err := i.login(ctx)
-	if err != nil && errors.Is(err, databrickscfg.ErrNoConfiguration) {
+	if err != nil && errors.Is(err, profile.ErrNoConfiguration) {
 		cfg, err := i.Installer.envAwareConfig(ctx)
 		if err != nil {
 			return err
diff --git a/cmd/root/auth.go b/cmd/root/auth.go
index 387b67f0d..107679105 100644
--- a/cmd/root/auth.go
+++ b/cmd/root/auth.go
@@ -7,7 +7,7 @@ import (
 	"net/http"
 
 	"github.com/databricks/cli/libs/cmdio"
-	"github.com/databricks/cli/libs/databrickscfg"
+	"github.com/databricks/cli/libs/databrickscfg/profile"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/config"
 	"github.com/manifoldco/promptui"
@@ -37,7 +37,7 @@ func (e ErrNoAccountProfiles) Error() string {
 
 func initProfileFlag(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringP("profile", "p", "", "~/.databrickscfg profile")
-	cmd.RegisterFlagCompletionFunc("profile", databrickscfg.ProfileCompletion)
+	cmd.RegisterFlagCompletionFunc("profile", profile.ProfileCompletion)
 }
 
 func profileFlagValue(cmd *cobra.Command) (string, bool) {
@@ -111,27 +111,29 @@ func MustAccountClient(cmd *cobra.Command, args []string) error {
 	cfg := &config.Config{}
 
 	// The command-line profile flag takes precedence over DATABRICKS_CONFIG_PROFILE.
-	profile, hasProfileFlag := profileFlagValue(cmd)
+	pr, hasProfileFlag := profileFlagValue(cmd)
 	if hasProfileFlag {
-		cfg.Profile = profile
+		cfg.Profile = pr
 	}
 
 	ctx := cmd.Context()
 	ctx = context.WithValue(ctx, &configUsed, cfg)
 	cmd.SetContext(ctx)
 
+	profiler := profile.GetProfiler(ctx)
+
 	if cfg.Profile == "" {
 		// account-level CLI was not really done before, so here are the assumptions:
 		// 1. only admins will have account configured
 		// 2. 99% of admins will have access to just one account
 		// hence, we don't need to create a special "DEFAULT_ACCOUNT" profile yet
-		_, profiles, err := databrickscfg.LoadProfiles(cmd.Context(), databrickscfg.MatchAccountProfiles)
+		profiles, err := profiler.LoadProfiles(cmd.Context(), profile.MatchAccountProfiles)
 		if err == nil && len(profiles) == 1 {
 			cfg.Profile = profiles[0].Name
 		}
 
 		// if there is no config file, we don't want to fail and instead just skip it
-		if err != nil && !errors.Is(err, databrickscfg.ErrNoConfiguration) {
+		if err != nil && !errors.Is(err, profile.ErrNoConfiguration) {
 			return err
 		}
 	}
@@ -233,11 +235,12 @@ func SetAccountClient(ctx context.Context, a *databricks.AccountClient) context.
 }
 
 func AskForWorkspaceProfile(ctx context.Context) (string, error) {
-	path, err := databrickscfg.GetPath(ctx)
+	profiler := profile.GetProfiler(ctx)
+	path, err := profiler.GetPath(ctx)
 	if err != nil {
 		return "", fmt.Errorf("cannot determine Databricks config file path: %w", err)
 	}
-	file, profiles, err := databrickscfg.LoadProfiles(ctx, databrickscfg.MatchWorkspaceProfiles)
+	profiles, err := profiler.LoadProfiles(ctx, profile.MatchWorkspaceProfiles)
 	if err != nil {
 		return "", err
 	}
@@ -248,7 +251,7 @@ func AskForWorkspaceProfile(ctx context.Context) (string, error) {
 		return profiles[0].Name, nil
 	}
 	i, _, err := cmdio.RunSelect(ctx, &promptui.Select{
-		Label:             fmt.Sprintf("Workspace profiles defined in %s", file),
+		Label:             fmt.Sprintf("Workspace profiles defined in %s", path),
 		Items:             profiles,
 		Searcher:          profiles.SearchCaseInsensitive,
 		StartInSearchMode: true,
@@ -266,11 +269,12 @@ func AskForWorkspaceProfile(ctx context.Context) (string, error) {
 }
 
 func AskForAccountProfile(ctx context.Context) (string, error) {
-	path, err := databrickscfg.GetPath(ctx)
+	profiler := profile.GetProfiler(ctx)
+	path, err := profiler.GetPath(ctx)
 	if err != nil {
 		return "", fmt.Errorf("cannot determine Databricks config file path: %w", err)
 	}
-	file, profiles, err := databrickscfg.LoadProfiles(ctx, databrickscfg.MatchAccountProfiles)
+	profiles, err := profiler.LoadProfiles(ctx, profile.MatchAccountProfiles)
 	if err != nil {
 		return "", err
 	}
@@ -281,7 +285,7 @@ func AskForAccountProfile(ctx context.Context) (string, error) {
 		return profiles[0].Name, nil
 	}
 	i, _, err := cmdio.RunSelect(ctx, &promptui.Select{
-		Label:             fmt.Sprintf("Account profiles defined in %s", file),
+		Label:             fmt.Sprintf("Account profiles defined in %s", path),
 		Items:             profiles,
 		Searcher:          profiles.SearchCaseInsensitive,
 		StartInSearchMode: true,
diff --git a/libs/auth/cache/cache.go b/libs/auth/cache/cache.go
index 5511c1922..097353e74 100644
--- a/libs/auth/cache/cache.go
+++ b/libs/auth/cache/cache.go
@@ -1,106 +1,26 @@
 package cache
 
 import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
+	"context"
 
 	"golang.org/x/oauth2"
 )
 
-const (
-	// where the token cache is stored
-	tokenCacheFile = ".databricks/token-cache.json"
-
-	// only the owner of the file has full execute, read, and write access
-	ownerExecReadWrite = 0o700
-
-	// only the owner of the file has full read and write access
-	ownerReadWrite = 0o600
-
-	// format versioning leaves some room for format improvement
-	tokenCacheVersion = 1
-)
-
-var ErrNotConfigured = errors.New("databricks OAuth is not configured for this host")
-
-// this implementation requires the calling code to do a machine-wide lock,
-// otherwise the file might get corrupt.
-type TokenCache struct {
-	Version int                      `json:"version"`
-	Tokens  map[string]*oauth2.Token `json:"tokens"`
-
-	fileLocation string
+type TokenCache interface {
+	Store(key string, t *oauth2.Token) error
+	Lookup(key string) (*oauth2.Token, error)
 }
 
-func (c *TokenCache) Store(key string, t *oauth2.Token) error {
-	err := c.load()
-	if errors.Is(err, fs.ErrNotExist) {
-		dir := filepath.Dir(c.fileLocation)
-		err = os.MkdirAll(dir, ownerExecReadWrite)
-		if err != nil {
-			return fmt.Errorf("mkdir: %w", err)
-		}
-	} else if err != nil {
-		return fmt.Errorf("load: %w", err)
-	}
-	c.Version = tokenCacheVersion
-	if c.Tokens == nil {
-		c.Tokens = map[string]*oauth2.Token{}
-	}
-	c.Tokens[key] = t
-	raw, err := json.MarshalIndent(c, "", "  ")
-	if err != nil {
-		return fmt.Errorf("marshal: %w", err)
-	}
-	return os.WriteFile(c.fileLocation, raw, ownerReadWrite)
+var tokenCache int
+
+func WithTokenCache(ctx context.Context, c TokenCache) context.Context {
+	return context.WithValue(ctx, &tokenCache, c)
 }
 
-func (c *TokenCache) Lookup(key string) (*oauth2.Token, error) {
-	err := c.load()
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil, ErrNotConfigured
-	} else if err != nil {
-		return nil, fmt.Errorf("load: %w", err)
-	}
-	t, ok := c.Tokens[key]
+func GetTokenCache(ctx context.Context) TokenCache {
+	c, ok := ctx.Value(&tokenCache).(TokenCache)
 	if !ok {
-		return nil, ErrNotConfigured
+		return &FileTokenCache{}
 	}
-	return t, nil
-}
-
-func (c *TokenCache) location() (string, error) {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return "", fmt.Errorf("home: %w", err)
-	}
-	return filepath.Join(home, tokenCacheFile), nil
-}
-
-func (c *TokenCache) load() error {
-	loc, err := c.location()
-	if err != nil {
-		return err
-	}
-	c.fileLocation = loc
-	raw, err := os.ReadFile(loc)
-	if err != nil {
-		return fmt.Errorf("read: %w", err)
-	}
-	err = json.Unmarshal(raw, c)
-	if err != nil {
-		return fmt.Errorf("parse: %w", err)
-	}
-	if c.Version != tokenCacheVersion {
-		// in the later iterations we could do state upgraders,
-		// so that we transform token cache from v1 to v2 without
-		// losing the tokens and asking the user to re-authenticate.
-		return fmt.Errorf("needs version %d, got version %d",
-			tokenCacheVersion, c.Version)
-	}
-	return nil
+	return c
 }
diff --git a/libs/auth/cache/file.go b/libs/auth/cache/file.go
new file mode 100644
index 000000000..38dfea9f2
--- /dev/null
+++ b/libs/auth/cache/file.go
@@ -0,0 +1,108 @@
+package cache
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/oauth2"
+)
+
+const (
+	// where the token cache is stored
+	tokenCacheFile = ".databricks/token-cache.json"
+
+	// only the owner of the file has full execute, read, and write access
+	ownerExecReadWrite = 0o700
+
+	// only the owner of the file has full read and write access
+	ownerReadWrite = 0o600
+
+	// format versioning leaves some room for format improvement
+	tokenCacheVersion = 1
+)
+
+var ErrNotConfigured = errors.New("databricks OAuth is not configured for this host")
+
+// this implementation requires the calling code to do a machine-wide lock,
+// otherwise the file might get corrupt.
+type FileTokenCache struct {
+	Version int                      `json:"version"`
+	Tokens  map[string]*oauth2.Token `json:"tokens"`
+
+	fileLocation string
+}
+
+func (c *FileTokenCache) Store(key string, t *oauth2.Token) error {
+	err := c.load()
+	if errors.Is(err, fs.ErrNotExist) {
+		dir := filepath.Dir(c.fileLocation)
+		err = os.MkdirAll(dir, ownerExecReadWrite)
+		if err != nil {
+			return fmt.Errorf("mkdir: %w", err)
+		}
+	} else if err != nil {
+		return fmt.Errorf("load: %w", err)
+	}
+	c.Version = tokenCacheVersion
+	if c.Tokens == nil {
+		c.Tokens = map[string]*oauth2.Token{}
+	}
+	c.Tokens[key] = t
+	raw, err := json.MarshalIndent(c, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal: %w", err)
+	}
+	return os.WriteFile(c.fileLocation, raw, ownerReadWrite)
+}
+
+func (c *FileTokenCache) Lookup(key string) (*oauth2.Token, error) {
+	err := c.load()
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, ErrNotConfigured
+	} else if err != nil {
+		return nil, fmt.Errorf("load: %w", err)
+	}
+	t, ok := c.Tokens[key]
+	if !ok {
+		return nil, ErrNotConfigured
+	}
+	return t, nil
+}
+
+func (c *FileTokenCache) location() (string, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", fmt.Errorf("home: %w", err)
+	}
+	return filepath.Join(home, tokenCacheFile), nil
+}
+
+func (c *FileTokenCache) load() error {
+	loc, err := c.location()
+	if err != nil {
+		return err
+	}
+	c.fileLocation = loc
+	raw, err := os.ReadFile(loc)
+	if err != nil {
+		return fmt.Errorf("read: %w", err)
+	}
+	err = json.Unmarshal(raw, c)
+	if err != nil {
+		return fmt.Errorf("parse: %w", err)
+	}
+	if c.Version != tokenCacheVersion {
+		// in the later iterations we could do state upgraders,
+		// so that we transform token cache from v1 to v2 without
+		// losing the tokens and asking the user to re-authenticate.
+		return fmt.Errorf("needs version %d, got version %d",
+			tokenCacheVersion, c.Version)
+	}
+	return nil
+}
+
+var _ TokenCache = (*FileTokenCache)(nil)
diff --git a/libs/auth/cache/cache_test.go b/libs/auth/cache/file_test.go
similarity index 93%
rename from libs/auth/cache/cache_test.go
rename to libs/auth/cache/file_test.go
index 6529882c7..3e4aae36f 100644
--- a/libs/auth/cache/cache_test.go
+++ b/libs/auth/cache/file_test.go
@@ -27,7 +27,7 @@ func setup(t *testing.T) string {
 
 func TestStoreAndLookup(t *testing.T) {
 	setup(t)
-	c := &TokenCache{}
+	c := &FileTokenCache{}
 	err := c.Store("x", &oauth2.Token{
 		AccessToken: "abc",
 	})
@@ -38,7 +38,7 @@ func TestStoreAndLookup(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	l := &TokenCache{}
+	l := &FileTokenCache{}
 	tok, err := l.Lookup("x")
 	require.NoError(t, err)
 	assert.Equal(t, "abc", tok.AccessToken)
@@ -50,7 +50,7 @@ func TestStoreAndLookup(t *testing.T) {
 
 func TestNoCacheFileReturnsErrNotConfigured(t *testing.T) {
 	setup(t)
-	l := &TokenCache{}
+	l := &FileTokenCache{}
 	_, err := l.Lookup("x")
 	assert.Equal(t, ErrNotConfigured, err)
 }
@@ -63,7 +63,7 @@ func TestLoadCorruptFile(t *testing.T) {
 	err = os.WriteFile(f, []byte("abc"), ownerExecReadWrite)
 	require.NoError(t, err)
 
-	l := &TokenCache{}
+	l := &FileTokenCache{}
 	_, err = l.Lookup("x")
 	assert.EqualError(t, err, "load: parse: invalid character 'a' looking for beginning of value")
 }
@@ -76,14 +76,14 @@ func TestLoadWrongVersion(t *testing.T) {
 	err = os.WriteFile(f, []byte(`{"version": 823, "things": []}`), ownerExecReadWrite)
 	require.NoError(t, err)
 
-	l := &TokenCache{}
+	l := &FileTokenCache{}
 	_, err = l.Lookup("x")
 	assert.EqualError(t, err, "load: needs version 1, got version 823")
 }
 
 func TestDevNull(t *testing.T) {
 	t.Setenv(homeEnvVar, "/dev/null")
-	l := &TokenCache{}
+	l := &FileTokenCache{}
 	_, err := l.Lookup("x")
 	// macOS/Linux: load: read: open /dev/null/.databricks/token-cache.json:
 	// windows: databricks OAuth is not configured for this host
@@ -95,7 +95,7 @@ func TestStoreOnDev(t *testing.T) {
 		t.SkipNow()
 	}
 	t.Setenv(homeEnvVar, "/dev")
-	c := &TokenCache{}
+	c := &FileTokenCache{}
 	err := c.Store("x", &oauth2.Token{
 		AccessToken: "abc",
 	})
diff --git a/libs/auth/cache/in_memory.go b/libs/auth/cache/in_memory.go
new file mode 100644
index 000000000..469d45575
--- /dev/null
+++ b/libs/auth/cache/in_memory.go
@@ -0,0 +1,26 @@
+package cache
+
+import (
+	"golang.org/x/oauth2"
+)
+
+type InMemoryTokenCache struct {
+	Tokens map[string]*oauth2.Token
+}
+
+// Lookup implements TokenCache.
+func (i *InMemoryTokenCache) Lookup(key string) (*oauth2.Token, error) {
+	token, ok := i.Tokens[key]
+	if !ok {
+		return nil, ErrNotConfigured
+	}
+	return token, nil
+}
+
+// Store implements TokenCache.
+func (i *InMemoryTokenCache) Store(key string, t *oauth2.Token) error {
+	i.Tokens[key] = t
+	return nil
+}
+
+var _ TokenCache = (*InMemoryTokenCache)(nil)
diff --git a/libs/auth/cache/in_memory_test.go b/libs/auth/cache/in_memory_test.go
new file mode 100644
index 000000000..d8394d3b2
--- /dev/null
+++ b/libs/auth/cache/in_memory_test.go
@@ -0,0 +1,44 @@
+package cache
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/oauth2"
+)
+
+func TestInMemoryCacheHit(t *testing.T) {
+	token := &oauth2.Token{
+		AccessToken: "abc",
+	}
+	c := &InMemoryTokenCache{
+		Tokens: map[string]*oauth2.Token{
+			"key": token,
+		},
+	}
+	res, err := c.Lookup("key")
+	assert.Equal(t, res, token)
+	assert.NoError(t, err)
+}
+
+func TestInMemoryCacheMiss(t *testing.T) {
+	c := &InMemoryTokenCache{
+		Tokens: map[string]*oauth2.Token{},
+	}
+	_, err := c.Lookup("key")
+	assert.ErrorIs(t, err, ErrNotConfigured)
+}
+
+func TestInMemoryCacheStore(t *testing.T) {
+	token := &oauth2.Token{
+		AccessToken: "abc",
+	}
+	c := &InMemoryTokenCache{
+		Tokens: map[string]*oauth2.Token{},
+	}
+	err := c.Store("key", token)
+	assert.NoError(t, err)
+	res, err := c.Lookup("key")
+	assert.Equal(t, res, token)
+	assert.NoError(t, err)
+}
diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go
index 4ce0d4def..1f3e032de 100644
--- a/libs/auth/oauth.go
+++ b/libs/auth/oauth.go
@@ -20,6 +20,20 @@ import (
 	"golang.org/x/oauth2/authhandler"
 )
 
+var apiClientForOauth int
+
+func WithApiClientForOAuth(ctx context.Context, c *httpclient.ApiClient) context.Context {
+	return context.WithValue(ctx, &apiClientForOauth, c)
+}
+
+func GetApiClientForOAuth(ctx context.Context) *httpclient.ApiClient {
+	c, ok := ctx.Value(&apiClientForOauth).(*httpclient.ApiClient)
+	if !ok {
+		return httpclient.NewApiClient(httpclient.ClientConfig{})
+	}
+	return c
+}
+
 const (
 	// these values are predefined by Databricks as a public client
 	// and is specific to this application only. Using these values
@@ -28,7 +42,7 @@ const (
 	appRedirectAddr = "localhost:8020"
 
 	// maximum amount of time to acquire listener on appRedirectAddr
-	DefaultTimeout = 45 * time.Second
+	listenerTimeout = 45 * time.Second
 )
 
 var ( // Databricks SDK API: `databricks OAuth is not` will be checked for presence
@@ -42,14 +56,13 @@ type PersistentAuth struct {
 	AccountID string
 
 	http    *httpclient.ApiClient
-	cache   tokenCache
+	cache   cache.TokenCache
 	ln      net.Listener
 	browser func(string) error
 }
 
-type tokenCache interface {
-	Store(key string, t *oauth2.Token) error
-	Lookup(key string) (*oauth2.Token, error)
+func (a *PersistentAuth) SetApiClient(h *httpclient.ApiClient) {
+	a.http = h
 }
 
 func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) {
@@ -136,12 +149,10 @@ func (a *PersistentAuth) init(ctx context.Context) error {
 		return ErrFetchCredentials
 	}
 	if a.http == nil {
-		a.http = httpclient.NewApiClient(httpclient.ClientConfig{
-			// noop
-		})
+		a.http = GetApiClientForOAuth(ctx)
 	}
 	if a.cache == nil {
-		a.cache = &cache.TokenCache{}
+		a.cache = cache.GetTokenCache(ctx)
 	}
 	if a.browser == nil {
 		a.browser = browser.OpenURL
@@ -149,7 +160,7 @@ func (a *PersistentAuth) init(ctx context.Context) error {
 	// try acquire listener, which we also use as a machine-local
 	// exclusive lock to prevent token cache corruption in the scope
 	// of developer machine, where this command runs.
-	listener, err := retries.Poll(ctx, DefaultTimeout,
+	listener, err := retries.Poll(ctx, listenerTimeout,
 		func() (*net.Listener, *retries.Err) {
 			var lc net.ListenConfig
 			l, err := lc.Listen(ctx, "tcp", appRedirectAddr)
diff --git a/libs/databrickscfg/loader_test.go b/libs/databrickscfg/loader_test.go
index 4525115e0..c42fcdbdd 100644
--- a/libs/databrickscfg/loader_test.go
+++ b/libs/databrickscfg/loader_test.go
@@ -68,7 +68,7 @@ func TestLoaderErrorsOnInvalidFile(t *testing.T) {
 		Loaders: []config.Loader{
 			ResolveProfileFromHost,
 		},
-		ConfigFile: "testdata/badcfg",
+		ConfigFile: "profile/testdata/badcfg",
 		Host:       "https://default",
 	}
 
@@ -81,7 +81,7 @@ func TestLoaderSkipsNoMatchingHost(t *testing.T) {
 		Loaders: []config.Loader{
 			ResolveProfileFromHost,
 		},
-		ConfigFile: "testdata/databrickscfg",
+		ConfigFile: "profile/testdata/databrickscfg",
 		Host:       "https://noneofthehostsmatch",
 	}
 
@@ -95,7 +95,7 @@ func TestLoaderMatchingHost(t *testing.T) {
 		Loaders: []config.Loader{
 			ResolveProfileFromHost,
 		},
-		ConfigFile: "testdata/databrickscfg",
+		ConfigFile: "profile/testdata/databrickscfg",
 		Host:       "https://default",
 	}
 
@@ -110,7 +110,7 @@ func TestLoaderMatchingHostWithQuery(t *testing.T) {
 		Loaders: []config.Loader{
 			ResolveProfileFromHost,
 		},
-		ConfigFile: "testdata/databrickscfg",
+		ConfigFile: "profile/testdata/databrickscfg",
 		Host:       "https://query/?foo=bar",
 	}
 
@@ -125,7 +125,7 @@ func TestLoaderErrorsOnMultipleMatches(t *testing.T) {
 		Loaders: []config.Loader{
 			ResolveProfileFromHost,
 		},
-		ConfigFile: "testdata/databrickscfg",
+		ConfigFile: "profile/testdata/databrickscfg",
 		Host:       "https://foo/bar",
 	}
 
diff --git a/libs/databrickscfg/ops_test.go b/libs/databrickscfg/ops_test.go
index 233555fe2..3ea92024c 100644
--- a/libs/databrickscfg/ops_test.go
+++ b/libs/databrickscfg/ops_test.go
@@ -30,7 +30,7 @@ func TestLoadOrCreate_NotAllowed(t *testing.T) {
 }
 
 func TestLoadOrCreate_Bad(t *testing.T) {
-	path := "testdata/badcfg"
+	path := "profile/testdata/badcfg"
 	file, err := loadOrCreateConfigFile(path)
 	assert.Error(t, err)
 	assert.Nil(t, file)
@@ -40,7 +40,7 @@ func TestMatchOrCreateSection_Direct(t *testing.T) {
 	cfg := &config.Config{
 		Profile: "query",
 	}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
@@ -54,7 +54,7 @@ func TestMatchOrCreateSection_AccountID(t *testing.T) {
 	cfg := &config.Config{
 		AccountID: "abc",
 	}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
@@ -68,7 +68,7 @@ func TestMatchOrCreateSection_NormalizeHost(t *testing.T) {
 	cfg := &config.Config{
 		Host: "https://query/?o=abracadabra",
 	}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
@@ -80,7 +80,7 @@ func TestMatchOrCreateSection_NormalizeHost(t *testing.T) {
 
 func TestMatchOrCreateSection_NoProfileOrHost(t *testing.T) {
 	cfg := &config.Config{}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
@@ -92,7 +92,7 @@ func TestMatchOrCreateSection_MultipleProfiles(t *testing.T) {
 	cfg := &config.Config{
 		Host: "https://foo",
 	}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
@@ -105,7 +105,7 @@ func TestMatchOrCreateSection_NewProfile(t *testing.T) {
 		Host:    "https://bar",
 		Profile: "delirium",
 	}
-	file, err := loadOrCreateConfigFile("testdata/databrickscfg")
+	file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg")
 	assert.NoError(t, err)
 
 	ctx := context.Background()
diff --git a/libs/databrickscfg/profile/context.go b/libs/databrickscfg/profile/context.go
new file mode 100644
index 000000000..fa4d2ad8a
--- /dev/null
+++ b/libs/databrickscfg/profile/context.go
@@ -0,0 +1,17 @@
+package profile
+
+import "context"
+
+var profiler int
+
+func WithProfiler(ctx context.Context, p Profiler) context.Context {
+	return context.WithValue(ctx, &profiler, p)
+}
+
+func GetProfiler(ctx context.Context) Profiler {
+	p, ok := ctx.Value(&profiler).(Profiler)
+	if !ok {
+		return DefaultProfiler
+	}
+	return p
+}
diff --git a/libs/databrickscfg/profile/file.go b/libs/databrickscfg/profile/file.go
new file mode 100644
index 000000000..1b743014e
--- /dev/null
+++ b/libs/databrickscfg/profile/file.go
@@ -0,0 +1,100 @@
+package profile
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/cli/libs/env"
+	"github.com/databricks/databricks-sdk-go/config"
+	"github.com/spf13/cobra"
+)
+
+type FileProfilerImpl struct{}
+
+func (f FileProfilerImpl) getPath(ctx context.Context, replaceHomeDirWithTilde bool) (string, error) {
+	configFile := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
+	if configFile == "" {
+		configFile = "~/.databrickscfg"
+	}
+	if !replaceHomeDirWithTilde {
+		return configFile, nil
+	}
+	homedir, err := env.UserHomeDir(ctx)
+	if err != nil {
+		return "", err
+	}
+	configFile = strings.Replace(configFile, homedir, "~", 1)
+	return configFile, nil
+}
+
+// Get the path to the .databrickscfg file, falling back to the default in the current user's home directory.
+func (f FileProfilerImpl) GetPath(ctx context.Context) (string, error) {
+	fp, err := f.getPath(ctx, true)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Clean(fp), nil
+}
+
+var ErrNoConfiguration = errors.New("no configuration file found")
+
+func (f FileProfilerImpl) Get(ctx context.Context) (*config.File, error) {
+	path, err := f.getPath(ctx, false)
+	if err != nil {
+		return nil, fmt.Errorf("cannot determine Databricks config file path: %w", err)
+	}
+	if strings.HasPrefix(path, "~") {
+		homedir, err := env.UserHomeDir(ctx)
+		if err != nil {
+			return nil, err
+		}
+		path = filepath.Join(homedir, path[1:])
+	}
+	configFile, err := config.LoadFile(path)
+	if errors.Is(err, fs.ErrNotExist) {
+		// downstreams depend on ErrNoConfiguration. TODO: expose this error through SDK
+		return nil, fmt.Errorf("%w at %s; please create one by running 'databricks configure'", ErrNoConfiguration, path)
+	} else if err != nil {
+		return nil, err
+	}
+	return configFile, nil
+}
+
+func (f FileProfilerImpl) LoadProfiles(ctx context.Context, fn ProfileMatchFunction) (profiles Profiles, err error) {
+	file, err := f.Get(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("cannot load Databricks config file: %w", err)
+	}
+
+	// Iterate over sections and collect matching profiles.
+	for _, v := range file.Sections() {
+		all := v.KeysHash()
+		host, ok := all["host"]
+		if !ok {
+			// invalid profile
+			continue
+		}
+		profile := Profile{
+			Name:      v.Name(),
+			Host:      host,
+			AccountID: all["account_id"],
+		}
+		if fn(profile) {
+			profiles = append(profiles, profile)
+		}
+	}
+
+	return
+}
+
+func ProfileCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	profiles, err := DefaultProfiler.LoadProfiles(cmd.Context(), MatchAllProfiles)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveError
+	}
+	return profiles.Names(), cobra.ShellCompDirectiveNoFileComp
+}
diff --git a/libs/databrickscfg/profiles_test.go b/libs/databrickscfg/profile/file_test.go
similarity index 82%
rename from libs/databrickscfg/profiles_test.go
rename to libs/databrickscfg/profile/file_test.go
index 33a5c9dfd..8e5cfefc0 100644
--- a/libs/databrickscfg/profiles_test.go
+++ b/libs/databrickscfg/profile/file_test.go
@@ -1,4 +1,4 @@
-package databrickscfg
+package profile
 
 import (
 	"context"
@@ -32,7 +32,8 @@ func TestLoadProfilesReturnsHomedirAsTilde(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.WithUserHomeDir(ctx, "testdata")
 	ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg")
-	file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true })
+	profiler := FileProfilerImpl{}
+	file, err := profiler.GetPath(ctx)
 	require.NoError(t, err)
 	require.Equal(t, filepath.Clean("~/databrickscfg"), file)
 }
@@ -41,7 +42,8 @@ func TestLoadProfilesReturnsHomedirAsTildeExoticFile(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.WithUserHomeDir(ctx, "testdata")
 	ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "~/databrickscfg")
-	file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true })
+	profiler := FileProfilerImpl{}
+	file, err := profiler.GetPath(ctx)
 	require.NoError(t, err)
 	require.Equal(t, filepath.Clean("~/databrickscfg"), file)
 }
@@ -49,7 +51,8 @@ func TestLoadProfilesReturnsHomedirAsTildeExoticFile(t *testing.T) {
 func TestLoadProfilesReturnsHomedirAsTildeDefaultFile(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.WithUserHomeDir(ctx, "testdata/sample-home")
-	file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true })
+	profiler := FileProfilerImpl{}
+	file, err := profiler.GetPath(ctx)
 	require.NoError(t, err)
 	require.Equal(t, filepath.Clean("~/.databrickscfg"), file)
 }
@@ -57,14 +60,16 @@ func TestLoadProfilesReturnsHomedirAsTildeDefaultFile(t *testing.T) {
 func TestLoadProfilesNoConfiguration(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.WithUserHomeDir(ctx, "testdata")
-	_, _, err := LoadProfiles(ctx, func(p Profile) bool { return true })
+	profiler := FileProfilerImpl{}
+	_, err := profiler.LoadProfiles(ctx, MatchAllProfiles)
 	require.ErrorIs(t, err, ErrNoConfiguration)
 }
 
 func TestLoadProfilesMatchWorkspace(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg")
-	_, profiles, err := LoadProfiles(ctx, MatchWorkspaceProfiles)
+	profiler := FileProfilerImpl{}
+	profiles, err := profiler.LoadProfiles(ctx, MatchWorkspaceProfiles)
 	require.NoError(t, err)
 	assert.Equal(t, []string{"DEFAULT", "query", "foo1", "foo2"}, profiles.Names())
 }
@@ -72,7 +77,8 @@ func TestLoadProfilesMatchWorkspace(t *testing.T) {
 func TestLoadProfilesMatchAccount(t *testing.T) {
 	ctx := context.Background()
 	ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg")
-	_, profiles, err := LoadProfiles(ctx, MatchAccountProfiles)
+	profiler := FileProfilerImpl{}
+	profiles, err := profiler.LoadProfiles(ctx, MatchAccountProfiles)
 	require.NoError(t, err)
 	assert.Equal(t, []string{"acc"}, profiles.Names())
 }
diff --git a/libs/databrickscfg/profile/in_memory.go b/libs/databrickscfg/profile/in_memory.go
new file mode 100644
index 000000000..902ae42e6
--- /dev/null
+++ b/libs/databrickscfg/profile/in_memory.go
@@ -0,0 +1,25 @@
+package profile
+
+import "context"
+
+type InMemoryProfiler struct {
+	Profiles Profiles
+}
+
+// GetPath implements Profiler.
+func (i InMemoryProfiler) GetPath(context.Context) (string, error) {
+	return "<in memory>", nil
+}
+
+// LoadProfiles implements Profiler.
+func (i InMemoryProfiler) LoadProfiles(ctx context.Context, f ProfileMatchFunction) (Profiles, error) {
+	res := make(Profiles, 0)
+	for _, p := range i.Profiles {
+		if f(p) {
+			res = append(res, p)
+		}
+	}
+	return res, nil
+}
+
+var _ Profiler = InMemoryProfiler{}
diff --git a/libs/databrickscfg/profile/profile.go b/libs/databrickscfg/profile/profile.go
new file mode 100644
index 000000000..510e5c9e5
--- /dev/null
+++ b/libs/databrickscfg/profile/profile.go
@@ -0,0 +1,49 @@
+package profile
+
+import (
+	"strings"
+
+	"github.com/databricks/databricks-sdk-go/config"
+)
+
+// Profile holds a subset of the keys in a databrickscfg profile.
+// It should only be used for prompting and filtering.
+// Use its name to construct a config.Config.
+type Profile struct {
+	Name      string
+	Host      string
+	AccountID string
+}
+
+func (p Profile) Cloud() string {
+	cfg := config.Config{Host: p.Host}
+	switch {
+	case cfg.IsAws():
+		return "AWS"
+	case cfg.IsAzure():
+		return "Azure"
+	case cfg.IsGcp():
+		return "GCP"
+	default:
+		return ""
+	}
+}
+
+type Profiles []Profile
+
+// SearchCaseInsensitive implements the promptui.Searcher interface.
+// This allows the user to immediately starting typing to narrow down the list.
+func (p Profiles) SearchCaseInsensitive(input string, index int) bool {
+	input = strings.ToLower(input)
+	name := strings.ToLower(p[index].Name)
+	host := strings.ToLower(p[index].Host)
+	return strings.Contains(name, input) || strings.Contains(host, input)
+}
+
+func (p Profiles) Names() []string {
+	names := make([]string, len(p))
+	for i, v := range p {
+		names[i] = v.Name
+	}
+	return names
+}
diff --git a/libs/databrickscfg/profile/profiler.go b/libs/databrickscfg/profile/profiler.go
new file mode 100644
index 000000000..c0a549256
--- /dev/null
+++ b/libs/databrickscfg/profile/profiler.go
@@ -0,0 +1,32 @@
+package profile
+
+import (
+	"context"
+)
+
+type ProfileMatchFunction func(Profile) bool
+
+func MatchWorkspaceProfiles(p Profile) bool {
+	return p.AccountID == ""
+}
+
+func MatchAccountProfiles(p Profile) bool {
+	return p.Host != "" && p.AccountID != ""
+}
+
+func MatchAllProfiles(p Profile) bool {
+	return true
+}
+
+func WithName(name string) ProfileMatchFunction {
+	return func(p Profile) bool {
+		return p.Name == name
+	}
+}
+
+type Profiler interface {
+	LoadProfiles(context.Context, ProfileMatchFunction) (Profiles, error)
+	GetPath(context.Context) (string, error)
+}
+
+var DefaultProfiler = FileProfilerImpl{}
diff --git a/libs/databrickscfg/testdata/badcfg b/libs/databrickscfg/profile/testdata/badcfg
similarity index 100%
rename from libs/databrickscfg/testdata/badcfg
rename to libs/databrickscfg/profile/testdata/badcfg
diff --git a/libs/databrickscfg/testdata/databrickscfg b/libs/databrickscfg/profile/testdata/databrickscfg
similarity index 100%
rename from libs/databrickscfg/testdata/databrickscfg
rename to libs/databrickscfg/profile/testdata/databrickscfg
diff --git a/libs/databrickscfg/testdata/sample-home/.databrickscfg b/libs/databrickscfg/profile/testdata/sample-home/.databrickscfg
similarity index 100%
rename from libs/databrickscfg/testdata/sample-home/.databrickscfg
rename to libs/databrickscfg/profile/testdata/sample-home/.databrickscfg
diff --git a/libs/databrickscfg/profiles.go b/libs/databrickscfg/profiles.go
deleted file mode 100644
index 200ac9c87..000000000
--- a/libs/databrickscfg/profiles.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package databrickscfg
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/fs"
-	"path/filepath"
-	"strings"
-
-	"github.com/databricks/cli/libs/env"
-	"github.com/databricks/databricks-sdk-go/config"
-	"github.com/spf13/cobra"
-)
-
-// Profile holds a subset of the keys in a databrickscfg profile.
-// It should only be used for prompting and filtering.
-// Use its name to construct a config.Config.
-type Profile struct {
-	Name      string
-	Host      string
-	AccountID string
-}
-
-func (p Profile) Cloud() string {
-	cfg := config.Config{Host: p.Host}
-	switch {
-	case cfg.IsAws():
-		return "AWS"
-	case cfg.IsAzure():
-		return "Azure"
-	case cfg.IsGcp():
-		return "GCP"
-	default:
-		return ""
-	}
-}
-
-type Profiles []Profile
-
-func (p Profiles) Names() []string {
-	names := make([]string, len(p))
-	for i, v := range p {
-		names[i] = v.Name
-	}
-	return names
-}
-
-// SearchCaseInsensitive implements the promptui.Searcher interface.
-// This allows the user to immediately starting typing to narrow down the list.
-func (p Profiles) SearchCaseInsensitive(input string, index int) bool {
-	input = strings.ToLower(input)
-	name := strings.ToLower(p[index].Name)
-	host := strings.ToLower(p[index].Host)
-	return strings.Contains(name, input) || strings.Contains(host, input)
-}
-
-type ProfileMatchFunction func(Profile) bool
-
-func MatchWorkspaceProfiles(p Profile) bool {
-	return p.AccountID == ""
-}
-
-func MatchAccountProfiles(p Profile) bool {
-	return p.Host != "" && p.AccountID != ""
-}
-
-func MatchAllProfiles(p Profile) bool {
-	return true
-}
-
-// Get the path to the .databrickscfg file, falling back to the default in the current user's home directory.
-func GetPath(ctx context.Context) (string, error) {
-	configFile := env.Get(ctx, "DATABRICKS_CONFIG_FILE")
-	if configFile == "" {
-		configFile = "~/.databrickscfg"
-	}
-	if strings.HasPrefix(configFile, "~") {
-		homedir, err := env.UserHomeDir(ctx)
-		if err != nil {
-			return "", err
-		}
-		configFile = filepath.Join(homedir, configFile[1:])
-	}
-	return configFile, nil
-}
-
-var ErrNoConfiguration = errors.New("no configuration file found")
-
-func Get(ctx context.Context) (*config.File, error) {
-	path, err := GetPath(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("cannot determine Databricks config file path: %w", err)
-	}
-	configFile, err := config.LoadFile(path)
-	if errors.Is(err, fs.ErrNotExist) {
-		// downstreams depend on ErrNoConfiguration. TODO: expose this error through SDK
-		return nil, fmt.Errorf("%w at %s; please create one by running 'databricks configure'", ErrNoConfiguration, path)
-	} else if err != nil {
-		return nil, err
-	}
-	return configFile, nil
-}
-
-func LoadProfiles(ctx context.Context, fn ProfileMatchFunction) (file string, profiles Profiles, err error) {
-	f, err := Get(ctx)
-	if err != nil {
-		return "", nil, fmt.Errorf("cannot load Databricks config file: %w", err)
-	}
-
-	// Replace homedir with ~ if applicable.
-	// This is to make the output more readable.
-	file = filepath.Clean(f.Path())
-	home, err := env.UserHomeDir(ctx)
-	if err != nil {
-		return "", nil, err
-	}
-	homedir := filepath.Clean(home)
-	if strings.HasPrefix(file, homedir) {
-		file = "~" + file[len(homedir):]
-	}
-
-	// Iterate over sections and collect matching profiles.
-	for _, v := range f.Sections() {
-		all := v.KeysHash()
-		host, ok := all["host"]
-		if !ok {
-			// invalid profile
-			continue
-		}
-		profile := Profile{
-			Name:      v.Name(),
-			Host:      host,
-			AccountID: all["account_id"],
-		}
-		if fn(profile) {
-			profiles = append(profiles, profile)
-		}
-	}
-
-	return
-}
-
-func ProfileCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	_, profiles, err := LoadProfiles(cmd.Context(), MatchAllProfiles)
-	if err != nil {
-		return nil, cobra.ShellCompDirectiveError
-	}
-	return profiles.Names(), cobra.ShellCompDirectiveNoFileComp
-}

From 4556d33e6b341f8efe3558b8961cf618299a3648 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 May 2024 11:02:30 +0200
Subject: [PATCH 23/41] Don't hide commands of services that are already hidden
 (#1438)

## Changes

Currently, the help output of services in preview doesn't show any of
their commands because the commands themselves are hidden as well.

This change updates that behavior to not hide commands in preview if the
service itself is also in preview. This makes the help output of
services in preview actually usable.

## Tests

n/a
---
 .codegen/service.go.tmpl                      |  6 ++--
 cmd/workspace/apps/apps.go                    | 30 -------------------
 .../consumer-fulfillments.go                  |  6 ----
 .../consumer-installations.go                 | 15 ----------
 .../consumer-listings/consumer-listings.go    |  9 ------
 .../consumer-personalization-requests.go      |  9 ------
 .../consumer-providers/consumer-providers.go  |  6 ----
 .../provider-exchange-filters.go              | 12 --------
 .../provider-exchanges/provider-exchanges.go  | 27 -----------------
 .../provider-files/provider-files.go          | 12 --------
 .../provider-listings/provider-listings.go    | 15 ----------
 .../provider-personalization-requests.go      |  6 ----
 .../provider-provider-analytics-dashboards.go | 12 --------
 .../provider-providers/provider-providers.go  | 15 ----------
 14 files changed, 4 insertions(+), 176 deletions(-)

diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 492b2132f..ad482ebe6 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -39,6 +39,7 @@ import (
 
 {{define "service"}}
 {{- $excludeMethods := list "put-secret" -}}
+{{- $hideService := .IsPrivatePreview }}
 
 // Slice with functions to override default command behavior.
 // Functions can be added from the `init()` function in manually curated files in this directory.
@@ -57,7 +58,7 @@ func New() *cobra.Command {
 			"package": "{{ .Package.Name }}",
 		},
 		{{- end }}
-		{{- if .IsPrivatePreview }}
+		{{- if $hideService }}
 
 		// This service is being previewed; hide from help output.
 		Hidden: true,
@@ -190,7 +191,8 @@ func new{{.PascalName}}() *cobra.Command {
     {{- end -}}
 	`
 	{{- end }}
-	{{- if .IsPrivatePreview }}
+	{{/* Don't hide commands if the service itself is already hidden. */}}
+	{{- if and (not $hideService) .IsPrivatePreview }}
 
 	// This command is being previewed; hide from help output.
 	cmd.Hidden = true
diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go
index 2ccd16c0c..1d6de4775 100755
--- a/cmd/workspace/apps/apps.go
+++ b/cmd/workspace/apps/apps.go
@@ -89,9 +89,6 @@ func newCreate() *cobra.Command {
       characters and hyphens and be between 2 and 30 characters long. It must be
       unique within the workspace.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -192,9 +189,6 @@ func newCreateDeployment() *cobra.Command {
     APP_NAME: The name of the app.
     SOURCE_CODE_PATH: The source code path of the deployment.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -288,9 +282,6 @@ func newDelete() *cobra.Command {
   Arguments:
     NAME: The name of the app.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -349,9 +340,6 @@ func newGet() *cobra.Command {
   Arguments:
     NAME: The name of the app.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -412,9 +400,6 @@ func newGetDeployment() *cobra.Command {
     APP_NAME: The name of the app.
     DEPLOYMENT_ID: The unique id of the deployment.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -474,9 +459,6 @@ func newGetEnvironment() *cobra.Command {
   Arguments:
     NAME: The name of the app.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -535,9 +517,6 @@ func newList() *cobra.Command {
   
   Lists all apps in the workspace.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -594,9 +573,6 @@ func newListDeployments() *cobra.Command {
   Arguments:
     APP_NAME: The name of the app.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -652,9 +628,6 @@ func newStop() *cobra.Command {
   Arguments:
     NAME: The name of the app.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -719,9 +692,6 @@ func newUpdate() *cobra.Command {
       characters and hyphens and be between 2 and 30 characters long. It must be
       unique within the workspace.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go
index cd92002a4..6f3ba4b42 100755
--- a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go
+++ b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go
@@ -64,9 +64,6 @@ func newGet() *cobra.Command {
   
   Get a high level preview of the metadata of listing installable content.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -126,9 +123,6 @@ func newList() *cobra.Command {
   Personalized installations contain metadata about the attached share or git
   repo, as well as the Delta Sharing recipient type.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go
index 9d6c7c894..d176e5b39 100755
--- a/cmd/workspace/consumer-installations/consumer-installations.go
+++ b/cmd/workspace/consumer-installations/consumer-installations.go
@@ -76,9 +76,6 @@ func newCreate() *cobra.Command {
   
   Install payload associated with a Databricks Marketplace listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -140,9 +137,6 @@ func newDelete() *cobra.Command {
   
   Uninstall an installation associated with a Databricks Marketplace listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -202,9 +196,6 @@ func newList() *cobra.Command {
   
   List all installations across all listings.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -258,9 +249,6 @@ func newListListingInstallations() *cobra.Command {
   
   List all installations for a particular listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -321,9 +309,6 @@ func newUpdate() *cobra.Command {
   the rotateToken flag is true 2. the token will be forcibly rotate if the
   rotateToken flag is true and the tokenInfo field is empty`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go
index 70295dfb3..f75f03b3a 100755
--- a/cmd/workspace/consumer-listings/consumer-listings.go
+++ b/cmd/workspace/consumer-listings/consumer-listings.go
@@ -66,9 +66,6 @@ func newGet() *cobra.Command {
   Get a published listing in the Databricks Marketplace that the consumer has
   access to.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -148,9 +145,6 @@ func newList() *cobra.Command {
   List all published listings in the Databricks Marketplace that the consumer
   has access to.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -215,9 +209,6 @@ func newSearch() *cobra.Command {
   Arguments:
     QUERY: Fuzzy matches query`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go
index 40ae4c848..c55ca4ee1 100755
--- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go
+++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go
@@ -75,9 +75,6 @@ func newCreate() *cobra.Command {
   
   Create a personalization request for a listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -142,9 +139,6 @@ func newGet() *cobra.Command {
   Get the personalization request for a listing. Each consumer can make at
   *most* one personalization request for a listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -203,9 +197,6 @@ func newList() *cobra.Command {
   
   List personalization requests for a consumer across all listings.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/consumer-providers/consumer-providers.go b/cmd/workspace/consumer-providers/consumer-providers.go
index 5a0849dce..d8ac0ec12 100755
--- a/cmd/workspace/consumer-providers/consumer-providers.go
+++ b/cmd/workspace/consumer-providers/consumer-providers.go
@@ -64,9 +64,6 @@ func newGet() *cobra.Command {
   Get a provider in the Databricks Marketplace with at least one visible
   listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -139,9 +136,6 @@ func newList() *cobra.Command {
   List all providers in the Databricks Marketplace with at least one visible
   listing.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go
index 43ae6da7e..4ab36b5d0 100755
--- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go
+++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go
@@ -68,9 +68,6 @@ func newCreate() *cobra.Command {
   
   Add an exchange filter.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -128,9 +125,6 @@ func newDelete() *cobra.Command {
   
   Delete an exchange filter`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -201,9 +195,6 @@ func newList() *cobra.Command {
   
   List exchange filter`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -258,9 +249,6 @@ func newUpdate() *cobra.Command {
   
   Update an exchange filter.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go
index c9f5818f5..7ff73e0d1 100755
--- a/cmd/workspace/provider-exchanges/provider-exchanges.go
+++ b/cmd/workspace/provider-exchanges/provider-exchanges.go
@@ -74,9 +74,6 @@ func newAddListingToExchange() *cobra.Command {
   
   Associate an exchange with a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -152,9 +149,6 @@ func newCreate() *cobra.Command {
   
   Create an exchange`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -212,9 +206,6 @@ func newDelete() *cobra.Command {
   
   This removes a listing from marketplace.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -270,9 +261,6 @@ func newDeleteListingFromExchange() *cobra.Command {
   
   Disassociate an exchange with a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -328,9 +316,6 @@ func newGet() *cobra.Command {
   
   Get an exchange.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -389,9 +374,6 @@ func newList() *cobra.Command {
   
   List exchanges visible to provider`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -445,9 +427,6 @@ func newListExchangesForListing() *cobra.Command {
   
   List exchanges associated with a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -503,9 +482,6 @@ func newListListingsForExchange() *cobra.Command {
   
   List listings associated with an exchange`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -560,9 +536,6 @@ func newUpdate() *cobra.Command {
   
   Update an exchange`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go
index b9357f131..25e1addf5 100755
--- a/cmd/workspace/provider-files/provider-files.go
+++ b/cmd/workspace/provider-files/provider-files.go
@@ -72,9 +72,6 @@ func newCreate() *cobra.Command {
   Create a file. Currently, only provider icons and attached notebooks are
   supported.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -132,9 +129,6 @@ func newDelete() *cobra.Command {
   
   Delete a file`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -202,9 +196,6 @@ func newGet() *cobra.Command {
   
   Get a file`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -277,9 +268,6 @@ func newList() *cobra.Command {
   
   List files attached to a parent entity.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go
index 4f90f7b9e..0abdf51d8 100755
--- a/cmd/workspace/provider-listings/provider-listings.go
+++ b/cmd/workspace/provider-listings/provider-listings.go
@@ -70,9 +70,6 @@ func newCreate() *cobra.Command {
   
   Create a new listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -130,9 +127,6 @@ func newDelete() *cobra.Command {
   
   Delete a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -200,9 +194,6 @@ func newGet() *cobra.Command {
   
   Get a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -273,9 +264,6 @@ func newList() *cobra.Command {
   
   List listings owned by this provider`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -328,9 +316,6 @@ func newUpdate() *cobra.Command {
   
   Update a listing`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go
index 58b3cba1d..a38d9f420 100755
--- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go
+++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go
@@ -69,9 +69,6 @@ func newList() *cobra.Command {
   List personalization requests to this provider. This will return all
   personalization requests, regardless of which listing they are for.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -128,9 +125,6 @@ func newUpdate() *cobra.Command {
   Update personalization request. This method only permits updating the status
   of the request.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go
index 70ef0f320..8cee6e4eb 100755
--- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go
+++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go
@@ -60,9 +60,6 @@ func newCreate() *cobra.Command {
   Create provider analytics dashboard. Returns Marketplace specific id. Not to
   be confused with the Lakeview dashboard id.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -105,9 +102,6 @@ func newGet() *cobra.Command {
   
   Get provider analytics dashboard.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -150,9 +144,6 @@ func newGetLatestVersion() *cobra.Command {
   
   Get latest version of provider analytics dashboard.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -207,9 +198,6 @@ func newUpdate() *cobra.Command {
   Arguments:
     ID: id is immutable property and can't be updated.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go
index 52f4c45ae..b7273a344 100755
--- a/cmd/workspace/provider-providers/provider-providers.go
+++ b/cmd/workspace/provider-providers/provider-providers.go
@@ -69,9 +69,6 @@ func newCreate() *cobra.Command {
   
   Create a provider`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -129,9 +126,6 @@ func newDelete() *cobra.Command {
   
   Delete provider`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -199,9 +193,6 @@ func newGet() *cobra.Command {
   
   Get provider profile`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.PreRunE = root.MustWorkspaceClient
@@ -272,9 +263,6 @@ func newList() *cobra.Command {
   
   List provider profiles for account.`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
@@ -327,9 +315,6 @@ func newUpdate() *cobra.Command {
   
   Update provider profile`
 
-	// This command is being previewed; hide from help output.
-	cmd.Hidden = true
-
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {

From dd941078537581da78a8e6424d4c51953ddbca81 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 17 May 2024 11:26:09 +0200
Subject: [PATCH 24/41] Remove dependency on `ConfigFilePath` from path
 translation mutator (#1437)

## Changes

This is one step toward removing the `path.Paths` struct embedding from
resource types.

Going forward, we'll exclusively use the `dyn.Value` tree for location
information.

## Tests

Existing unit tests that cover path resolution with fallback behavior
pass.
---
 bundle/config/mutator/translate_paths.go      | 28 +++++++++++++++++++
 bundle/config/mutator/translate_paths_jobs.go | 19 ++++---------
 .../mutator/translate_paths_pipelines.go      | 15 ++--------
 bundle/config/paths/paths.go                  | 10 -------
 4 files changed, 37 insertions(+), 35 deletions(-)

diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go
index 018fd79c6..18a09dfd6 100644
--- a/bundle/config/mutator/translate_paths.go
+++ b/bundle/config/mutator/translate_paths.go
@@ -213,3 +213,31 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
 
 	return diag.FromErr(err)
 }
+
+func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) {
+	var fallback = make(map[string]string)
+	var pattern = dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey())
+
+	// Previous behavior was to use a resource's location as the base path to resolve
+	// relative paths in its definition. With the introduction of [dyn.Value] throughout,
+	// we can use the location of the [dyn.Value] of the relative path itself.
+	//
+	// This is more flexible, as resources may have overrides that are not
+	// located in the same directory as the resource configuration file.
+	//
+	// To maintain backwards compatibility, we allow relative paths to be resolved using
+	// the original approach as fallback if the [dyn.Value] location cannot be resolved.
+	_, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+		key := p[2].Key()
+		dir, err := v.Location().Directory()
+		if err != nil {
+			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for %s: %w", p, err)
+		}
+		fallback[key] = dir
+		return v, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return fallback, nil
+}
diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go
index d41660728..58b5e0fb0 100644
--- a/bundle/config/mutator/translate_paths_jobs.go
+++ b/bundle/config/mutator/translate_paths_jobs.go
@@ -55,21 +55,14 @@ func rewritePatterns(base dyn.Pattern) []jobRewritePattern {
 }
 
 func (m *translatePaths) applyJobTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) {
-	var fallback = make(map[string]string)
+	fallback, err := gatherFallbackPaths(v, "jobs")
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
+	// Do not translate job task paths if using Git source
 	var ignore []string
-	var err error
-
 	for key, job := range b.Config.Resources.Jobs {
-		dir, err := job.ConfigFileDirectory()
-		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
-		}
-
-		// If we cannot resolve the relative path using the [dyn.Value] location itself,
-		// use the job's location as fallback. This is necessary for backwards compatibility.
-		fallback[key] = dir
-
-		// Do not translate job task paths if using git source
 		if job.GitSource != nil {
 			ignore = append(ignore, key)
 		}
diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go
index caec4198e..5b2a2c346 100644
--- a/bundle/config/mutator/translate_paths_pipelines.go
+++ b/bundle/config/mutator/translate_paths_pipelines.go
@@ -8,18 +8,9 @@ import (
 )
 
 func (m *translatePaths) applyPipelineTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) {
-	var fallback = make(map[string]string)
-	var err error
-
-	for key, pipeline := range b.Config.Resources.Pipelines {
-		dir, err := pipeline.ConfigFileDirectory()
-		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err)
-		}
-
-		// If we cannot resolve the relative path using the [dyn.Value] location itself,
-		// use the pipeline's location as fallback. This is necessary for backwards compatibility.
-		fallback[key] = dir
+	fallback, err := gatherFallbackPaths(v, "pipelines")
+	if err != nil {
+		return dyn.InvalidValue, err
 	}
 
 	// Base pattern to match all libraries in all pipelines.
diff --git a/bundle/config/paths/paths.go b/bundle/config/paths/paths.go
index 68c32a48c..95977ee37 100644
--- a/bundle/config/paths/paths.go
+++ b/bundle/config/paths/paths.go
@@ -1,9 +1,6 @@
 package paths
 
 import (
-	"fmt"
-	"path/filepath"
-
 	"github.com/databricks/cli/libs/dyn"
 )
 
@@ -23,10 +20,3 @@ func (p *Paths) ConfigureConfigFilePath() {
 	}
 	p.ConfigFilePath = p.DynamicValue.Location().File
 }
-
-func (p *Paths) ConfigFileDirectory() (string, error) {
-	if p.ConfigFilePath == "" {
-		return "", fmt.Errorf("config file path not configured")
-	}
-	return filepath.Dir(p.ConfigFilePath), nil
-}

From 04e56aa4720e821bff4ceea1c34894e4f9c5dc89 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Fri, 17 May 2024 11:34:39 +0200
Subject: [PATCH 25/41] Add `merge.Override` transform (#1428)

## Changes
Add `merge.Override` transform. It allows the override one `dyn.Value`
with another, preserving source locations for parts of the sub-tree
where nothing has changed. This is different from merging, where values
are concatenated.

`OverrideVisitor` is visiting the changes during the override process
and allows to control of what changes are allowed or update the
effective value.

The primary use case is Python code updating bundle configuration.

During override, we update locations only for changed values. This
allows us to keep track of locations where values were initially defined
and used for error reporting. For instance, merging:

```yaml
resources:               # location=left.yaml:0
  jobs:                  # location=left.yaml:1
    job_0:               # location=left.yaml:2
      name: "job_0"      # location=left.yaml:3
```

with

```yaml
resources:               # location=right.yaml:0
  jobs:                  # location=right.yaml:1
    job_0:               # location=right.yaml:2
      name: "job_0"      # location=right.yaml:3
      description: job 0 # location=right.yaml:4
    job_1:               # location=right.yaml:5
      name: "job_1"      # location=right.yaml:5
```

produces

```yaml
resources:               # location=left.yaml:0
  jobs:                  # location=left.yaml:1
    job_0:               # location=left.yaml:2
      name: "job_0"      # location=left.yaml:3
      description: job 0 # location=right.yaml:4
    job_1:               # location=right.yaml:5
      name: "job_1"      # location=right.yaml:5
```

## Tests
Unit tests
---
 libs/dyn/merge/override.go      | 198 +++++++++++++++
 libs/dyn/merge/override_test.go | 434 ++++++++++++++++++++++++++++++++
 2 files changed, 632 insertions(+)
 create mode 100644 libs/dyn/merge/override.go
 create mode 100644 libs/dyn/merge/override_test.go

diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go
new file mode 100644
index 000000000..97e8f1009
--- /dev/null
+++ b/libs/dyn/merge/override.go
@@ -0,0 +1,198 @@
+package merge
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/libs/dyn"
+)
+
+// OverrideVisitor is visiting the changes during the override process
+// and allows to control what changes are allowed, or update the effective
+// value.
+//
+// For instance, it can disallow changes outside the specific path(s), or update
+// the location of the effective value.
+//
+// 'VisitDelete' is called when a value is removed from mapping or sequence
+// 'VisitInsert' is called when a new value is added to mapping or sequence
+// 'VisitUpdate' is called when a leaf value is updated
+type OverrideVisitor struct {
+	VisitDelete func(valuePath dyn.Path, left dyn.Value) error
+	VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error)
+	VisitUpdate func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error)
+}
+
+// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values
+// haven't changed. Preserving 'location' is important to preserve the original source of the value
+// for error reporting.
+func Override(leftRoot dyn.Value, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) {
+	return override(dyn.EmptyPath, leftRoot, rightRoot, visitor)
+}
+
+func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) {
+	if left == dyn.NilValue && right == dyn.NilValue {
+		return dyn.NilValue, nil
+	}
+
+	if left.Kind() != right.Kind() {
+		return visitor.VisitUpdate(basePath, left, right)
+	}
+
+	// NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings
+	// we don't know if value was updated or not
+
+	switch left.Kind() {
+	case dyn.KindMap:
+		merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor)
+
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		return dyn.NewValue(merged, left.Location()), nil
+
+	case dyn.KindSequence:
+		// some sequences are keyed, and we can detect which elements are added/removed/updated,
+		// but we don't have this information
+		merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor)
+
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		return dyn.NewValue(merged, left.Location()), nil
+
+	case dyn.KindString:
+		if left.MustString() == right.MustString() {
+			return left, nil
+		} else {
+			return visitor.VisitUpdate(basePath, left, right)
+		}
+
+	case dyn.KindFloat:
+		// TODO consider comparison with epsilon if normalization doesn't help, where do we use floats?
+
+		if left.MustFloat() == right.MustFloat() {
+			return left, nil
+		} else {
+			return visitor.VisitUpdate(basePath, left, right)
+		}
+
+	case dyn.KindBool:
+		if left.MustBool() == right.MustBool() {
+			return left, nil
+		} else {
+			return visitor.VisitUpdate(basePath, left, right)
+		}
+
+	case dyn.KindTime:
+		if left.MustTime() == right.MustTime() {
+			return left, nil
+		} else {
+			return visitor.VisitUpdate(basePath, left, right)
+		}
+
+	case dyn.KindInt:
+		if left.MustInt() == right.MustInt() {
+			return left, nil
+		} else {
+			return visitor.VisitUpdate(basePath, left, right)
+		}
+	}
+
+	return dyn.InvalidValue, fmt.Errorf("unexpected kind %s", left.Kind())
+}
+
+func overrideMapping(basePath dyn.Path, leftMapping dyn.Mapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) {
+	out := dyn.NewMapping()
+
+	for _, leftPair := range leftMapping.Pairs() {
+		// detect if key was removed
+		if _, ok := rightMapping.GetPair(leftPair.Key); !ok {
+			path := basePath.Append(dyn.Key(leftPair.Key.MustString()))
+
+			err := visitor.VisitDelete(path, leftPair.Value)
+
+			if err != nil {
+				return dyn.NewMapping(), err
+			}
+		}
+	}
+
+	// iterating only right mapping will remove keys not present anymore
+	// and insert new keys
+
+	for _, rightPair := range rightMapping.Pairs() {
+		if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok {
+			path := basePath.Append(dyn.Key(rightPair.Key.MustString()))
+			newValue, err := override(path, leftPair.Value, rightPair.Value, visitor)
+
+			if err != nil {
+				return dyn.NewMapping(), err
+			}
+
+			// key was there before, so keep its location
+			err = out.Set(leftPair.Key, newValue)
+
+			if err != nil {
+				return dyn.NewMapping(), err
+			}
+		} else {
+			path := basePath.Append(dyn.Key(rightPair.Key.MustString()))
+
+			newValue, err := visitor.VisitInsert(path, rightPair.Value)
+
+			if err != nil {
+				return dyn.NewMapping(), err
+			}
+
+			err = out.Set(rightPair.Key, newValue)
+
+			if err != nil {
+				return dyn.NewMapping(), err
+			}
+		}
+	}
+
+	return out, nil
+}
+
+func overrideSequence(basePath dyn.Path, left []dyn.Value, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) {
+	minLen := min(len(left), len(right))
+	var values []dyn.Value
+
+	for i := 0; i < minLen; i++ {
+		path := basePath.Append(dyn.Index(i))
+		merged, err := override(path, left[i], right[i], visitor)
+
+		if err != nil {
+			return nil, err
+		}
+
+		values = append(values, merged)
+	}
+
+	if len(right) > len(left) {
+		for i := minLen; i < len(right); i++ {
+			path := basePath.Append(dyn.Index(i))
+			newValue, err := visitor.VisitInsert(path, right[i])
+
+			if err != nil {
+				return nil, err
+			}
+
+			values = append(values, newValue)
+		}
+	} else if len(left) > len(right) {
+		for i := minLen; i < len(left); i++ {
+			path := basePath.Append(dyn.Index(i))
+			err := visitor.VisitDelete(path, left[i])
+
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return values, nil
+}
diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go
new file mode 100644
index 000000000..dbf249d12
--- /dev/null
+++ b/libs/dyn/merge/override_test.go
@@ -0,0 +1,434 @@
+package merge
+
+import (
+	"testing"
+	"time"
+
+	"github.com/databricks/cli/libs/dyn"
+	assert "github.com/databricks/cli/libs/dyn/dynassert"
+)
+
+type overrideTestCase struct {
+	name     string
+	left     dyn.Value
+	right    dyn.Value
+	state    visitorState
+	expected dyn.Value
+}
+
+func TestOverride_Primitive(t *testing.T) {
+	leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1}
+	rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1}
+
+	modifiedTestCases := []overrideTestCase{
+		{
+			name:     "string (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue("a", leftLocation),
+			right:    dyn.NewValue("b", rightLocation),
+			expected: dyn.NewValue("b", rightLocation),
+		},
+		{
+			name:     "string (not updated)",
+			state:    visitorState{},
+			left:     dyn.NewValue("a", leftLocation),
+			right:    dyn.NewValue("a", rightLocation),
+			expected: dyn.NewValue("a", leftLocation),
+		},
+		{
+			name:     "bool (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue(true, leftLocation),
+			right:    dyn.NewValue(false, rightLocation),
+			expected: dyn.NewValue(false, rightLocation),
+		},
+		{
+			name:     "bool (not updated)",
+			state:    visitorState{},
+			left:     dyn.NewValue(true, leftLocation),
+			right:    dyn.NewValue(true, rightLocation),
+			expected: dyn.NewValue(true, leftLocation),
+		},
+		{
+			name:     "int (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue(1, leftLocation),
+			right:    dyn.NewValue(2, rightLocation),
+			expected: dyn.NewValue(2, rightLocation),
+		},
+		{
+			name:     "int (not updated)",
+			state:    visitorState{},
+			left:     dyn.NewValue(int32(1), leftLocation),
+			right:    dyn.NewValue(int64(1), rightLocation),
+			expected: dyn.NewValue(int32(1), leftLocation),
+		},
+		{
+			name:     "float (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue(1.0, leftLocation),
+			right:    dyn.NewValue(2.0, rightLocation),
+			expected: dyn.NewValue(2.0, rightLocation),
+		},
+		{
+			name:     "float (not updated)",
+			state:    visitorState{},
+			left:     dyn.NewValue(float32(1.0), leftLocation),
+			right:    dyn.NewValue(float64(1.0), rightLocation),
+			expected: dyn.NewValue(float32(1.0), leftLocation),
+		},
+		{
+			name:     "time (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue(time.UnixMilli(10000), leftLocation),
+			right:    dyn.NewValue(time.UnixMilli(10001), rightLocation),
+			expected: dyn.NewValue(time.UnixMilli(10001), rightLocation),
+		},
+		{
+			name:     "time (not updated)",
+			state:    visitorState{},
+			left:     dyn.NewValue(time.UnixMilli(10000), leftLocation),
+			right:    dyn.NewValue(time.UnixMilli(10000), rightLocation),
+			expected: dyn.NewValue(time.UnixMilli(10000), leftLocation),
+		},
+		{
+			name:     "different types (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue("a", leftLocation),
+			right:    dyn.NewValue(42, rightLocation),
+			expected: dyn.NewValue(42, rightLocation),
+		},
+		{
+			name: "map - remove 'a', update 'b'",
+			state: visitorState{
+				removed: []string{"root.a"},
+				updated: []string{"root.b"},
+			},
+			left: dyn.NewValue(
+				map[string]dyn.Value{
+					"a": dyn.NewValue(42, leftLocation),
+					"b": dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				map[string]dyn.Value{
+					"b": dyn.NewValue(20, rightLocation),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				map[string]dyn.Value{
+					"b": dyn.NewValue(20, rightLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name: "map - add 'a'",
+			state: visitorState{
+				added: []string{"root.a"},
+			},
+			left: dyn.NewValue(
+				map[string]dyn.Value{
+					"b": dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				map[string]dyn.Value{
+					"a": dyn.NewValue(42, rightLocation),
+					"b": dyn.NewValue(10, rightLocation),
+				},
+				leftLocation,
+			),
+			expected: dyn.NewValue(
+				map[string]dyn.Value{
+					"a": dyn.NewValue(42, rightLocation),
+					// location hasn't changed because value hasn't changed
+					"b": dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name: "map - remove 'a'",
+			state: visitorState{
+				removed: []string{"root.a"},
+			},
+			left: dyn.NewValue(
+				map[string]dyn.Value{
+					"a": dyn.NewValue(42, leftLocation),
+					"b": dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				map[string]dyn.Value{
+					"b": dyn.NewValue(10, rightLocation),
+				},
+				leftLocation,
+			),
+			expected: dyn.NewValue(
+				map[string]dyn.Value{
+					// location hasn't changed because value hasn't changed
+					"b": dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name: "map - add 'jobs.job_1'",
+			state: visitorState{
+				added: []string{"root.jobs.job_1"},
+			},
+			left: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, leftLocation),
+						},
+						leftLocation,
+					),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, rightLocation),
+							"job_1": dyn.NewValue(1337, rightLocation),
+						},
+						rightLocation,
+					),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, leftLocation),
+							"job_1": dyn.NewValue(1337, rightLocation),
+						},
+						leftLocation,
+					),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name:  "map - remove nested key",
+			state: visitorState{removed: []string{"root.jobs.job_1"}},
+			left: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, leftLocation),
+							"job_1": dyn.NewValue(1337, rightLocation),
+						},
+						leftLocation,
+					),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, rightLocation),
+						},
+						rightLocation,
+					),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				map[string]dyn.Value{
+					"jobs": dyn.NewValue(
+						map[string]dyn.Value{
+							"job_0": dyn.NewValue(42, leftLocation),
+						},
+						leftLocation,
+					),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name:  "sequence - add",
+			state: visitorState{added: []string{"root[1]"}},
+			left: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, rightLocation),
+					dyn.NewValue(10, rightLocation),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, leftLocation),
+					dyn.NewValue(10, rightLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name:  "sequence - remove",
+			state: visitorState{removed: []string{"root[1]"}},
+			left: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, leftLocation),
+					dyn.NewValue(10, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, rightLocation),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				[]dyn.Value{
+					// location hasn't changed because value hasn't changed
+					dyn.NewValue(42, leftLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name:  "sequence (not updated)",
+			state: visitorState{},
+			left: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, leftLocation),
+				},
+				leftLocation,
+			),
+			right: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, rightLocation),
+				},
+				rightLocation,
+			),
+			expected: dyn.NewValue(
+				[]dyn.Value{
+					dyn.NewValue(42, leftLocation),
+				},
+				leftLocation,
+			),
+		},
+		{
+			name:     "nil (not updated)",
+			state:    visitorState{},
+			left:     dyn.NilValue,
+			right:    dyn.NilValue,
+			expected: dyn.NilValue,
+		},
+		{
+			name:     "nil (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NilValue,
+			right:    dyn.NewValue(42, rightLocation),
+			expected: dyn.NewValue(42, rightLocation),
+		},
+		{
+			name:     "change kind (updated)",
+			state:    visitorState{updated: []string{"root"}},
+			left:     dyn.NewValue(42.0, leftLocation),
+			right:    dyn.NewValue(42, rightLocation),
+			expected: dyn.NewValue(42, rightLocation),
+		},
+	}
+
+	for _, tc := range modifiedTestCases {
+		t.Run(tc.name, func(t *testing.T) {
+			s, visitor := createVisitor()
+			out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor)
+
+			assert.NoError(t, err)
+			assert.Equal(t, tc.state, *s)
+			assert.Equal(t, tc.expected, out)
+		})
+	}
+}
+
+func TestOverride_PreserveMappingKeys(t *testing.T) {
+	leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1}
+	leftKeyLocation := dyn.Location{File: "left.yml", Line: 2, Column: 1}
+	leftValueLocation := dyn.Location{File: "left.yml", Line: 3, Column: 1}
+
+	rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1}
+	rightKeyLocation := dyn.Location{File: "right.yml", Line: 2, Column: 1}
+	rightValueLocation := dyn.Location{File: "right.yml", Line: 3, Column: 1}
+
+	left := dyn.NewMapping()
+	left.Set(dyn.NewValue("a", leftKeyLocation), dyn.NewValue(42, leftValueLocation))
+
+	right := dyn.NewMapping()
+	right.Set(dyn.NewValue("a", rightKeyLocation), dyn.NewValue(7, rightValueLocation))
+
+	state, visitor := createVisitor()
+
+	out, err := override(
+		dyn.EmptyPath,
+		dyn.NewValue(left, leftLocation),
+		dyn.NewValue(right, rightLocation),
+		visitor,
+	)
+
+	assert.NoError(t, err)
+
+	if err != nil {
+		outPairs := out.MustMap().Pairs()
+
+		assert.Equal(t, visitorState{updated: []string{"a"}}, state)
+		assert.Equal(t, 1, len(outPairs))
+
+		// mapping was first defined in left, so it should keep its location
+		assert.Equal(t, leftLocation, out.Location())
+
+		// if there is a validation error for key value, it should point
+		// to where it was initially defined
+		assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location())
+
+		// the value should have updated location, because it has changed
+		assert.Equal(t, rightValueLocation, outPairs[0].Value.Location())
+	}
+}
+
+type visitorState struct {
+	added   []string
+	removed []string
+	updated []string
+}
+
+func createVisitor() (*visitorState, OverrideVisitor) {
+	s := visitorState{}
+
+	return &s, OverrideVisitor{
+		VisitUpdate: func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) {
+			s.updated = append(s.updated, valuePath.String())
+
+			return right, nil
+		},
+		VisitDelete: func(valuePath dyn.Path, left dyn.Value) error {
+			s.removed = append(s.removed, valuePath.String())
+
+			return nil
+		},
+		VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) {
+			s.added = append(s.added, valuePath.String())
+
+			return right, nil
+		},
+	}
+}

From a014d50a6af94c911ee46e57a4e2ffd3c13e8e53 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Fri, 17 May 2024 12:10:17 +0200
Subject: [PATCH 26/41] Fixed panic when loading incorrectly defined jobs
 (#1402)

## Changes
If only key was defined for a job in YAML config, validate previously
failed with segfault.

This PR validates that jobs are correctly defined and returns an error
if not.

## Tests
Added regression test
---
 .../config/mutator/default_queueing_test.go   | 12 +++++-
 bundle/config/resources.go                    | 42 +++++++++++++++++++
 bundle/config/resources/job.go                |  9 ++++
 bundle/config/resources/mlflow_experiment.go  | 28 +++++++++++++
 bundle/config/resources/mlflow_model.go       | 28 +++++++++++++
 .../resources/model_serving_endpoint.go       | 28 +++++++++++++
 bundle/config/resources/pipeline.go           |  9 ++++
 bundle/config/resources/registered_model.go   | 28 +++++++++++++
 bundle/config/root.go                         |  8 ++++
 bundle/permissions/filter_test.go             |  7 ++++
 bundle/permissions/mutator_test.go            | 19 ++++++++-
 bundle/permissions/workspace_root_test.go     |  4 +-
 .../my_first_job/resource.yml                 |  1 +
 .../my_second_job/resource.yml                |  1 +
 bundle/tests/include_with_glob/job.yml        |  1 +
 bundle/tests/undefined_job/databricks.yml     |  8 ++++
 bundle/tests/undefined_job_test.go            | 12 ++++++
 17 files changed, 239 insertions(+), 6 deletions(-)
 create mode 100644 bundle/tests/undefined_job/databricks.yml
 create mode 100644 bundle/tests/undefined_job_test.go

diff --git a/bundle/config/mutator/default_queueing_test.go b/bundle/config/mutator/default_queueing_test.go
index ea60daf7f..d3621663b 100644
--- a/bundle/config/mutator/default_queueing_test.go
+++ b/bundle/config/mutator/default_queueing_test.go
@@ -56,7 +56,11 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) {
 		Config: config.Root{
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
-					"job": {},
+					"job": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job",
+						},
+					},
 				},
 			},
 		},
@@ -77,7 +81,11 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) {
 							Queue: &jobs.QueueSettings{Enabled: false},
 						},
 					},
-					"job2": {},
+					"job2": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job",
+						},
+					},
 					"job3": {
 						JobSettings: &jobs.JobSettings{
 							Queue: &jobs.QueueSettings{Enabled: true},
diff --git a/bundle/config/resources.go b/bundle/config/resources.go
index 457360a0c..41ffc25cd 100644
--- a/bundle/config/resources.go
+++ b/bundle/config/resources.go
@@ -126,6 +126,47 @@ func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker,
 	return tracker, nil
 }
 
+type resource struct {
+	resource      ConfigResource
+	resource_type string
+	key           string
+}
+
+func (r *Resources) allResources() []resource {
+	all := make([]resource, 0)
+	for k, e := range r.Jobs {
+		all = append(all, resource{resource_type: "job", resource: e, key: k})
+	}
+	for k, e := range r.Pipelines {
+		all = append(all, resource{resource_type: "pipeline", resource: e, key: k})
+	}
+	for k, e := range r.Models {
+		all = append(all, resource{resource_type: "model", resource: e, key: k})
+	}
+	for k, e := range r.Experiments {
+		all = append(all, resource{resource_type: "experiment", resource: e, key: k})
+	}
+	for k, e := range r.ModelServingEndpoints {
+		all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k})
+	}
+	for k, e := range r.RegisteredModels {
+		all = append(all, resource{resource_type: "registered model", resource: e, key: k})
+	}
+	return all
+}
+
+func (r *Resources) VerifyAllResourcesDefined() error {
+	all := r.allResources()
+	for _, e := range all {
+		err := e.resource.Validate()
+		if err != nil {
+			return fmt.Errorf("%s %s is not defined", e.resource_type, e.key)
+		}
+	}
+
+	return nil
+}
+
 // ConfigureConfigFilePath sets the specified path for all resources contained in this instance.
 // This property is used to correctly resolve paths relative to the path
 // of the configuration file they were defined in.
@@ -153,6 +194,7 @@ func (r *Resources) ConfigureConfigFilePath() {
 type ConfigResource interface {
 	Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error)
 	TerraformResourceName() string
+	Validate() error
 }
 
 func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) {
diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go
index 45e9662d9..dde5d5663 100644
--- a/bundle/config/resources/job.go
+++ b/bundle/config/resources/job.go
@@ -2,6 +2,7 @@ package resources
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 
 	"github.com/databricks/cli/bundle/config/paths"
@@ -47,3 +48,11 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri
 func (j *Job) TerraformResourceName() string {
 	return "databricks_job"
 }
+
+func (j *Job) Validate() error {
+	if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil {
+		return fmt.Errorf("job is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go
index 0f53096a0..7854ee7e8 100644
--- a/bundle/config/resources/mlflow_experiment.go
+++ b/bundle/config/resources/mlflow_experiment.go
@@ -1,7 +1,12 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/ml"
 )
@@ -23,3 +28,26 @@ func (s *MlflowExperiment) UnmarshalJSON(b []byte) error {
 func (s MlflowExperiment) MarshalJSON() ([]byte, error) {
 	return marshal.Marshal(s)
 }
+
+func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.Experiments.GetExperiment(ctx, ml.GetExperimentRequest{
+		ExperimentId: id,
+	})
+	if err != nil {
+		log.Debugf(ctx, "experiment %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *MlflowExperiment) TerraformResourceName() string {
+	return "databricks_mlflow_experiment"
+}
+
+func (s *MlflowExperiment) Validate() error {
+	if s == nil || !s.DynamicValue.IsValid() {
+		return fmt.Errorf("experiment is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go
index 59893aa47..40da9f87d 100644
--- a/bundle/config/resources/mlflow_model.go
+++ b/bundle/config/resources/mlflow_model.go
@@ -1,7 +1,12 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/ml"
 )
@@ -23,3 +28,26 @@ func (s *MlflowModel) UnmarshalJSON(b []byte) error {
 func (s MlflowModel) MarshalJSON() ([]byte, error) {
 	return marshal.Marshal(s)
 }
+
+func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.ModelRegistry.GetModel(ctx, ml.GetModelRequest{
+		Name: id,
+	})
+	if err != nil {
+		log.Debugf(ctx, "model %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *MlflowModel) TerraformResourceName() string {
+	return "databricks_mlflow_model"
+}
+
+func (s *MlflowModel) Validate() error {
+	if s == nil || !s.DynamicValue.IsValid() {
+		return fmt.Errorf("model is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go
index d1d57bafc..503cfbbb7 100644
--- a/bundle/config/resources/model_serving_endpoint.go
+++ b/bundle/config/resources/model_serving_endpoint.go
@@ -1,7 +1,12 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/serving"
 )
@@ -33,3 +38,26 @@ func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error {
 func (s ModelServingEndpoint) MarshalJSON() ([]byte, error) {
 	return marshal.Marshal(s)
 }
+
+func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.ServingEndpoints.Get(ctx, serving.GetServingEndpointRequest{
+		Name: id,
+	})
+	if err != nil {
+		log.Debugf(ctx, "serving endpoint %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *ModelServingEndpoint) TerraformResourceName() string {
+	return "databricks_model_serving"
+}
+
+func (s *ModelServingEndpoint) Validate() error {
+	if s == nil || !s.DynamicValue.IsValid() {
+		return fmt.Errorf("serving endpoint is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go
index 2f9ff8d0d..7e914b909 100644
--- a/bundle/config/resources/pipeline.go
+++ b/bundle/config/resources/pipeline.go
@@ -2,6 +2,7 @@ package resources
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/cli/libs/log"
@@ -42,3 +43,11 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id
 func (p *Pipeline) TerraformResourceName() string {
 	return "databricks_pipeline"
 }
+
+func (p *Pipeline) Validate() error {
+	if p == nil || !p.DynamicValue.IsValid() {
+		return fmt.Errorf("pipeline is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go
index 7b4b70d1a..fba643c69 100644
--- a/bundle/config/resources/registered_model.go
+++ b/bundle/config/resources/registered_model.go
@@ -1,7 +1,12 @@
 package resources
 
 import (
+	"context"
+	"fmt"
+
 	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/marshal"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 )
@@ -34,3 +39,26 @@ func (s *RegisteredModel) UnmarshalJSON(b []byte) error {
 func (s RegisteredModel) MarshalJSON() ([]byte, error) {
 	return marshal.Marshal(s)
 }
+
+func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.RegisteredModels.Get(ctx, catalog.GetRegisteredModelRequest{
+		FullName: id,
+	})
+	if err != nil {
+		log.Debugf(ctx, "registered model %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *RegisteredModel) TerraformResourceName() string {
+	return "databricks_registered_model"
+}
+
+func (s *RegisteredModel) Validate() error {
+	if s == nil || !s.DynamicValue.IsValid() {
+		return fmt.Errorf("registered model is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/config/root.go b/bundle/config/root.go
index fda3759dd..88197c2b8 100644
--- a/bundle/config/root.go
+++ b/bundle/config/root.go
@@ -138,6 +138,14 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error {
 	// Assign the normalized configuration tree.
 	r.value = nv
 
+	// At the moment the check has to be done as part of updateWithDynamicValue
+	// because otherwise ConfigureConfigFilePath will fail with a panic.
+	// In the future, we should move this check to a separate mutator in initialise phase.
+	err = r.Resources.VerifyAllResourcesDefined()
+	if err != nil {
+		return err
+	}
+
 	// Assign config file paths after converting to typed configuration.
 	r.ConfigureConfigFilePath()
 	return nil
diff --git a/bundle/permissions/filter_test.go b/bundle/permissions/filter_test.go
index 410fa4be8..121ce10dc 100644
--- a/bundle/permissions/filter_test.go
+++ b/bundle/permissions/filter_test.go
@@ -8,6 +8,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -45,9 +46,15 @@ func testFixture(userName string) *bundle.Bundle {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job1": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job1",
+						},
 						Permissions: p,
 					},
 					"job2": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job2",
+						},
 						Permissions: p,
 					},
 				},
diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go
index 438a15061..1a177d902 100644
--- a/bundle/permissions/mutator_test.go
+++ b/bundle/permissions/mutator_test.go
@@ -7,6 +7,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -23,8 +24,16 @@ func TestApplyBundlePermissions(t *testing.T) {
 			},
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
-					"job_1": {},
-					"job_2": {},
+					"job_1": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job_1",
+						},
+					},
+					"job_2": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job_2",
+						},
+					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline_1": {},
@@ -109,11 +118,17 @@ func TestWarningOnOverlapPermission(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job_1": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job_1",
+						},
 						Permissions: []resources.Permission{
 							{Level: CAN_VIEW, UserName: "TestUser"},
 						},
 					},
 					"job_2": {
+						JobSettings: &jobs.JobSettings{
+							Name: "job_2",
+						},
 						Permissions: []resources.Permission{
 							{Level: CAN_VIEW, UserName: "TestUser2"},
 						},
diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go
index 7dd97b62d..5e23a1da8 100644
--- a/bundle/permissions/workspace_root_test.go
+++ b/bundle/permissions/workspace_root_test.go
@@ -30,8 +30,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
 			},
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
-					"job_1": {JobSettings: &jobs.JobSettings{}},
-					"job_2": {JobSettings: &jobs.JobSettings{}},
+					"job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}},
+					"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
diff --git a/bundle/tests/include_multiple/my_first_job/resource.yml b/bundle/tests/include_multiple/my_first_job/resource.yml
index c2be5a160..4bd7c7164 100644
--- a/bundle/tests/include_multiple/my_first_job/resource.yml
+++ b/bundle/tests/include_multiple/my_first_job/resource.yml
@@ -2,3 +2,4 @@ resources:
   jobs:
     my_first_job:
       id: 1
+      name: "My First Job"
diff --git a/bundle/tests/include_multiple/my_second_job/resource.yml b/bundle/tests/include_multiple/my_second_job/resource.yml
index 2c28c4622..3a1514055 100644
--- a/bundle/tests/include_multiple/my_second_job/resource.yml
+++ b/bundle/tests/include_multiple/my_second_job/resource.yml
@@ -2,3 +2,4 @@ resources:
   jobs:
     my_second_job:
       id: 2
+      name: "My Second Job"
diff --git a/bundle/tests/include_with_glob/job.yml b/bundle/tests/include_with_glob/job.yml
index 3d609c529..a98577818 100644
--- a/bundle/tests/include_with_glob/job.yml
+++ b/bundle/tests/include_with_glob/job.yml
@@ -2,3 +2,4 @@ resources:
   jobs:
     my_job:
       id: 1
+      name: "My Job"
diff --git a/bundle/tests/undefined_job/databricks.yml b/bundle/tests/undefined_job/databricks.yml
new file mode 100644
index 000000000..12c19f946
--- /dev/null
+++ b/bundle/tests/undefined_job/databricks.yml
@@ -0,0 +1,8 @@
+bundle:
+  name: undefined-job
+
+resources:
+  jobs:
+    undefined:
+    test:
+      name: "Test Job"
diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go
new file mode 100644
index 000000000..ed502c471
--- /dev/null
+++ b/bundle/tests/undefined_job_test.go
@@ -0,0 +1,12 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUndefinedJobLoadsWithError(t *testing.T) {
+	_, diags := loadTargetWithDiags("./undefined_job", "default")
+	assert.ErrorContains(t, diags.Error(), "job undefined is not defined")
+}

From dc13e4b37e0bccce1d9d120a53065b1ebd42f521 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 08:29:24 +0200
Subject: [PATCH 27/41] Bump github.com/fatih/color from 1.16.0 to 1.17.0
 (#1441)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps [github.com/fatih/color](https://github.com/fatih/color) from
1.16.0 to 1.17.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/fatih/color/releases">github.com/fatih/color's
releases</a>.</em></p>
<blockquote>
<h2>v1.17.0</h2>
<h2>What's Changed</h2>
<ul>
<li>Fix multi-parameter println spacing by <a
href="https://github.com/klauspost"><code>@​klauspost</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/228">fatih/color#228</a></li>
<li>ci: update Go and Staticcheck versions by <a
href="https://github.com/fatih"><code>@​fatih</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/222">fatih/color#222</a></li>
<li>Bump golang.org/x/sys from 0.14.0 to 0.17.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/221">fatih/color#221</a></li>
<li>Bump actions/setup-go from 4 to 5 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/217">fatih/color#217</a></li>
<li>Bump golang.org/x/sys from 0.17.0 to 0.18.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/fatih/color/pull/224">fatih/color#224</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a href="https://github.com/klauspost"><code>@​klauspost</code></a>
made their first contribution in <a
href="https://redirect.github.com/fatih/color/pull/228">fatih/color#228</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/fatih/color/compare/v1.16.0...v1.17.0">https://github.com/fatih/color/compare/v1.16.0...v1.17.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/fatih/color/commit/b6598b12a645b3159c1bce51b0e3fafc269510be"><code>b6598b1</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/228">#228</a> from
klauspost/fix-println-issue-218</li>
<li><a
href="https://github.com/fatih/color/commit/00b1811a3d97bdd8f097cc128ab91600400e4c81"><code>00b1811</code></a>
Fix multi-parameter println spacing</li>
<li><a
href="https://github.com/fatih/color/commit/04994a819d7d5aa7183dc53be4cfe79fe8b4394c"><code>04994a8</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/224">#224</a> from
fatih/dependabot/go_modules/golang.org/x/sys-0.18.0</li>
<li><a
href="https://github.com/fatih/color/commit/7526cad07e356c227632f38d0c8a787b42dd5487"><code>7526cad</code></a>
Merge branch 'main' into
dependabot/go_modules/golang.org/x/sys-0.18.0</li>
<li><a
href="https://github.com/fatih/color/commit/8d058ca54cef8604970cccd2534773ebf90fa35b"><code>8d058ca</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/222">#222</a> from
fatih/ci-updates</li>
<li><a
href="https://github.com/fatih/color/commit/2ac809fb73860bc102e1369dc94860443c7d6514"><code>2ac809f</code></a>
Bump golang.org/x/sys from 0.17.0 to 0.18.0</li>
<li><a
href="https://github.com/fatih/color/commit/51a7bbf7dca7c369826997b8760aa8dcb026227a"><code>51a7bbf</code></a>
ci: update Go and Staticcheck versions</li>
<li><a
href="https://github.com/fatih/color/commit/799c49c6395ebde70c50f02c82cdf990e2ec31d2"><code>799c49c</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/217">#217</a> from
fatih/dependabot/github_actions/actions/setup-go-5</li>
<li><a
href="https://github.com/fatih/color/commit/f8e0ec9c97e622f589f2a9544186dc44859938ed"><code>f8e0ec9</code></a>
Merge branch 'main' into
dependabot/github_actions/actions/setup-go-5</li>
<li><a
href="https://github.com/fatih/color/commit/298abd8b9e6944ceb5b62a6368f05e2726df0752"><code>298abd8</code></a>
Merge pull request <a
href="https://redirect.github.com/fatih/color/issues/221">#221</a> from
fatih/dependabot/go_modules/golang.org/x/sys-0.17.0</li>
<li>Additional commits viewable in <a
href="https://github.com/fatih/color/compare/v1.16.0...v1.17.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/fatih/color&package-manager=go_modules&previous-version=1.16.0&new-version=1.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 6c8e845a5..d42a5d0f2 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1 // MIT
 	github.com/briandowns/spinner v1.23.0 // Apache 2.0
 	github.com/databricks/databricks-sdk-go v0.40.1 // Apache 2.0
-	github.com/fatih/color v1.16.0 // MIT
+	github.com/fatih/color v1.17.0 // MIT
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.6.0 // MPL 2.0
diff --git a/go.sum b/go.sum
index 222ce1e4c..37d848d24 100644
--- a/go.sum
+++ b/go.sum
@@ -40,8 +40,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
+github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=

From 7262138b4daf7b2cbb51d69f90e67b500682147c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 08:29:50 +0200
Subject: [PATCH 28/41] Bump github.com/hashicorp/terraform-json from 0.21.0 to
 0.22.1 (#1440)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[github.com/hashicorp/terraform-json](https://github.com/hashicorp/terraform-json)
from 0.21.0 to 0.22.1.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/terraform-json/releases">github.com/hashicorp/terraform-json's
releases</a>.</em></p>
<blockquote>
<h2>v0.22.1</h2>
<p>BUG FIXES:</p>
<ul>
<li>tfjson: Update <code>Complete</code> to a pointer value for older
Terraform versions by <a
href="https://github.com/austinvalle"><code>@​austinvalle</code></a> in
<a
href="https://redirect.github.com/hashicorp/terraform-json/pull/131">hashicorp/terraform-json#131</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/hashicorp/terraform-json/compare/v0.22.0...v0.22.1">https://github.com/hashicorp/terraform-json/compare/v0.22.0...v0.22.1</a></p>
<h2>v0.22.0</h2>
<p>ENHANCEMENTS:</p>
<ul>
<li>tfjson: Add <code>DeferredChanges</code> and <code>Complete</code>
to <code>Plan</code> JSON by <a
href="https://github.com/austinvalle"><code>@​austinvalle</code></a> in
<a
href="https://redirect.github.com/hashicorp/terraform-json/pull/123">hashicorp/terraform-json#123</a></li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>Bump github.com/zclconf/go-cty from 1.14.1 to 1.14.2 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/120">hashicorp/terraform-json#120</a></li>
<li>Bump github.com/zclconf/go-cty from 1.14.2 to 1.14.3 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/121">hashicorp/terraform-json#121</a></li>
<li>Bump github.com/zclconf/go-cty from 1.14.3 to 1.14.4 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/122">hashicorp/terraform-json#122</a></li>
<li>build(deps): Bump workflows to latest trusted versions by <a
href="https://github.com/hashicorp-tsccr"><code>@​hashicorp-tsccr</code></a>
in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/124">hashicorp/terraform-json#124</a></li>
<li>build(deps): Bump workflows to latest trusted versions by <a
href="https://github.com/hashicorp-tsccr"><code>@​hashicorp-tsccr</code></a>
in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/127">hashicorp/terraform-json#127</a></li>
<li>github: Set up Dependabot to manage HashiCorp-owned Actions versions
by <a href="https://github.com/xiehan"><code>@​xiehan</code></a> in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/128">hashicorp/terraform-json#128</a></li>
<li>Bump hashicorp/setup-copywrite from 1.1.2 to 1.1.3 in the
github-actions-backward-compatible group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/130">hashicorp/terraform-json#130</a></li>
</ul>
<p>New Contributors</p>
<ul>
<li><a
href="https://github.com/austinvalle"><code>@​austinvalle</code></a>
made their first contribution in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/123">hashicorp/terraform-json#123</a></li>
<li><a href="https://github.com/xiehan"><code>@​xiehan</code></a> made
their first contribution in <a
href="https://redirect.github.com/hashicorp/terraform-json/pull/128">hashicorp/terraform-json#128</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/hashicorp/terraform-json/compare/v0.21.0...v0.22.0">https://github.com/hashicorp/terraform-json/compare/v0.21.0...v0.22.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/7e28e2d84614a86416fd8a9991129670173d4125"><code>7e28e2d</code></a>
tfjson: Update <code>Complete</code> to a pointer value for older
Terraform versions (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/131">#131</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/5e08e15af60aeaeb1ee04686fbb2f6e9de819840"><code>5e08e15</code></a>
Bump hashicorp/setup-copywrite (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/130">#130</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/4a9d1e78cc5db19f77bdb36161c3fdd6b0cfa089"><code>4a9d1e7</code></a>
github: Set up Dependabot to manage HashiCorp-owned Actions versions (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/128">#128</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/11f603ef1c8eb2a559347e031e79515f06165630"><code>11f603e</code></a>
Result of tsccr-helper -log-level=info gha update -latest . (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/127">#127</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/6e83e7bb60b92923d7c4178609e36616c658b74e"><code>6e83e7b</code></a>
Result of tsccr-helper -log-level=info gha update -latest . (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/124">#124</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/3b8a921d9133ceaa919dc749289b904ced1995ca"><code>3b8a921</code></a>
tfjson: Add <code>DeferredChanges</code> and <code>Complete</code> to
<code>Plan</code> JSON (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/123">#123</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/8cba21aed5ce23c8d464e21d6c04d233ca3d50f6"><code>8cba21a</code></a>
Bump github.com/zclconf/go-cty from 1.14.3 to 1.14.4 (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/122">#122</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/d5065f28bc5d8954facbe2de110c4b1a5634cba6"><code>d5065f2</code></a>
Bump github.com/zclconf/go-cty from 1.14.2 to 1.14.3 (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/121">#121</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-json/commit/1498774802abe883780f85a5feb87d8e371d0884"><code>1498774</code></a>
Bump github.com/zclconf/go-cty from 1.14.1 to 1.14.2 (<a
href="https://redirect.github.com/hashicorp/terraform-json/issues/120">#120</a>)</li>
<li>See full diff in <a
href="https://github.com/hashicorp/terraform-json/compare/v0.21.0...v0.22.1">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/terraform-json&package-manager=go_modules&previous-version=0.21.0&new-version=0.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index d42a5d0f2..4a6ea2e03 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.6.4 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.20.0 // MPL 2.0
-	github.com/hashicorp/terraform-json v0.21.0 // MPL 2.0
+	github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
 	github.com/mattn/go-isatty v0.0.20 // MIT
 	github.com/nwidger/jsoncolor v0.3.2 // MIT
@@ -51,7 +51,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/zclconf/go-cty v1.14.1 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
diff --git a/go.sum b/go.sum
index 37d848d24..051774b4b 100644
--- a/go.sum
+++ b/go.sum
@@ -103,8 +103,8 @@ github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9
 github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA=
 github.com/hashicorp/terraform-exec v0.20.0 h1:DIZnPsqzPGuUnq6cH8jWcPunBfY+C+M8JyYF3vpnuEo=
 github.com/hashicorp/terraform-exec v0.20.0/go.mod h1:ckKGkJWbsNqFKV1itgMnE0hY9IYf1HoiekpuN0eWoDw=
-github.com/hashicorp/terraform-json v0.21.0 h1:9NQxbLNqPbEMze+S6+YluEdXgJmhQykRyRNd+zTI05U=
-github.com/hashicorp/terraform-json v0.21.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk=
+github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=
+github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -154,8 +154,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
-github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA=
-github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
+github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=

From 3ce833f82602ede05801078adfd44a51d317cdd8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 21 May 2024 08:48:16 +0200
Subject: [PATCH 29/41] Bump github.com/hashicorp/terraform-exec from 0.20.0 to
 0.21.0 (#1442)

Bumps
[github.com/hashicorp/terraform-exec](https://github.com/hashicorp/terraform-exec)
from 0.20.0 to 0.21.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/terraform-exec/releases">github.com/hashicorp/terraform-exec's
releases</a>.</em></p>
<blockquote>
<h2>v0.21.0</h2>
<p>ENHANCEMENTS:</p>
<ul>
<li>tfexec: Add <code>-allow-deferral</code> to
<code>(Terraform).Apply()</code> and <code>(Terraform).Plan()</code>
methods (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/447">#447</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/terraform-exec/blob/main/CHANGELOG.md">github.com/hashicorp/terraform-exec's
changelog</a>.</em></p>
<blockquote>
<h1>0.21.0 (May 17, 2024)</h1>
<p>ENHANCEMENTS:</p>
<ul>
<li>tfexec: Add <code>-allow-deferral</code> to
<code>(Terraform).Apply()</code> and <code>(Terraform).Plan()</code>
methods (<a
href="https://redirect.github.com/hashicorp/terraform-exec/pull/447">#447</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/b6ae175a49f3e552231eeeceed13d9a4d28e711e"><code>b6ae175</code></a>
v0.21.0 [skip ci]</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/67e92f47c08de2aecfb4bc2493442475d4779986"><code>67e92f4</code></a>
build(deps): bump github.com/hashicorp/terraform-json from 0.22.0 to
0.22.1 (...</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/64df8d266b23598a5765cd77a3081ace4eff08a2"><code>64df8d2</code></a>
build(deps): bump github.com/hashicorp/terraform-json from 0.21.0 to
0.22.0 (...</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/af05782eead4ebc78f05157c45e0c7cccb671100"><code>af05782</code></a>
build(deps): Bump workflows to latest trusted versions (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/450">#450</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/1df7d521ed16040ff415e63607f29bf98fc15fc9"><code>1df7d52</code></a>
build(deps): bump golang.org/x/net from 0.22.0 to 0.23.0 (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/444">#444</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/6ea72951a2437cb38f987422bff8f1ca0bd10fcd"><code>6ea7295</code></a>
build(deps): bump hashicorp/setup-copywrite from 1.1.2 to 1.1.3 in the
github...</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/a9c9728194c4c0aa04da955610d9cf0ada95e2bd"><code>a9c9728</code></a>
tfexec: Add <code>-allow-deferral</code> experimental options to
<code>Plan</code> and <code>Apply</code> comm...</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/c07c6785b7056aa790f30d42d4f277dd68dd3cd9"><code>c07c678</code></a>
Reenable Dependabot for internal GitHub actions (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/455">#455</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/259b9e96e91633cb4e322c4b6360bc038d9180de"><code>259b9e9</code></a>
build(deps): bump github.com/hashicorp/hc-install from 0.6.3 to 0.6.4
(<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/443">#443</a>)</li>
<li><a
href="https://github.com/hashicorp/terraform-exec/commit/46360f17ab656eacfacc2ada9bcc4a355e53d08d"><code>46360f1</code></a>
build(deps): bump github.com/zclconf/go-cty from 1.14.3 to 1.14.4 (<a
href="https://redirect.github.com/hashicorp/terraform-exec/issues/441">#441</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/hashicorp/terraform-exec/compare/v0.20.0...v0.21.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/terraform-exec&package-manager=go_modules&previous-version=0.20.0&new-version=0.21.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 4a6ea2e03..ddebe9727 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.6.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.6.4 // MPL 2.0
-	github.com/hashicorp/terraform-exec v0.20.0 // MPL 2.0
+	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
 	github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
 	github.com/mattn/go-isatty v0.0.20 // MIT
diff --git a/go.sum b/go.sum
index 051774b4b..1dccbb2f9 100644
--- a/go.sum
+++ b/go.sum
@@ -101,8 +101,8 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9Km8e0=
 github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA=
-github.com/hashicorp/terraform-exec v0.20.0 h1:DIZnPsqzPGuUnq6cH8jWcPunBfY+C+M8JyYF3vpnuEo=
-github.com/hashicorp/terraform-exec v0.20.0/go.mod h1:ckKGkJWbsNqFKV1itgMnE0hY9IYf1HoiekpuN0eWoDw=
+github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
+github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
 github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=
 github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

From 09aa3cb9e93581d4cce0aef024ad48723c45a683 Mon Sep 17 00:00:00 2001
From: Gleb Kanterov <kanterov@users.noreply.github.com>
Date: Tue, 21 May 2024 08:48:42 +0200
Subject: [PATCH 30/41] Add more tests for `merge.Override` (#1439)

## Changes
Add test coverage to ensure we respect return value and error

## Tests
Unit tests
---
 libs/dyn/merge/override_test.go | 65 ++++++++++++++++++++++++++++++---
 1 file changed, 59 insertions(+), 6 deletions(-)

diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go
index dbf249d12..a34f23424 100644
--- a/libs/dyn/merge/override_test.go
+++ b/libs/dyn/merge/override_test.go
@@ -1,6 +1,7 @@
 package merge
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -351,13 +352,48 @@ func TestOverride_Primitive(t *testing.T) {
 
 	for _, tc := range modifiedTestCases {
 		t.Run(tc.name, func(t *testing.T) {
-			s, visitor := createVisitor()
+			s, visitor := createVisitor(visitorOpts{})
 			out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor)
 
 			assert.NoError(t, err)
 			assert.Equal(t, tc.state, *s)
 			assert.Equal(t, tc.expected, out)
 		})
+
+		modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0
+
+		// visitor is not used unless there is a change
+
+		if modified {
+			t.Run(tc.name+" - visitor has error", func(t *testing.T) {
+				_, visitor := createVisitor(visitorOpts{error: fmt.Errorf("unexpected change in test")})
+				_, err := override(dyn.EmptyPath, tc.left, tc.right, visitor)
+
+				assert.EqualError(t, err, "unexpected change in test")
+			})
+
+			t.Run(tc.name+" - visitor overrides value", func(t *testing.T) {
+				expected := dyn.NewValue("return value", dyn.Location{})
+				s, visitor := createVisitor(visitorOpts{returnValue: &expected})
+				out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor)
+
+				assert.NoError(t, err)
+
+				for _, added := range s.added {
+					actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added))
+
+					assert.NoError(t, err)
+					assert.Equal(t, expected, actual)
+				}
+
+				for _, updated := range s.updated {
+					actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated))
+
+					assert.NoError(t, err)
+					assert.Equal(t, expected, actual)
+				}
+			})
+		}
 	}
 }
 
@@ -376,7 +412,7 @@ func TestOverride_PreserveMappingKeys(t *testing.T) {
 	right := dyn.NewMapping()
 	right.Set(dyn.NewValue("a", rightKeyLocation), dyn.NewValue(7, rightValueLocation))
 
-	state, visitor := createVisitor()
+	state, visitor := createVisitor(visitorOpts{})
 
 	out, err := override(
 		dyn.EmptyPath,
@@ -411,24 +447,41 @@ type visitorState struct {
 	updated []string
 }
 
-func createVisitor() (*visitorState, OverrideVisitor) {
+type visitorOpts struct {
+	error       error
+	returnValue *dyn.Value
+}
+
+func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) {
 	s := visitorState{}
 
 	return &s, OverrideVisitor{
 		VisitUpdate: func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) {
 			s.updated = append(s.updated, valuePath.String())
 
-			return right, nil
+			if opts.error != nil {
+				return dyn.NilValue, opts.error
+			} else if opts.returnValue != nil {
+				return *opts.returnValue, nil
+			} else {
+				return right, nil
+			}
 		},
 		VisitDelete: func(valuePath dyn.Path, left dyn.Value) error {
 			s.removed = append(s.removed, valuePath.String())
 
-			return nil
+			return opts.error
 		},
 		VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) {
 			s.added = append(s.added, valuePath.String())
 
-			return right, nil
+			if opts.error != nil {
+				return dyn.NilValue, opts.error
+			} else if opts.returnValue != nil {
+				return *opts.returnValue, nil
+			} else {
+				return right, nil
+			}
 		},
 	}
 }

From 3f8036f2dfbf2624be3c20edc43393487579c8a2 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Tue, 21 May 2024 12:00:04 +0200
Subject: [PATCH 31/41] Fixed seg fault when specifying environment key for
 tasks (#1443)

## Changes
Fixed seg fault when specifying environment key for tasks
---
 bundle/artifacts/artifacts.go                    |  4 ++++
 bundle/libraries/libraries.go                    |  4 ++++
 bundle/libraries/match.go                        |  4 ++++
 bundle/tests/enviroment_key_test.go              | 11 +++++++++++
 bundle/tests/environment_key_only/databricks.yml | 16 ++++++++++++++++
 5 files changed, 39 insertions(+)
 create mode 100644 bundle/tests/environment_key_only/databricks.yml

diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go
index 101b598dd..470c329a1 100644
--- a/bundle/artifacts/artifacts.go
+++ b/bundle/artifacts/artifacts.go
@@ -150,6 +150,10 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
 
 			for i := range job.Environments {
 				env := &job.Environments[i]
+				if env.Spec == nil {
+					continue
+				}
+
 				for j := range env.Spec.Dependencies {
 					lib := env.Spec.Dependencies[j]
 					if isArtifactMatchLibrary(f, lib, b) {
diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go
index a79adedbf..84ead052b 100644
--- a/bundle/libraries/libraries.go
+++ b/bundle/libraries/libraries.go
@@ -30,6 +30,10 @@ func FindAllEnvironments(b *bundle.Bundle) map[string]([]jobs.JobEnvironment) {
 
 func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool {
 	for _, e := range envs {
+		if e.Spec == nil {
+			continue
+		}
+
 		for _, l := range e.Spec.Dependencies {
 			if IsEnvironmentDependencyLocal(l) {
 				return true
diff --git a/bundle/libraries/match.go b/bundle/libraries/match.go
index 096cdf4a5..4feb4225d 100644
--- a/bundle/libraries/match.go
+++ b/bundle/libraries/match.go
@@ -62,6 +62,10 @@ func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error {
 
 func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error {
 	for _, env := range envs {
+		if env.Spec == nil {
+			continue
+		}
+
 		for _, dep := range env.Spec.Dependencies {
 			matches, err := filepath.Glob(filepath.Join(b.RootPath, dep))
 			if err != nil {
diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go
index 3e12ddb68..aed3964db 100644
--- a/bundle/tests/enviroment_key_test.go
+++ b/bundle/tests/enviroment_key_test.go
@@ -1,8 +1,11 @@
 package config_tests
 
 import (
+	"context"
 	"testing"
 
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/libraries"
 	"github.com/stretchr/testify/require"
 )
 
@@ -10,3 +13,11 @@ func TestEnvironmentKeySupported(t *testing.T) {
 	_, diags := loadTargetWithDiags("./python_wheel/environment_key", "default")
 	require.Empty(t, diags)
 }
+
+func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) {
+	b, diags := loadTargetWithDiags("./environment_key_only", "default")
+	require.Empty(t, diags)
+
+	diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist())
+	require.Empty(t, diags)
+}
diff --git a/bundle/tests/environment_key_only/databricks.yml b/bundle/tests/environment_key_only/databricks.yml
new file mode 100644
index 000000000..caa34f8e3
--- /dev/null
+++ b/bundle/tests/environment_key_only/databricks.yml
@@ -0,0 +1,16 @@
+bundle:
+  name: environment_key_only
+
+resources:
+  jobs:
+    test_job:
+      name: "My Wheel Job"
+      tasks:
+        - task_key: TestTask
+          existing_cluster_id: "0717-132531-5opeqon1"
+          python_wheel_task:
+            package_name: "my_test_code"
+            entry_point: "run"
+          environment_key:  "test_env"
+      environments:
+        - environment_key:  "test_env"

From c5032644a0c218e5b4c96f49eeaeb7a7b03985e4 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Tue, 21 May 2024 17:23:00 +0530
Subject: [PATCH 32/41] Fix conversion of zero valued scalar pointers to a
 dynamic value (#1433)

## Changes
This PR also fixes empty values variable overrides using the --var flag.
Now, using `--var="my_variable="` will set the value of `my_variable` to
the empty string instead of ignoring the flag altogether.

## Tests
The change using a unit test. Manually verified the `--var` flag works
now.
---
 libs/dyn/convert/end_to_end_test.go | 45 +++++++++++++++++++++++++++++
 libs/dyn/convert/from_typed.go      | 42 ++++++++++++++++++---------
 2 files changed, 73 insertions(+), 14 deletions(-)

diff --git a/libs/dyn/convert/end_to_end_test.go b/libs/dyn/convert/end_to_end_test.go
index 33902bea8..f0e428a69 100644
--- a/libs/dyn/convert/end_to_end_test.go
+++ b/libs/dyn/convert/end_to_end_test.go
@@ -67,4 +67,49 @@ func TestAdditional(t *testing.T) {
 			SliceOfPointer: []*string{nil},
 		})
 	})
+
+	t.Run("pointer to a empty string", func(t *testing.T) {
+		s := ""
+		assertFromTypedToTypedEqual(t, &s)
+	})
+
+	t.Run("nil pointer", func(t *testing.T) {
+		var s *string
+		assertFromTypedToTypedEqual(t, s)
+	})
+
+	t.Run("pointer to struct with scalar values", func(t *testing.T) {
+		s := ""
+		type foo struct {
+			A string  `json:"a"`
+			B int     `json:"b"`
+			C bool    `json:"c"`
+			D *string `json:"d"`
+		}
+		assertFromTypedToTypedEqual(t, &foo{
+			A: "a",
+			B: 1,
+			C: true,
+			D: &s,
+		})
+		assertFromTypedToTypedEqual(t, &foo{
+			A: "",
+			B: 0,
+			C: false,
+			D: nil,
+		})
+	})
+
+	t.Run("map with scalar values", func(t *testing.T) {
+		assertFromTypedToTypedEqual(t, map[string]string{
+			"a": "a",
+			"b": "b",
+			"c": "",
+		})
+		assertFromTypedToTypedEqual(t, map[string]int{
+			"a": 1,
+			"b": 0,
+			"c": 2,
+		})
+	})
 }
diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go
index c344d12df..ae491d8ab 100644
--- a/libs/dyn/convert/from_typed.go
+++ b/libs/dyn/convert/from_typed.go
@@ -12,16 +12,22 @@ import (
 type fromTypedOptions int
 
 const (
-	// Use the zero value instead of setting zero values to nil. This is useful
-	// for types where the zero values and nil are semantically different. That is
-	// strings, bools, ints, floats.
+	// If this flag is set, zero values for scalars (strings, bools, ints, floats)
+	// would resolve to corresponding zero values in the dynamic representation.
+	// Otherwise, zero values for scalars resolve to dyn.NilValue.
 	//
-	// Note: this is not needed for structs because dyn.NilValue is converted back
-	// to a zero value when using the convert.ToTyped function.
+	// This flag exists to reconcile the default values for scalars in a Go struct
+	// being zero values with zero values in a dynamic representation. In a Go struct,
+	// zero values are the same as the values not being set at all. This is not the case
+	// in the dynamic representation.
 	//
-	// Values in maps and slices should be set to zero values, and not nil in the
-	// dynamic representation.
-	includeZeroValues fromTypedOptions = 1 << iota
+	// If a scalar value in a typed Go struct is zero, in the dynamic representation
+	// we would set it to dyn.NilValue, i.e. equivalent to the value not being set at all.
+	//
+	// If a scalar value in a Go map, slice or pointer is set to zero, we will set it
+	// to the zero value in the dynamic representation, and not dyn.NilValue. This is
+	// equivalent to the value being intentionally set to zero.
+	includeZeroValuedScalars fromTypedOptions = 1 << iota
 )
 
 // FromTyped converts changes made in the typed structure w.r.t. the configuration value
@@ -41,6 +47,14 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value,
 			return dyn.NilValue, nil
 		}
 		srcv = srcv.Elem()
+
+		// If a pointer to a scalar type points to a zero value, we should include
+		// that zero value in the dynamic representation.
+		// This is because by default a pointer is nil in Go, and it not being nil
+		// indicates its value was intentionally set to zero.
+		if !slices.Contains(options, includeZeroValuedScalars) {
+			options = append(options, includeZeroValuedScalars)
+		}
 	}
 
 	switch srcv.Kind() {
@@ -129,7 +143,7 @@ func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 		}
 
 		// Convert entry taking into account the reference value (may be equal to dyn.NilValue).
-		nv, err := fromTyped(v.Interface(), refv, includeZeroValues)
+		nv, err := fromTyped(v.Interface(), refv, includeZeroValuedScalars)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
@@ -160,7 +174,7 @@ func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) {
 		v := src.Index(i)
 
 		// Convert entry taking into account the reference value (may be equal to dyn.NilValue).
-		nv, err := fromTyped(v.Interface(), ref.Index(i), includeZeroValues)
+		nv, err := fromTyped(v.Interface(), ref.Index(i), includeZeroValuedScalars)
 		if err != nil {
 			return dyn.InvalidValue, err
 		}
@@ -183,7 +197,7 @@ func fromTypedString(src reflect.Value, ref dyn.Value, options ...fromTypedOptio
 	case dyn.KindNil:
 		// This field is not set in the reference. We set it to nil if it's zero
 		// valued in the typed representation and the includeZeroValues option is not set.
-		if src.IsZero() && !slices.Contains(options, includeZeroValues) {
+		if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) {
 			return dyn.NilValue, nil
 		}
 		return dyn.V(src.String()), nil
@@ -203,7 +217,7 @@ func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions
 	case dyn.KindNil:
 		// This field is not set in the reference. We set it to nil if it's zero
 		// valued in the typed representation and the includeZeroValues option is not set.
-		if src.IsZero() && !slices.Contains(options, includeZeroValues) {
+		if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) {
 			return dyn.NilValue, nil
 		}
 		return dyn.V(src.Bool()), nil
@@ -228,7 +242,7 @@ func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions)
 	case dyn.KindNil:
 		// This field is not set in the reference. We set it to nil if it's zero
 		// valued in the typed representation and the includeZeroValues option is not set.
-		if src.IsZero() && !slices.Contains(options, includeZeroValues) {
+		if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) {
 			return dyn.NilValue, nil
 		}
 		return dyn.V(src.Int()), nil
@@ -253,7 +267,7 @@ func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOption
 	case dyn.KindNil:
 		// This field is not set in the reference. We set it to nil if it's zero
 		// valued in the typed representation and the includeZeroValues option is not set.
-		if src.IsZero() && !slices.Contains(options, includeZeroValues) {
+		if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) {
 			return dyn.NilValue, nil
 		}
 		return dyn.V(src.Float()), nil

From 63ceede3350ad87929ecf0cb6df78fd6a3c1ae37 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 22 May 2024 09:41:32 +0200
Subject: [PATCH 33/41] Update Go SDK to v0.41.0 (#1445)

## Changes

Release notes at
https://github.com/databricks/databricks-sdk-go/releases/tag/v0.41.0.

## Tests

n/a
---
 .codegen/_openapi_sha                         |   2 +-
 .gitattributes                                |   2 +-
 bundle/schema/docs/bundle_descriptions.json   | 216 ++++++++++++++----
 .../esm-enablement-account.go                 |   3 -
 .../automatic-cluster-update.go               |   3 -
 cmd/workspace/clusters/clusters.go            |  23 +-
 cmd/workspace/cmd.go                          |   4 +-
 .../compliance-security-profile.go            |   3 -
 cmd/workspace/connections/connections.go      |  22 +-
 .../consumer-listings/consumer-listings.go    |   4 +-
 .../enhanced-security-monitoring.go           |   3 -
 cmd/workspace/libraries/libraries.go          |   7 +-
 cmd/workspace/pipelines/pipelines.go          |   1 +
 .../quality-monitors.go}                      |  31 +--
 .../serving-endpoints/serving-endpoints.go    |   6 +-
 cmd/workspace/shares/shares.go                |   5 +
 .../system-schemas/system-schemas.go          |  12 +-
 .../vector-search-indexes.go                  |  71 ++++++
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 20 files changed, 297 insertions(+), 127 deletions(-)
 rename cmd/workspace/{lakehouse-monitors/lakehouse-monitors.go => quality-monitors/quality-monitors.go} (95%)

diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index f07cf44e5..8c62ac620 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-9bb7950fa3390afb97abaa552934bc0a2e069de5
\ No newline at end of file
+7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
index fb42588a7..c11257e9e 100755
--- a/.gitattributes
+++ b/.gitattributes
@@ -62,7 +62,6 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true
 cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true
 cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true
 cmd/workspace/jobs/jobs.go linguist-generated=true
-cmd/workspace/lakehouse-monitors/lakehouse-monitors.go linguist-generated=true
 cmd/workspace/lakeview/lakeview.go linguist-generated=true
 cmd/workspace/libraries/libraries.go linguist-generated=true
 cmd/workspace/metastores/metastores.go linguist-generated=true
@@ -81,6 +80,7 @@ cmd/workspace/provider-personalization-requests/provider-personalization-request
 cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go linguist-generated=true
 cmd/workspace/provider-providers/provider-providers.go linguist-generated=true
 cmd/workspace/providers/providers.go linguist-generated=true
+cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true
 cmd/workspace/queries/queries.go linguist-generated=true
 cmd/workspace/query-history/query-history.go linguist-generated=true
 cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true
diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json
index ba6fe8ce2..b6d0235aa 100644
--- a/bundle/schema/docs/bundle_descriptions.json
+++ b/bundle/schema/docs/bundle_descriptions.json
@@ -348,7 +348,7 @@
                       "description": "If new_cluster, a description of a cluster that is created for each task.",
                       "properties": {
                         "apply_policy_default_values": {
-                          "description": ""
+                          "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
                         },
                         "autoscale": {
                           "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@@ -424,14 +424,6 @@
                             }
                           }
                         },
-                        "clone_from": {
-                          "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
-                          "properties": {
-                            "source_cluster_id": {
-                              "description": "The cluster that is being cloned."
-                            }
-                          }
-                        },
                         "cluster_log_conf": {
                           "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                           "properties": {
@@ -474,9 +466,6 @@
                         "cluster_name": {
                           "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
                         },
-                        "cluster_source": {
-                          "description": ""
-                        },
                         "custom_tags": {
                           "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
                           "additionalproperties": {
@@ -975,7 +964,7 @@
                       "description": "If new_cluster, a description of a new cluster that is created for each run.",
                       "properties": {
                         "apply_policy_default_values": {
-                          "description": ""
+                          "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
                         },
                         "autoscale": {
                           "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@@ -1051,14 +1040,6 @@
                             }
                           }
                         },
-                        "clone_from": {
-                          "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
-                          "properties": {
-                            "source_cluster_id": {
-                              "description": "The cluster that is being cloned."
-                            }
-                          }
-                        },
                         "cluster_log_conf": {
                           "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                           "properties": {
@@ -1101,9 +1082,6 @@
                         "cluster_name": {
                           "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
                         },
-                        "cluster_source": {
-                          "description": ""
-                        },
                         "custom_tags": {
                           "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
                           "additionalproperties": {
@@ -1419,7 +1397,7 @@
                           }
                         },
                         "python_named_params": {
-                          "description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.",
+                          "description": "",
                           "additionalproperties": {
                             "description": ""
                           }
@@ -1853,6 +1831,15 @@
                             "openai_config": {
                               "description": "OpenAI Config. Only required if the provider is 'openai'.",
                               "properties": {
+                                "microsoft_entra_client_id": {
+                                  "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n"
+                                },
+                                "microsoft_entra_client_secret": {
+                                  "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n"
+                                },
+                                "microsoft_entra_tenant_id": {
+                                  "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n"
+                                },
                                 "openai_api_base": {
                                   "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n"
                                 },
@@ -2009,6 +1996,9 @@
                   }
                 }
               },
+              "route_optimized": {
+                "description": "Enable route optimization for the serving endpoint."
+              },
               "tags": {
                 "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.",
                 "items": {
@@ -2469,6 +2459,23 @@
                   }
                 }
               },
+              "gateway_definition": {
+                "description": "The definition of a gateway pipeline to support CDC.",
+                "properties": {
+                  "connection_id": {
+                    "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source."
+                  },
+                  "gateway_storage_catalog": {
+                    "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location."
+                  },
+                  "gateway_storage_name": {
+                    "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n"
+                  },
+                  "gateway_storage_schema": {
+                    "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location."
+                  }
+                }
+              },
               "id": {
                 "description": "Unique identifier for this pipeline."
               },
@@ -2500,6 +2507,23 @@
                             },
                             "source_schema": {
                               "description": "Required. Schema name in the source database."
+                            },
+                            "table_configuration": {
+                              "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.",
+                              "properties": {
+                                "primary_keys": {
+                                  "description": "The primary key of the table used to apply changes.",
+                                  "items": {
+                                    "description": ""
+                                  }
+                                },
+                                "salesforce_include_formula_fields": {
+                                  "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                                },
+                                "scd_type": {
+                                  "description": "The SCD type to use to ingest the table."
+                                }
+                              }
                             }
                           }
                         },
@@ -2523,11 +2547,45 @@
                             },
                             "source_table": {
                               "description": "Required. Table name in the source database."
+                            },
+                            "table_configuration": {
+                              "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.",
+                              "properties": {
+                                "primary_keys": {
+                                  "description": "The primary key of the table used to apply changes.",
+                                  "items": {
+                                    "description": ""
+                                  }
+                                },
+                                "salesforce_include_formula_fields": {
+                                  "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                                },
+                                "scd_type": {
+                                  "description": "The SCD type to use to ingest the table."
+                                }
+                              }
                             }
                           }
                         }
                       }
                     }
+                  },
+                  "table_configuration": {
+                    "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.",
+                    "properties": {
+                      "primary_keys": {
+                        "description": "The primary key of the table used to apply changes.",
+                        "items": {
+                          "description": ""
+                        }
+                      },
+                      "salesforce_include_formula_fields": {
+                        "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                      },
+                      "scd_type": {
+                        "description": "The SCD type to use to ingest the table."
+                      }
+                    }
                   }
                 }
               },
@@ -3071,7 +3129,7 @@
                             "description": "If new_cluster, a description of a cluster that is created for each task.",
                             "properties": {
                               "apply_policy_default_values": {
-                                "description": ""
+                                "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
                               },
                               "autoscale": {
                                 "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@@ -3147,14 +3205,6 @@
                                   }
                                 }
                               },
-                              "clone_from": {
-                                "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
-                                "properties": {
-                                  "source_cluster_id": {
-                                    "description": "The cluster that is being cloned."
-                                  }
-                                }
-                              },
                               "cluster_log_conf": {
                                 "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                                 "properties": {
@@ -3197,9 +3247,6 @@
                               "cluster_name": {
                                 "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
                               },
-                              "cluster_source": {
-                                "description": ""
-                              },
                               "custom_tags": {
                                 "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
                                 "additionalproperties": {
@@ -3698,7 +3745,7 @@
                             "description": "If new_cluster, a description of a new cluster that is created for each run.",
                             "properties": {
                               "apply_policy_default_values": {
-                                "description": ""
+                                "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
                               },
                               "autoscale": {
                                 "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@@ -3774,14 +3821,6 @@
                                   }
                                 }
                               },
-                              "clone_from": {
-                                "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
-                                "properties": {
-                                  "source_cluster_id": {
-                                    "description": "The cluster that is being cloned."
-                                  }
-                                }
-                              },
                               "cluster_log_conf": {
                                 "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                                 "properties": {
@@ -3824,9 +3863,6 @@
                               "cluster_name": {
                                 "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
                               },
-                              "cluster_source": {
-                                "description": ""
-                              },
                               "custom_tags": {
                                 "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
                                 "additionalproperties": {
@@ -4142,7 +4178,7 @@
                                 }
                               },
                               "python_named_params": {
-                                "description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.",
+                                "description": "",
                                 "additionalproperties": {
                                   "description": ""
                                 }
@@ -4576,6 +4612,15 @@
                                   "openai_config": {
                                     "description": "OpenAI Config. Only required if the provider is 'openai'.",
                                     "properties": {
+                                      "microsoft_entra_client_id": {
+                                        "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n"
+                                      },
+                                      "microsoft_entra_client_secret": {
+                                        "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n"
+                                      },
+                                      "microsoft_entra_tenant_id": {
+                                        "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n"
+                                      },
                                       "openai_api_base": {
                                         "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n"
                                       },
@@ -4732,6 +4777,9 @@
                         }
                       }
                     },
+                    "route_optimized": {
+                      "description": "Enable route optimization for the serving endpoint."
+                    },
                     "tags": {
                       "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.",
                       "items": {
@@ -5192,6 +5240,23 @@
                         }
                       }
                     },
+                    "gateway_definition": {
+                      "description": "The definition of a gateway pipeline to support CDC.",
+                      "properties": {
+                        "connection_id": {
+                          "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source."
+                        },
+                        "gateway_storage_catalog": {
+                          "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location."
+                        },
+                        "gateway_storage_name": {
+                          "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n"
+                        },
+                        "gateway_storage_schema": {
+                          "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location."
+                        }
+                      }
+                    },
                     "id": {
                       "description": "Unique identifier for this pipeline."
                     },
@@ -5223,6 +5288,23 @@
                                   },
                                   "source_schema": {
                                     "description": "Required. Schema name in the source database."
+                                  },
+                                  "table_configuration": {
+                                    "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.",
+                                    "properties": {
+                                      "primary_keys": {
+                                        "description": "The primary key of the table used to apply changes.",
+                                        "items": {
+                                          "description": ""
+                                        }
+                                      },
+                                      "salesforce_include_formula_fields": {
+                                        "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                                      },
+                                      "scd_type": {
+                                        "description": "The SCD type to use to ingest the table."
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -5246,11 +5328,45 @@
                                   },
                                   "source_table": {
                                     "description": "Required. Table name in the source database."
+                                  },
+                                  "table_configuration": {
+                                    "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.",
+                                    "properties": {
+                                      "primary_keys": {
+                                        "description": "The primary key of the table used to apply changes.",
+                                        "items": {
+                                          "description": ""
+                                        }
+                                      },
+                                      "salesforce_include_formula_fields": {
+                                        "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                                      },
+                                      "scd_type": {
+                                        "description": "The SCD type to use to ingest the table."
+                                      }
+                                    }
                                   }
                                 }
                               }
                             }
                           }
+                        },
+                        "table_configuration": {
+                          "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.",
+                          "properties": {
+                            "primary_keys": {
+                              "description": "The primary key of the table used to apply changes.",
+                              "items": {
+                                "description": ""
+                              }
+                            },
+                            "salesforce_include_formula_fields": {
+                              "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
+                            },
+                            "scd_type": {
+                              "description": "The SCD type to use to ingest the table."
+                            }
+                          }
                         }
                       }
                     },
diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go
index 49c21eb48..71149e5ad 100755
--- a/cmd/account/esm-enablement-account/esm-enablement-account.go
+++ b/cmd/account/esm-enablement-account/esm-enablement-account.go
@@ -25,9 +25,6 @@ func New() *cobra.Command {
   setting is disabled for new workspaces. After workspace creation, account
   admins can enable enhanced security monitoring individually for each
   workspace.`,
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
index 681dba7b3..2385195bb 100755
--- a/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
+++ b/cmd/workspace/automatic-cluster-update/automatic-cluster-update.go
@@ -22,9 +22,6 @@ func New() *cobra.Command {
 		Short: `Controls whether automatic cluster update is enabled for the current workspace.`,
 		Long: `Controls whether automatic cluster update is enabled for the current
   workspace. By default, it is turned off.`,
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go
index e657fd9c3..f4baab3b2 100755
--- a/cmd/workspace/clusters/clusters.go
+++ b/cmd/workspace/clusters/clusters.go
@@ -188,7 +188,7 @@ func newCreate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
-	cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, ``)
+	cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, `When set to true, fixed and default values from the policy will be used for fields that are omitted.`)
 	// TODO: complex arg: autoscale
 	cmd.Flags().IntVar(&createReq.AutoterminationMinutes, "autotermination-minutes", createReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`)
 	// TODO: complex arg: aws_attributes
@@ -196,15 +196,6 @@ func newCreate() *cobra.Command {
 	// TODO: complex arg: clone_from
 	// TODO: complex arg: cluster_log_conf
 	cmd.Flags().StringVar(&createReq.ClusterName, "cluster-name", createReq.ClusterName, `Cluster name requested by the user.`)
-	cmd.Flags().Var(&createReq.ClusterSource, "cluster-source", `Determines whether the cluster was created by a user through the UI, created by the Databricks Jobs Scheduler, or through an API request. Supported values: [
-  API,
-  JOB,
-  MODELS,
-  PIPELINE,
-  PIPELINE_MAINTENANCE,
-  SQL,
-  UI,
-]`)
 	// TODO: map via StringToStringVar: custom_tags
 	cmd.Flags().Var(&createReq.DataSecurityMode, "data-security-mode", `Data security mode decides what data governance model to use when accessing data from a cluster. Supported values: [
   LEGACY_PASSTHROUGH,
@@ -443,23 +434,13 @@ func newEdit() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&editJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
-	cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, ``)
+	cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, `When set to true, fixed and default values from the policy will be used for fields that are omitted.`)
 	// TODO: complex arg: autoscale
 	cmd.Flags().IntVar(&editReq.AutoterminationMinutes, "autotermination-minutes", editReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`)
 	// TODO: complex arg: aws_attributes
 	// TODO: complex arg: azure_attributes
-	// TODO: complex arg: clone_from
 	// TODO: complex arg: cluster_log_conf
 	cmd.Flags().StringVar(&editReq.ClusterName, "cluster-name", editReq.ClusterName, `Cluster name requested by the user.`)
-	cmd.Flags().Var(&editReq.ClusterSource, "cluster-source", `Determines whether the cluster was created by a user through the UI, created by the Databricks Jobs Scheduler, or through an API request. Supported values: [
-  API,
-  JOB,
-  MODELS,
-  PIPELINE,
-  PIPELINE_MAINTENANCE,
-  SQL,
-  UI,
-]`)
 	// TODO: map via StringToStringVar: custom_tags
 	cmd.Flags().Var(&editReq.DataSecurityMode, "data-security-mode", `Data security mode decides what data governance model to use when accessing data from a cluster. Supported values: [
   LEGACY_PASSTHROUGH,
diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go
index a78b9bc1e..7ad9389a8 100755
--- a/cmd/workspace/cmd.go
+++ b/cmd/workspace/cmd.go
@@ -32,7 +32,6 @@ import (
 	instance_profiles "github.com/databricks/cli/cmd/workspace/instance-profiles"
 	ip_access_lists "github.com/databricks/cli/cmd/workspace/ip-access-lists"
 	jobs "github.com/databricks/cli/cmd/workspace/jobs"
-	lakehouse_monitors "github.com/databricks/cli/cmd/workspace/lakehouse-monitors"
 	lakeview "github.com/databricks/cli/cmd/workspace/lakeview"
 	libraries "github.com/databricks/cli/cmd/workspace/libraries"
 	metastores "github.com/databricks/cli/cmd/workspace/metastores"
@@ -51,6 +50,7 @@ import (
 	provider_provider_analytics_dashboards "github.com/databricks/cli/cmd/workspace/provider-provider-analytics-dashboards"
 	provider_providers "github.com/databricks/cli/cmd/workspace/provider-providers"
 	providers "github.com/databricks/cli/cmd/workspace/providers"
+	quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors"
 	queries "github.com/databricks/cli/cmd/workspace/queries"
 	query_history "github.com/databricks/cli/cmd/workspace/query-history"
 	query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations"
@@ -113,7 +113,6 @@ func All() []*cobra.Command {
 	out = append(out, instance_profiles.New())
 	out = append(out, ip_access_lists.New())
 	out = append(out, jobs.New())
-	out = append(out, lakehouse_monitors.New())
 	out = append(out, lakeview.New())
 	out = append(out, libraries.New())
 	out = append(out, metastores.New())
@@ -132,6 +131,7 @@ func All() []*cobra.Command {
 	out = append(out, provider_provider_analytics_dashboards.New())
 	out = append(out, provider_providers.New())
 	out = append(out, providers.New())
+	out = append(out, quality_monitors.New())
 	out = append(out, queries.New())
 	out = append(out, query_history.New())
 	out = append(out, query_visualizations.New())
diff --git a/cmd/workspace/compliance-security-profile/compliance-security-profile.go b/cmd/workspace/compliance-security-profile/compliance-security-profile.go
index efafb4627..a7b45901f 100755
--- a/cmd/workspace/compliance-security-profile/compliance-security-profile.go
+++ b/cmd/workspace/compliance-security-profile/compliance-security-profile.go
@@ -25,9 +25,6 @@ func New() *cobra.Command {
   off.
   
   This settings can NOT be disabled once it is enabled.`,
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go
index bdb266685..f76420fbe 100755
--- a/cmd/workspace/connections/connections.go
+++ b/cmd/workspace/connections/connections.go
@@ -154,7 +154,7 @@ func newDelete() *cobra.Command {
 		if len(args) == 0 {
 			promptSpinner := cmdio.Spinner(ctx)
 			promptSpinner <- "No NAME argument specified. Loading names for Connections drop-down."
-			names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx)
+			names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx, catalog.ListConnectionsRequest{})
 			close(promptSpinner)
 			if err != nil {
 				return fmt.Errorf("failed to load names for Connections drop-down. Please manually specify required arguments. Original error: %w", err)
@@ -224,7 +224,7 @@ func newGet() *cobra.Command {
 		if len(args) == 0 {
 			promptSpinner := cmdio.Spinner(ctx)
 			promptSpinner <- "No NAME argument specified. Loading names for Connections drop-down."
-			names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx)
+			names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx, catalog.ListConnectionsRequest{})
 			close(promptSpinner)
 			if err != nil {
 				return fmt.Errorf("failed to load names for Connections drop-down. Please manually specify required arguments. Original error: %w", err)
@@ -265,11 +265,19 @@ func newGet() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var listOverrides []func(
 	*cobra.Command,
+	*catalog.ListConnectionsRequest,
 )
 
 func newList() *cobra.Command {
 	cmd := &cobra.Command{}
 
+	var listReq catalog.ListConnectionsRequest
+
+	// TODO: short flags
+
+	cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of connections to return.`)
+	cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`)
+
 	cmd.Use = "list"
 	cmd.Short = `List connections.`
 	cmd.Long = `List connections.
@@ -278,11 +286,17 @@ func newList() *cobra.Command {
 
 	cmd.Annotations = make(map[string]string)
 
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(0)
+		return check(cmd, args)
+	}
+
 	cmd.PreRunE = root.MustWorkspaceClient
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
-		response := w.Connections.List(ctx)
+
+		response := w.Connections.List(ctx, listReq)
 		return cmdio.RenderIterator(ctx, response)
 	}
 
@@ -292,7 +306,7 @@ func newList() *cobra.Command {
 
 	// Apply optional overrides to this command.
 	for _, fn := range listOverrides {
-		fn(cmd)
+		fn(cmd, &listReq)
 	}
 
 	return cmd
diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go
index f75f03b3a..8669dfae5 100755
--- a/cmd/workspace/consumer-listings/consumer-listings.go
+++ b/cmd/workspace/consumer-listings/consumer-listings.go
@@ -129,13 +129,14 @@ func newList() *cobra.Command {
 
 	// TODO: array: assets
 	// TODO: array: categories
+	cmd.Flags().BoolVar(&listReq.IsAscending, "is-ascending", listReq.IsAscending, ``)
 	cmd.Flags().BoolVar(&listReq.IsFree, "is-free", listReq.IsFree, `Filters each listing based on if it is free.`)
 	cmd.Flags().BoolVar(&listReq.IsPrivateExchange, "is-private-exchange", listReq.IsPrivateExchange, `Filters each listing based on if it is a private exchange.`)
 	cmd.Flags().BoolVar(&listReq.IsStaffPick, "is-staff-pick", listReq.IsStaffPick, `Filters each listing based on whether it is a staff pick.`)
 	cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``)
 	cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``)
 	// TODO: array: provider_ids
-	// TODO: complex arg: sort_by_spec
+	cmd.Flags().Var(&listReq.SortBy, "sort-by", `Criteria for sorting the resulting set of listings. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`)
 	// TODO: array: tags
 
 	cmd.Use = "list"
@@ -191,6 +192,7 @@ func newSearch() *cobra.Command {
 
 	// TODO: array: assets
 	// TODO: array: categories
+	cmd.Flags().BoolVar(&searchReq.IsAscending, "is-ascending", searchReq.IsAscending, ``)
 	cmd.Flags().BoolVar(&searchReq.IsFree, "is-free", searchReq.IsFree, ``)
 	cmd.Flags().BoolVar(&searchReq.IsPrivateExchange, "is-private-exchange", searchReq.IsPrivateExchange, ``)
 	cmd.Flags().IntVar(&searchReq.PageSize, "page-size", searchReq.PageSize, ``)
diff --git a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
index 86b4244d5..a8acc5cd1 100755
--- a/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
+++ b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go
@@ -27,9 +27,6 @@ func New() *cobra.Command {
   
   If the compliance security profile is disabled, you can enable or disable this
   setting and it is not permanent.`,
-
-		// This service is being previewed; hide from help output.
-		Hidden: true,
 	}
 
 	// Add methods
diff --git a/cmd/workspace/libraries/libraries.go b/cmd/workspace/libraries/libraries.go
index aed8843dc..2c10d8161 100755
--- a/cmd/workspace/libraries/libraries.go
+++ b/cmd/workspace/libraries/libraries.go
@@ -80,11 +80,8 @@ func newAllClusterStatuses() *cobra.Command {
 	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()
 		w := root.WorkspaceClient(ctx)
-		response, err := w.Libraries.AllClusterStatuses(ctx)
-		if err != nil {
-			return err
-		}
-		return cmdio.Render(ctx, response)
+		response := w.Libraries.AllClusterStatuses(ctx)
+		return cmdio.RenderIterator(ctx, response)
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go
index 5a55fd72b..f1cc4e3f7 100755
--- a/cmd/workspace/pipelines/pipelines.go
+++ b/cmd/workspace/pipelines/pipelines.go
@@ -945,6 +945,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().StringVar(&updateReq.Edition, "edition", updateReq.Edition, `Pipeline product edition.`)
 	cmd.Flags().Int64Var(&updateReq.ExpectedLastModified, "expected-last-modified", updateReq.ExpectedLastModified, `If present, the last-modified time of the pipeline settings before the edit.`)
 	// TODO: complex arg: filters
+	// TODO: complex arg: gateway_definition
 	cmd.Flags().StringVar(&updateReq.Id, "id", updateReq.Id, `Unique identifier for this pipeline.`)
 	// TODO: complex arg: ingestion_definition
 	// TODO: array: libraries
diff --git a/cmd/workspace/lakehouse-monitors/lakehouse-monitors.go b/cmd/workspace/quality-monitors/quality-monitors.go
similarity index 95%
rename from cmd/workspace/lakehouse-monitors/lakehouse-monitors.go
rename to cmd/workspace/quality-monitors/quality-monitors.go
index 465ed6f92..95d992164 100755
--- a/cmd/workspace/lakehouse-monitors/lakehouse-monitors.go
+++ b/cmd/workspace/quality-monitors/quality-monitors.go
@@ -1,6 +1,6 @@
 // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT.
 
-package lakehouse_monitors
+package quality_monitors
 
 import (
 	"fmt"
@@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command)
 
 func New() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "lakehouse-monitors",
+		Use:   "quality-monitors",
 		Short: `A monitor computes and monitors data or model quality metrics for a table over time.`,
 		Long: `A monitor computes and monitors data or model quality metrics for a table over
   time. It generates metrics tables and a dashboard that you can use to monitor
@@ -105,7 +105,7 @@ func newCancelRefresh() *cobra.Command {
 		cancelRefreshReq.TableName = args[0]
 		cancelRefreshReq.RefreshId = args[1]
 
-		err = w.LakehouseMonitors.CancelRefresh(ctx, cancelRefreshReq)
+		err = w.QualityMonitors.CancelRefresh(ctx, cancelRefreshReq)
 		if err != nil {
 			return err
 		}
@@ -208,7 +208,7 @@ func newCreate() *cobra.Command {
 			createReq.OutputSchemaName = args[2]
 		}
 
-		response, err := w.LakehouseMonitors.Create(ctx, createReq)
+		response, err := w.QualityMonitors.Create(ctx, createReq)
 		if err != nil {
 			return err
 		}
@@ -233,13 +233,13 @@ func newCreate() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var deleteOverrides []func(
 	*cobra.Command,
-	*catalog.DeleteLakehouseMonitorRequest,
+	*catalog.DeleteQualityMonitorRequest,
 )
 
 func newDelete() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var deleteReq catalog.DeleteLakehouseMonitorRequest
+	var deleteReq catalog.DeleteQualityMonitorRequest
 
 	// TODO: short flags
 
@@ -278,7 +278,7 @@ func newDelete() *cobra.Command {
 
 		deleteReq.TableName = args[0]
 
-		err = w.LakehouseMonitors.Delete(ctx, deleteReq)
+		err = w.QualityMonitors.Delete(ctx, deleteReq)
 		if err != nil {
 			return err
 		}
@@ -303,13 +303,13 @@ func newDelete() *cobra.Command {
 // Functions can be added from the `init()` function in manually curated files in this directory.
 var getOverrides []func(
 	*cobra.Command,
-	*catalog.GetLakehouseMonitorRequest,
+	*catalog.GetQualityMonitorRequest,
 )
 
 func newGet() *cobra.Command {
 	cmd := &cobra.Command{}
 
-	var getReq catalog.GetLakehouseMonitorRequest
+	var getReq catalog.GetQualityMonitorRequest
 
 	// TODO: short flags
 
@@ -347,7 +347,7 @@ func newGet() *cobra.Command {
 
 		getReq.TableName = args[0]
 
-		response, err := w.LakehouseMonitors.Get(ctx, getReq)
+		response, err := w.QualityMonitors.Get(ctx, getReq)
 		if err != nil {
 			return err
 		}
@@ -416,7 +416,7 @@ func newGetRefresh() *cobra.Command {
 		getRefreshReq.TableName = args[0]
 		getRefreshReq.RefreshId = args[1]
 
-		response, err := w.LakehouseMonitors.GetRefresh(ctx, getRefreshReq)
+		response, err := w.QualityMonitors.GetRefresh(ctx, getRefreshReq)
 		if err != nil {
 			return err
 		}
@@ -484,7 +484,7 @@ func newListRefreshes() *cobra.Command {
 
 		listRefreshesReq.TableName = args[0]
 
-		response, err := w.LakehouseMonitors.ListRefreshes(ctx, listRefreshesReq)
+		response, err := w.QualityMonitors.ListRefreshes(ctx, listRefreshesReq)
 		if err != nil {
 			return err
 		}
@@ -552,7 +552,7 @@ func newRunRefresh() *cobra.Command {
 
 		runRefreshReq.TableName = args[0]
 
-		response, err := w.LakehouseMonitors.RunRefresh(ctx, runRefreshReq)
+		response, err := w.QualityMonitors.RunRefresh(ctx, runRefreshReq)
 		if err != nil {
 			return err
 		}
@@ -591,6 +591,7 @@ func newUpdate() *cobra.Command {
 
 	cmd.Flags().StringVar(&updateReq.BaselineTableName, "baseline-table-name", updateReq.BaselineTableName, `Name of the baseline table from which drift metrics are computed from.`)
 	// TODO: array: custom_metrics
+	cmd.Flags().StringVar(&updateReq.DashboardId, "dashboard-id", updateReq.DashboardId, `Id of dashboard that visualizes the computed metrics.`)
 	// TODO: complex arg: data_classification_config
 	// TODO: complex arg: inference_log
 	// TODO: complex arg: notifications
@@ -651,7 +652,7 @@ func newUpdate() *cobra.Command {
 			updateReq.OutputSchemaName = args[1]
 		}
 
-		response, err := w.LakehouseMonitors.Update(ctx, updateReq)
+		response, err := w.QualityMonitors.Update(ctx, updateReq)
 		if err != nil {
 			return err
 		}
@@ -670,4 +671,4 @@ func newUpdate() *cobra.Command {
 	return cmd
 }
 
-// end service LakehouseMonitors
+// end service QualityMonitors
diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go
index dee341ab4..b92f824d3 100755
--- a/cmd/workspace/serving-endpoints/serving-endpoints.go
+++ b/cmd/workspace/serving-endpoints/serving-endpoints.go
@@ -152,6 +152,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	// TODO: array: rate_limits
+	cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`)
 	// TODO: array: tags
 
 	cmd.Use = "create"
@@ -303,11 +304,12 @@ func newExportMetrics() *cobra.Command {
 
 		exportMetricsReq.Name = args[0]
 
-		err = w.ServingEndpoints.ExportMetrics(ctx, exportMetricsReq)
+		response, err := w.ServingEndpoints.ExportMetrics(ctx, exportMetricsReq)
 		if err != nil {
 			return err
 		}
-		return nil
+		defer response.Contents.Close()
+		return cmdio.Render(ctx, response.Contents)
 	}
 
 	// Disable completions since they are not applicable.
diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go
index 0e3523cec..c2fd779a7 100755
--- a/cmd/workspace/shares/shares.go
+++ b/cmd/workspace/shares/shares.go
@@ -67,6 +67,7 @@ func newCreate() *cobra.Command {
 	cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
 	cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `User-provided free-form text description.`)
+	cmd.Flags().StringVar(&createReq.StorageRoot, "storage-root", createReq.StorageRoot, `Storage root URL for the share.`)
 
 	cmd.Use = "create NAME"
 	cmd.Short = `Create a share.`
@@ -368,6 +369,7 @@ func newUpdate() *cobra.Command {
 	cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `User-provided free-form text description.`)
 	cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the share.`)
 	cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of share.`)
+	cmd.Flags().StringVar(&updateReq.StorageRoot, "storage-root", updateReq.StorageRoot, `Storage root URL for the share.`)
 	// TODO: array: updates
 
 	cmd.Use = "update NAME"
@@ -382,6 +384,9 @@ func newUpdate() *cobra.Command {
   In the case that the share name is changed, **updateShare** requires that the
   caller is both the share owner and a metastore admin.
   
+  If there are notebook files in the share, the __storage_root__ field cannot be
+  updated.
+  
   For each table that is added through this method, the share owner must also
   have **SELECT** privilege on the table. This privilege must be maintained
   indefinitely for recipients to be able to access the table. Typically, you
diff --git a/cmd/workspace/system-schemas/system-schemas.go b/cmd/workspace/system-schemas/system-schemas.go
index 070701d2f..3fe0580d7 100755
--- a/cmd/workspace/system-schemas/system-schemas.go
+++ b/cmd/workspace/system-schemas/system-schemas.go
@@ -3,8 +3,6 @@
 package system_schemas
 
 import (
-	"fmt"
-
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
@@ -81,10 +79,7 @@ func newDisable() *cobra.Command {
 		w := root.WorkspaceClient(ctx)
 
 		disableReq.MetastoreId = args[0]
-		_, err = fmt.Sscan(args[1], &disableReq.SchemaName)
-		if err != nil {
-			return fmt.Errorf("invalid SCHEMA_NAME: %s", args[1])
-		}
+		disableReq.SchemaName = args[1]
 
 		err = w.SystemSchemas.Disable(ctx, disableReq)
 		if err != nil {
@@ -145,10 +140,7 @@ func newEnable() *cobra.Command {
 		w := root.WorkspaceClient(ctx)
 
 		enableReq.MetastoreId = args[0]
-		_, err = fmt.Sscan(args[1], &enableReq.SchemaName)
-		if err != nil {
-			return fmt.Errorf("invalid SCHEMA_NAME: %s", args[1])
-		}
+		enableReq.SchemaName = args[1]
 
 		err = w.SystemSchemas.Enable(ctx, enableReq)
 		if err != nil {
diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go
index 32e023d44..dff8176ea 100755
--- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go
+++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go
@@ -42,6 +42,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(newGetIndex())
 	cmd.AddCommand(newListIndexes())
 	cmd.AddCommand(newQueryIndex())
+	cmd.AddCommand(newScanIndex())
 	cmd.AddCommand(newSyncIndex())
 	cmd.AddCommand(newUpsertDataVectorIndex())
 
@@ -468,6 +469,76 @@ func newQueryIndex() *cobra.Command {
 	return cmd
 }
 
+// start scan-index command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var scanIndexOverrides []func(
+	*cobra.Command,
+	*vectorsearch.ScanVectorIndexRequest,
+)
+
+func newScanIndex() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var scanIndexReq vectorsearch.ScanVectorIndexRequest
+	var scanIndexJson flags.JsonFlag
+
+	// TODO: short flags
+	cmd.Flags().Var(&scanIndexJson, "json", `either inline JSON string or @path/to/file.json with request body`)
+
+	cmd.Flags().StringVar(&scanIndexReq.LastPrimaryKey, "last-primary-key", scanIndexReq.LastPrimaryKey, `Primary key of the last entry returned in the previous scan.`)
+	cmd.Flags().IntVar(&scanIndexReq.NumResults, "num-results", scanIndexReq.NumResults, `Number of results to return.`)
+
+	cmd.Use = "scan-index INDEX_NAME"
+	cmd.Short = `Scan an index.`
+	cmd.Long = `Scan an index.
+  
+  Scan the specified vector index and return the first num_results entries
+  after the exclusive primary_key.
+
+  Arguments:
+    INDEX_NAME: Name of the vector index to scan.`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(1)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		if cmd.Flags().Changed("json") {
+			err = scanIndexJson.Unmarshal(&scanIndexReq)
+			if err != nil {
+				return err
+			}
+		}
+		scanIndexReq.IndexName = args[0]
+
+		response, err := w.VectorSearchIndexes.ScanIndex(ctx, scanIndexReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range scanIndexOverrides {
+		fn(cmd, &scanIndexReq)
+	}
+
+	return cmd
+}
+
 // start sync-index command
 
 // Slice with functions to override default command behavior.
diff --git a/go.mod b/go.mod
index ddebe9727..84933de2b 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/semver/v3 v3.2.1 // MIT
 	github.com/briandowns/spinner v1.23.0 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.40.1 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.41.0 // Apache 2.0
 	github.com/fatih/color v1.17.0 // MIT
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
diff --git a/go.sum b/go.sum
index 1dccbb2f9..16f9c9a1f 100644
--- a/go.sum
+++ b/go.sum
@@ -28,8 +28,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.40.1 h1:rE5yP9gIW2oap+6CnumixnZSDIsXwVojAuDBuKUl5GU=
-github.com/databricks/databricks-sdk-go v0.40.1/go.mod h1:rLIhh7DvifVLmf2QxMr/vMRGqdrTZazn8VYo4LilfCo=
+github.com/databricks/databricks-sdk-go v0.41.0 h1:OyhYY+Q6+gqkWeXmpGEiacoU2RStTeWPF0x4vmqbQdc=
+github.com/databricks/databricks-sdk-go v0.41.0/go.mod h1:rLIhh7DvifVLmf2QxMr/vMRGqdrTZazn8VYo4LilfCo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

From 46f6cbcfc37d6dac836b027f5d60d824d7fdd16e Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 22 May 2024 11:08:27 +0200
Subject: [PATCH 34/41] Release v0.220.0 (#1446)

CLI:
* Add line about Docker installation to README.md
([#1363](https://github.com/databricks/cli/pull/1363)).
* Improve token refresh flow
([#1434](https://github.com/databricks/cli/pull/1434)).

Bundles:
* Upgrade Terraform provider to v1.42.0
([#1418](https://github.com/databricks/cli/pull/1418)).
* Upgrade Terraform provider to v1.43.0
([#1429](https://github.com/databricks/cli/pull/1429)).
* Don't merge-in remote resources during deployments
([#1432](https://github.com/databricks/cli/pull/1432)).
* Remove dependency on `ConfigFilePath` from path translation mutator
([#1437](https://github.com/databricks/cli/pull/1437)).
* Add `merge.Override` transform
([#1428](https://github.com/databricks/cli/pull/1428)).
* Fixed panic when loading incorrectly defined jobs
([#1402](https://github.com/databricks/cli/pull/1402)).
* Add more tests for `merge.Override`
([#1439](https://github.com/databricks/cli/pull/1439)).
* Fixed seg fault when specifying environment key for tasks
([#1443](https://github.com/databricks/cli/pull/1443)).
* Fix conversion of zero valued scalar pointers to a dynamic value
([#1433](https://github.com/databricks/cli/pull/1433)).

Internal:
* Don't hide commands of services that are already hidden
([#1438](https://github.com/databricks/cli/pull/1438)).

API Changes:
 * Renamed `lakehouse-monitors` command group to `quality-monitors`.
 * Added `apps` command group.
* Renamed `csp-enablement` command group to
`compliance-security-profile`.
* Renamed `esm-enablement` command group to
`enhanced-security-monitoring`.
 * Added `databricks vector-search-indexes scan-index` command.

OpenAPI commit 7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92 (2024-05-21)

Dependency updates:
* Bump golang.org/x/text from 0.14.0 to 0.15.0
([#1419](https://github.com/databricks/cli/pull/1419)).
* Bump golang.org/x/oauth2 from 0.19.0 to 0.20.0
([#1421](https://github.com/databricks/cli/pull/1421)).
* Bump golang.org/x/term from 0.19.0 to 0.20.0
([#1422](https://github.com/databricks/cli/pull/1422)).
* Bump github.com/databricks/databricks-sdk-go from 0.39.0 to 0.40.1
([#1431](https://github.com/databricks/cli/pull/1431)).
* Bump github.com/fatih/color from 1.16.0 to 1.17.0
([#1441](https://github.com/databricks/cli/pull/1441)).
* Bump github.com/hashicorp/terraform-json from 0.21.0 to 0.22.1
([#1440](https://github.com/databricks/cli/pull/1440)).
* Bump github.com/hashicorp/terraform-exec from 0.20.0 to 0.21.0
([#1442](https://github.com/databricks/cli/pull/1442)).
* Update Go SDK to v0.41.0
([#1445](https://github.com/databricks/cli/pull/1445)).
---
 CHANGELOG.md | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1bd824daf..2fb35d479 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,44 @@
 # Version changelog
 
+## 0.220.0
+
+CLI:
+ * Add line about Docker installation to README.md ([#1363](https://github.com/databricks/cli/pull/1363)).
+ * Improve token refresh flow ([#1434](https://github.com/databricks/cli/pull/1434)).
+
+Bundles:
+ * Upgrade Terraform provider to v1.42.0 ([#1418](https://github.com/databricks/cli/pull/1418)).
+ * Upgrade Terraform provider to v1.43.0 ([#1429](https://github.com/databricks/cli/pull/1429)).
+ * Don't merge-in remote resources during deployments ([#1432](https://github.com/databricks/cli/pull/1432)).
+ * Remove dependency on `ConfigFilePath` from path translation mutator ([#1437](https://github.com/databricks/cli/pull/1437)).
+ * Add `merge.Override` transform ([#1428](https://github.com/databricks/cli/pull/1428)).
+ * Fixed panic when loading incorrectly defined jobs ([#1402](https://github.com/databricks/cli/pull/1402)).
+ * Add more tests for `merge.Override` ([#1439](https://github.com/databricks/cli/pull/1439)).
+ * Fixed seg fault when specifying environment key for tasks ([#1443](https://github.com/databricks/cli/pull/1443)).
+ * Fix conversion of zero valued scalar pointers to a dynamic value ([#1433](https://github.com/databricks/cli/pull/1433)).
+
+Internal:
+ * Don't hide commands of services that are already hidden ([#1438](https://github.com/databricks/cli/pull/1438)).
+
+API Changes:
+ * Renamed `lakehouse-monitors` command group to `quality-monitors`.
+ * Added `apps` command group.
+ * Renamed `csp-enablement` command group to `compliance-security-profile`.
+ * Renamed `esm-enablement` command group to `enhanced-security-monitoring`.
+ * Added `databricks vector-search-indexes scan-index` command.
+
+OpenAPI commit 7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92 (2024-05-21)
+
+Dependency updates:
+ * Bump golang.org/x/text from 0.14.0 to 0.15.0 ([#1419](https://github.com/databricks/cli/pull/1419)).
+ * Bump golang.org/x/oauth2 from 0.19.0 to 0.20.0 ([#1421](https://github.com/databricks/cli/pull/1421)).
+ * Bump golang.org/x/term from 0.19.0 to 0.20.0 ([#1422](https://github.com/databricks/cli/pull/1422)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.39.0 to 0.40.1 ([#1431](https://github.com/databricks/cli/pull/1431)).
+ * Bump github.com/fatih/color from 1.16.0 to 1.17.0 ([#1441](https://github.com/databricks/cli/pull/1441)).
+ * Bump github.com/hashicorp/terraform-json from 0.21.0 to 0.22.1 ([#1440](https://github.com/databricks/cli/pull/1440)).
+ * Bump github.com/hashicorp/terraform-exec from 0.20.0 to 0.21.0 ([#1442](https://github.com/databricks/cli/pull/1442)).
+ * Update Go SDK to v0.41.0 ([#1445](https://github.com/databricks/cli/pull/1445)).
+
 ## 0.219.0
 
 Bundles:

From 9a452f38ee4654e0f5d96905f9b4dae40e547a96 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 May 2024 14:20:11 +0200
Subject: [PATCH 35/41] Bump github.com/hashicorp/go-version from 1.6.0 to
 1.7.0 (#1454)

Bumps
[github.com/hashicorp/go-version](https://github.com/hashicorp/go-version)
from 1.6.0 to 1.7.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/go-version/releases">github.com/hashicorp/go-version's
releases</a>.</em></p>
<blockquote>
<h2>v1.7.0</h2>
<p>ENHANCEMENTS:</p>
<ul>
<li>Remove <code>reflect</code> dependency (<a
href="https://redirect.github.com/hashicorp/go-version/pull/91">#91</a>)</li>
<li>Implement the <code>database/sql.Scanner</code> and
<code>database/sql/driver.Value</code> interfaces for
<code>Version</code> (<a
href="https://redirect.github.com/hashicorp/go-version/pull/133">#133</a>)</li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>[COMPLIANCE] Add Copyright and License Headers (<a
href="https://redirect.github.com/hashicorp/go-version/pull/115">#115</a>)</li>
<li>[COMPLIANCE] Update MPL-2.0 LICENSE (<a
href="https://redirect.github.com/hashicorp/go-version/pull/105">#105</a>)</li>
<li>Bump actions/cache from 3.0.11 to 3.2.5 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/116">#116</a>)</li>
<li>Bump actions/checkout from 3.2.0 to 3.3.0 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/111">#111</a>)</li>
<li>Bump actions/upload-artifact from 3.1.1 to 3.1.2 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/112">#112</a>)</li>
<li>GHA Migration (<a
href="https://redirect.github.com/hashicorp/go-version/pull/103">#103</a>)</li>
<li>github: Pin external GitHub Actions to hashes (<a
href="https://redirect.github.com/hashicorp/go-version/pull/107">#107</a>)</li>
<li>SEC-090: Automated trusted workflow pinning (2023-04-05) (<a
href="https://redirect.github.com/hashicorp/go-version/pull/124">#124</a>)</li>
<li>update readme (<a
href="https://redirect.github.com/hashicorp/go-version/pull/104">#104</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Changelog</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/go-version/blob/main/CHANGELOG.md">github.com/hashicorp/go-version's
changelog</a>.</em></p>
<blockquote>
<h1>1.7.0 (May 24, 2024)</h1>
<p>ENHANCEMENTS:</p>
<ul>
<li>Remove <code>reflect</code> dependency (<a
href="https://redirect.github.com/hashicorp/go-version/pull/91">#91</a>)</li>
<li>Implement the <code>database/sql.Scanner</code> and
<code>database/sql/driver.Value</code> interfaces for
<code>Version</code> (<a
href="https://redirect.github.com/hashicorp/go-version/pull/133">#133</a>)</li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>[COMPLIANCE] Add Copyright and License Headers (<a
href="https://redirect.github.com/hashicorp/go-version/pull/115">#115</a>)</li>
<li>[COMPLIANCE] Update MPL-2.0 LICENSE (<a
href="https://redirect.github.com/hashicorp/go-version/pull/105">#105</a>)</li>
<li>Bump actions/cache from 3.0.11 to 3.2.5 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/116">#116</a>)</li>
<li>Bump actions/checkout from 3.2.0 to 3.3.0 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/111">#111</a>)</li>
<li>Bump actions/upload-artifact from 3.1.1 to 3.1.2 (<a
href="https://redirect.github.com/hashicorp/go-version/pull/112">#112</a>)</li>
<li>GHA Migration (<a
href="https://redirect.github.com/hashicorp/go-version/pull/103">#103</a>)</li>
<li>github: Pin external GitHub Actions to hashes (<a
href="https://redirect.github.com/hashicorp/go-version/pull/107">#107</a>)</li>
<li>SEC-090: Automated trusted workflow pinning (2023-04-05) (<a
href="https://redirect.github.com/hashicorp/go-version/pull/124">#124</a>)</li>
<li>update readme (<a
href="https://redirect.github.com/hashicorp/go-version/pull/104">#104</a>)</li>
</ul>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/go-version/commit/fcaa53267960208632e9308c93aa89535526a018"><code>fcaa532</code></a>
Update CHANGELOG.md</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/b85381af953b8b2b330eca5f06046cd4fcb69d56"><code>b85381a</code></a>
Update CHANGELOG.md</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/d55f21493f82b092e4869cfeb94b4602252d811e"><code>d55f214</code></a>
Implement the Scan and driver.Value SQL interfaces (<a
href="https://redirect.github.com/hashicorp/go-version/issues/133">#133</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/e04a8660deda04c35f8cbd29be7084224af3c78e"><code>e04a866</code></a>
remove reflection dependency (<a
href="https://redirect.github.com/hashicorp/go-version/issues/91">#91</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/94bab9eaa8230b5fae65c0671702bb550e4090c1"><code>94bab9e</code></a>
[COMPLIANCE] Add Copyright and License Headers (<a
href="https://redirect.github.com/hashicorp/go-version/issues/115">#115</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/73ddc6313d604f960faa3402c33675016c5c315e"><code>73ddc63</code></a>
github: Change Dependabot to only manage HashiCorp-owned Actions</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/bf1144e59f2a1e2418cce1f185ec3fe94011c5b7"><code>bf1144e</code></a>
SEC-090: Automated trusted workflow pinning (2023-04-05) (<a
href="https://redirect.github.com/hashicorp/go-version/issues/124">#124</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/644291d14038339745c2d883a1a114488e30b702"><code>644291d</code></a>
Bump actions/cache from 3.0.11 to 3.2.5 (<a
href="https://redirect.github.com/hashicorp/go-version/issues/116">#116</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/8f6487b06ed6b0260910943fc143c765291cdc06"><code>8f6487b</code></a>
Bump actions/upload-artifact from 3.1.1 to 3.1.2 (<a
href="https://redirect.github.com/hashicorp/go-version/issues/112">#112</a>)</li>
<li><a
href="https://github.com/hashicorp/go-version/commit/7f856b8ccc0579097f4b3a018a03f3e5e30135df"><code>7f856b8</code></a>
Bump actions/checkout from 3.2.0 to 3.3.0 (<a
href="https://redirect.github.com/hashicorp/go-version/issues/111">#111</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/hashicorp/go-version/compare/v1.6.0...v1.7.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/go-version&package-manager=go_modules&previous-version=1.6.0&new-version=1.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 84933de2b..8ccbeffc0 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/fatih/color v1.17.0 // MIT
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
-	github.com/hashicorp/go-version v1.6.0 // MPL 2.0
+	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
 	github.com/hashicorp/hc-install v0.6.4 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
 	github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
diff --git a/go.sum b/go.sum
index 16f9c9a1f..71bd69bd6 100644
--- a/go.sum
+++ b/go.sum
@@ -97,8 +97,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh
 github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
-github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9Km8e0=
 github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA=
 github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=

From 13b937cea818b536ea5b740e4fff410bfe8d769d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 May 2024 14:41:20 +0200
Subject: [PATCH 36/41] Bump github.com/hashicorp/hc-install from 0.6.4 to
 0.7.0 (#1453)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bumps
[github.com/hashicorp/hc-install](https://github.com/hashicorp/hc-install)
from 0.6.4 to 0.7.0.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/hashicorp/hc-install/releases">github.com/hashicorp/hc-install's
releases</a>.</em></p>
<blockquote>
<h2>v0.7.0</h2>
<p>ENHANCEMENTS:</p>
<ul>
<li>cmd/hc-install: Enable logging into a file by <a
href="https://github.com/radeksimko"><code>@​radeksimko</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/193">hashicorp/hc-install#193</a></li>
<li>Ensure license file gets packaged along w/ the CLI binary by <a
href="https://github.com/radeksimko"><code>@​radeksimko</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/205">hashicorp/hc-install#205</a></li>
<li>Ensure license files are tracked during installation so they can be
removed by <a
href="https://github.com/radeksimko"><code>@​radeksimko</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/206">hashicorp/hc-install#206</a></li>
<li>Add support for custom download URLs by <a
href="https://github.com/james0209"><code>@​james0209</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/203">hashicorp/hc-install#203</a></li>
</ul>
<p>BUG FIXES:</p>
<ul>
<li>releasesjson: Fix missing <code>Version</code> in unmarshaled
response by <a
href="https://github.com/radeksimko"><code>@​radeksimko</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/192">hashicorp/hc-install#192</a></li>
</ul>
<p>DEPENDENCIES:</p>
<ul>
<li>build(deps): bump golang.org/x/mod from 0.16.0 to 0.17.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/194">hashicorp/hc-install#194</a></li>
<li>build(deps): bump golang.org/x/net from 0.22.0 to 0.23.0 by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/195">hashicorp/hc-install#195</a></li>
<li>build(deps): bump hashicorp/action-setup-bob from 1.0.20 to 2 in the
github-actions-breaking group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/208">hashicorp/hc-install#208</a></li>
<li>build(deps): bump hashicorp/setup-copywrite from 1.1.2 to 1.1.3 in
the github-actions-backward-compatible group by <a
href="https://github.com/dependabot"><code>@​dependabot</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/212">hashicorp/hc-install#212</a></li>
<li>build(deps): bump the github-actions-breaking group with 2 updates
by <a href="https://github.com/dependabot"><code>@​dependabot</code></a>
in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/211">hashicorp/hc-install#211</a></li>
<li>build(deps): Bump workflows to latest trusted versions by <a
href="https://github.com/hashicorp-tsccr"><code>@​hashicorp-tsccr</code></a>
in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/199">hashicorp/hc-install#199</a></li>
<li>build(deps): Bump workflows to latest trusted versions by <a
href="https://github.com/hashicorp-tsccr"><code>@​hashicorp-tsccr</code></a>
in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/204">hashicorp/hc-install#204</a></li>
<li>build(deps): Bump workflows to latest trusted versions by <a
href="https://github.com/hashicorp-tsccr"><code>@​hashicorp-tsccr</code></a>
in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/209">hashicorp/hc-install#209</a></li>
</ul>
<p>INTERNAL:</p>
<ul>
<li>[chore] Fix minor lint issue by <a
href="https://github.com/james0209"><code>@​james0209</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/202">hashicorp/hc-install#202</a></li>
<li>github: Create CODEOWNERS by <a
href="https://github.com/xiehan"><code>@​xiehan</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/210">hashicorp/hc-install#210</a></li>
<li>github: Set up Dependabot to manage HashiCorp-owned Actions
versioning by <a
href="https://github.com/xiehan"><code>@​xiehan</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/207">hashicorp/hc-install#207</a></li>
<li>Lint Markdown by <a
href="https://github.com/james0209"><code>@​james0209</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/200">hashicorp/hc-install#200</a></li>
<li>Replace deprecated <code>ioutil</code> usage with the equivalent
replacements by <a
href="https://github.com/james0209"><code>@​james0209</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/198">hashicorp/hc-install#198</a></li>
<li>Support darwin/arm64 in tests by <a
href="https://github.com/radeksimko"><code>@​radeksimko</code></a> in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/196">hashicorp/hc-install#196</a></li>
</ul>
<h2>New Contributors</h2>
<ul>
<li><a href="https://github.com/james0209"><code>@​james0209</code></a>
made their first contribution in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/198">hashicorp/hc-install#198</a></li>
<li><a href="https://github.com/xiehan"><code>@​xiehan</code></a> made
their first contribution in <a
href="https://redirect.github.com/hashicorp/hc-install/pull/207">hashicorp/hc-install#207</a></li>
</ul>
<p><strong>Full Changelog</strong>: <a
href="https://github.com/hashicorp/hc-install/compare/v0.6.4...v0.7.0">https://github.com/hashicorp/hc-install/compare/v0.6.4...v0.7.0</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/hashicorp/hc-install/commit/152a3b6753f8b8910249f4f67e920e29768d24c8"><code>152a3b6</code></a>
Release v0.7.0</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/237ac6f0aa9b638a61660dc89b44583030228ea9"><code>237ac6f</code></a>
Ensure license files are tracked during installation so they can be
removed (...</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/5a74938d4a64b04c5fc0fc158ecb4efe2d8aa125"><code>5a74938</code></a>
github: Create CODEOWNERS (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/210">#210</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/40acb8c900a3fb3ddad314e32f89dd0f148eaea2"><code>40acb8c</code></a>
build(deps): bump the github-actions-breaking group with 2 updates (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/211">#211</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/b19d1fcd874505e0a700ade445e95cf09b6d895d"><code>b19d1fc</code></a>
build(deps): bump hashicorp/setup-copywrite from 1.1.2 to 1.1.3 in the
github...</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/e0945975dcd7acec27af46cf622afe10dc58d99f"><code>e094597</code></a>
Result of tsccr-helper -log-level=info gha update -latest . (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/209">#209</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/b5c313e484f7554e6b73d3ad7f8e27636a19c462"><code>b5c313e</code></a>
build(deps): bump hashicorp/action-setup-bob (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/208">#208</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/35884ef0d18568590b049db27a4350ce25800ed7"><code>35884ef</code></a>
github: Set up Dependabot to manage HashiCorp-owned Actions versioning
(<a
href="https://redirect.github.com/hashicorp/hc-install/issues/207">#207</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/704a29eb54bc04ef2a567ce6115e5d743c1bb770"><code>704a29e</code></a>
Add support for custom download URLs (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/203">#203</a>)</li>
<li><a
href="https://github.com/hashicorp/hc-install/commit/7de7b37ef8010de8e494416d471f3f72bce3afac"><code>7de7b37</code></a>
Ensure license file gets packaged along w/ the CLI binary (<a
href="https://redirect.github.com/hashicorp/hc-install/issues/205">#205</a>)</li>
<li>Additional commits viewable in <a
href="https://github.com/hashicorp/hc-install/compare/v0.6.4...v0.7.0">compare
view</a></li>
</ul>
</details>
<br />


[![Dependabot compatibility
score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/hc-install&package-manager=go_modules&previous-version=0.6.4&new-version=0.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)

Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.

[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)

---

<details>
<summary>Dependabot commands and options</summary>
<br />

You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)


</details>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 8ccbeffc0..1b6c9aeb3 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/hashicorp/go-version v1.7.0 // MPL 2.0
-	github.com/hashicorp/hc-install v0.6.4 // MPL 2.0
+	github.com/hashicorp/hc-install v0.7.0 // MPL 2.0
 	github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
 	github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0
 	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
diff --git a/go.sum b/go.sum
index 71bd69bd6..723057ad9 100644
--- a/go.sum
+++ b/go.sum
@@ -99,8 +99,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
 github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9Km8e0=
-github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA=
+github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk=
+github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA=
 github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ=
 github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg=
 github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec=

From b2ea9dd97134dfca601e05ce15c043afef3844b6 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 29 May 2024 17:30:26 +0200
Subject: [PATCH 37/41] Remove unnecessary `filepath.FromSlash` calls (#1458)

## Changes

The prior join call calls `filepath.Join` which returns a cleaned
result.

Path cleaning, in turn, calls `filepath.FromSlash`.

## Tests

* Unit tests.
---
 libs/filer/local_client.go    | 6 ------
 libs/filer/local_root_path.go | 1 -
 2 files changed, 7 deletions(-)

diff --git a/libs/filer/local_client.go b/libs/filer/local_client.go
index 958b6277d..9398958f5 100644
--- a/libs/filer/local_client.go
+++ b/libs/filer/local_client.go
@@ -34,7 +34,6 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader,
 		flags |= os.O_EXCL
 	}
 
-	absPath = filepath.FromSlash(absPath)
 	f, err := os.OpenFile(absPath, flags, 0644)
 	if os.IsNotExist(err) && slices.Contains(mode, CreateParentDirectories) {
 		// Create parent directories if they don't exist.
@@ -76,7 +75,6 @@ func (w *LocalClient) Read(ctx context.Context, name string) (io.ReadCloser, err
 	// This stat call serves two purposes:
 	// 1. Checks file at path exists, and throws an error if it does not
 	// 2. Allows us to error out if the path is a directory
-	absPath = filepath.FromSlash(absPath)
 	stat, err := os.Stat(absPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -103,7 +101,6 @@ func (w *LocalClient) Delete(ctx context.Context, name string, mode ...DeleteMod
 		return CannotDeleteRootError{}
 	}
 
-	absPath = filepath.FromSlash(absPath)
 	err = os.Remove(absPath)
 
 	// Return early on success.
@@ -131,7 +128,6 @@ func (w *LocalClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry,
 		return nil, err
 	}
 
-	absPath = filepath.FromSlash(absPath)
 	stat, err := os.Stat(absPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -153,7 +149,6 @@ func (w *LocalClient) Mkdir(ctx context.Context, name string) error {
 		return err
 	}
 
-	dirPath = filepath.FromSlash(dirPath)
 	return os.MkdirAll(dirPath, 0755)
 }
 
@@ -163,7 +158,6 @@ func (w *LocalClient) Stat(ctx context.Context, name string) (fs.FileInfo, error
 		return nil, err
 	}
 
-	absPath = filepath.FromSlash(absPath)
 	stat, err := os.Stat(absPath)
 	if os.IsNotExist(err) {
 		return nil, FileDoesNotExistError{path: absPath}
diff --git a/libs/filer/local_root_path.go b/libs/filer/local_root_path.go
index 15a542631..3f8843093 100644
--- a/libs/filer/local_root_path.go
+++ b/libs/filer/local_root_path.go
@@ -19,7 +19,6 @@ func NewLocalRootPath(root string) localRootPath {
 
 func (rp *localRootPath) Join(name string) (string, error) {
 	absPath := filepath.Join(rp.rootPath, name)
-
 	if !strings.HasPrefix(absPath, rp.rootPath) {
 		return "", fmt.Errorf("relative path escapes root: %s", name)
 	}

From 424499ec1d56db0ef153b68357bb49257c2fe6aa Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Thu, 30 May 2024 09:41:50 +0200
Subject: [PATCH 38/41] Abstract over filesystem interaction with libs/vfs
 (#1452)

## Changes

Introduce `libs/vfs` for an implementation of `fs.FS` and friends that
_includes_ the absolute path it is anchored to.

This is needed for:
1. Intercepting file operations to inject custom logic (e.g., logging,
access control).
2. Traversing directories to find specific leaf directories (e.g.,
`.git`).
3. Converting virtual paths to OS-native paths.

Options 2 and 3 are not possible with the standard `fs.FS` interface.
They are needed such that we can provide an instance to the sync package
and still detect the containing `.git` directory and convert paths to
native paths.

This change focuses on making the following packages use `vfs.Path`:
* libs/fileset
* libs/git
* libs/sync

All entries returned by `fileset.All` are now slash-separated. This has
2 consequences:
* The sync snapshot now always uses slash-separated paths
* We don't need to call `filepath.FromSlash` as much as we did

## Tests

* All unit tests pass
* All integration tests pass
* Manually confirmed that a deployment made on Windows by a previous
version of the CLI can be deployed by a new version of the CLI while
retaining the validity of the local sync snapshot as well as the remote
deployment state.
---
 bundle/bundle.go                              |  3 +-
 bundle/config/mutator/load_git_details.go     |  3 +-
 .../config/validate/validate_sync_patterns.go |  3 +-
 bundle/deploy/files/sync.go                   |  3 +-
 bundle/deploy/state.go                        | 13 ++-
 bundle/deploy/state_test.go                   | 23 ++----
 cmd/sync/sync.go                              |  3 +-
 cmd/sync/sync_test.go                         | 11 ++-
 internal/sync_test.go                         |  6 +-
 libs/fileset/file.go                          | 44 ++++++----
 libs/fileset/file_test.go                     | 15 ++--
 libs/fileset/fileset.go                       | 36 ++++----
 libs/fileset/glob.go                          | 19 ++---
 libs/fileset/glob_test.go                     | 82 ++++++-------------
 libs/git/config.go                            |  9 +-
 libs/git/fileset.go                           |  7 +-
 libs/git/fileset_test.go                      | 18 ++--
 libs/git/ignore.go                            | 25 ++++--
 libs/git/ignore_test.go                       |  7 +-
 libs/git/reference.go                         | 13 ++-
 libs/git/reference_test.go                    |  9 +-
 libs/git/repository.go                        | 49 +++++------
 libs/git/repository_test.go                   | 17 ++--
 libs/git/view.go                              | 35 ++++----
 libs/git/view_test.go                         | 29 +++----
 libs/notebook/detect.go                       | 21 +++--
 libs/notebook/detect_jupyter.go               | 20 +++--
 libs/sync/diff.go                             |  5 +-
 libs/sync/dirset.go                           |  5 +-
 libs/sync/snapshot.go                         |  5 ++
 libs/sync/snapshot_state.go                   | 32 +++++++-
 libs/sync/snapshot_state_test.go              | 39 +++++++--
 libs/sync/snapshot_test.go                    | 13 +--
 libs/sync/sync.go                             |  6 +-
 libs/sync/sync_test.go                        | 23 +++---
 libs/sync/watchdog.go                         |  4 +-
 libs/vfs/leaf.go                              | 29 +++++++
 libs/vfs/leaf_test.go                         | 38 +++++++++
 libs/vfs/os.go                                | 82 +++++++++++++++++++
 libs/vfs/os_test.go                           | 54 ++++++++++++
 libs/vfs/path.go                              | 29 +++++++
 libs/vfs/path_test.go                         |  1 +
 42 files changed, 603 insertions(+), 285 deletions(-)
 create mode 100644 libs/vfs/leaf.go
 create mode 100644 libs/vfs/leaf_test.go
 create mode 100644 libs/vfs/os.go
 create mode 100644 libs/vfs/os_test.go
 create mode 100644 libs/vfs/path.go
 create mode 100644 libs/vfs/path_test.go

diff --git a/bundle/bundle.go b/bundle/bundle.go
index 977ca2247..1dc98656a 100644
--- a/bundle/bundle.go
+++ b/bundle/bundle.go
@@ -22,6 +22,7 @@ import (
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/tags"
 	"github.com/databricks/cli/libs/terraform"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go"
 	sdkconfig "github.com/databricks/databricks-sdk-go/config"
 	"github.com/hashicorp/terraform-exec/tfexec"
@@ -208,7 +209,7 @@ func (b *Bundle) GitRepository() (*git.Repository, error) {
 		return nil, fmt.Errorf("unable to locate repository root: %w", err)
 	}
 
-	return git.NewRepository(rootPath)
+	return git.NewRepository(vfs.MustNew(rootPath))
 }
 
 // AuthEnv returns a map with environment variables and their values
diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go
index 7ce8476f1..d8b76f39e 100644
--- a/bundle/config/mutator/load_git_details.go
+++ b/bundle/config/mutator/load_git_details.go
@@ -8,6 +8,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 type loadGitDetails struct{}
@@ -22,7 +23,7 @@ func (m *loadGitDetails) Name() string {
 
 func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 	// Load relevant git repository
-	repo, err := git.NewRepository(b.RootPath)
+	repo, err := git.NewRepository(vfs.MustNew(b.RootPath))
 	if err != nil {
 		return diag.FromErr(err)
 	}
diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go
index 58acf6ae4..832efede9 100644
--- a/bundle/config/validate/validate_sync_patterns.go
+++ b/bundle/config/validate/validate_sync_patterns.go
@@ -8,6 +8,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -50,7 +51,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di
 		index := i
 		p := pattern
 		errs.Go(func() error {
-			fs, err := fileset.NewGlobSet(rb.RootPath(), []string{p})
+			fs, err := fileset.NewGlobSet(vfs.MustNew(rb.RootPath()), []string{p})
 			if err != nil {
 				return err
 			}
diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go
index d78ab2d74..8d6efdae3 100644
--- a/bundle/deploy/files/sync.go
+++ b/bundle/deploy/files/sync.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/sync"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 func GetSync(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.Sync, error) {
@@ -28,7 +29,7 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp
 	}
 
 	opts := &sync.SyncOptions{
-		LocalPath:  rb.RootPath(),
+		LocalPath:  vfs.MustNew(rb.RootPath()),
 		RemotePath: rb.Config().Workspace.FilePath,
 		Include:    includes,
 		Exclude:    rb.Config().Sync.Exclude,
diff --git a/bundle/deploy/state.go b/bundle/deploy/state.go
index ffcadc9d6..ccff64fe7 100644
--- a/bundle/deploy/state.go
+++ b/bundle/deploy/state.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 const DeploymentStateFileName = "deployment.json"
@@ -112,12 +113,18 @@ func FromSlice(files []fileset.File) (Filelist, error) {
 
 func (f Filelist) ToSlice(basePath string) []fileset.File {
 	var files []fileset.File
+	root := vfs.MustNew(basePath)
 	for _, file := range f {
-		absPath := filepath.Join(basePath, file.LocalPath)
+		entry := newEntry(filepath.Join(basePath, file.LocalPath))
+
+		// Snapshots created with versions <= v0.220.0 use platform-specific
+		// paths (i.e. with backslashes). Files returned by [libs/fileset] always
+		// contain forward slashes after this version. Normalize before using.
+		relative := filepath.ToSlash(file.LocalPath)
 		if file.IsNotebook {
-			files = append(files, fileset.NewNotebookFile(newEntry(absPath), absPath, file.LocalPath))
+			files = append(files, fileset.NewNotebookFile(root, entry, relative))
 		} else {
-			files = append(files, fileset.NewSourceFile(newEntry(absPath), absPath, file.LocalPath))
+			files = append(files, fileset.NewSourceFile(root, entry, relative))
 		}
 	}
 	return files
diff --git a/bundle/deploy/state_test.go b/bundle/deploy/state_test.go
index 15bdc96b4..efa051ab6 100644
--- a/bundle/deploy/state_test.go
+++ b/bundle/deploy/state_test.go
@@ -3,17 +3,17 @@ package deploy
 import (
 	"bytes"
 	"encoding/json"
-	"path/filepath"
 	"testing"
 
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/require"
 )
 
 func TestFromSlice(t *testing.T) {
 	tmpDir := t.TempDir()
-	fileset := fileset.New(tmpDir)
+	fileset := fileset.New(vfs.MustNew(tmpDir))
 	testutil.Touch(t, tmpDir, "test1.py")
 	testutil.Touch(t, tmpDir, "test2.py")
 	testutil.Touch(t, tmpDir, "test3.py")
@@ -32,7 +32,7 @@ func TestFromSlice(t *testing.T) {
 
 func TestToSlice(t *testing.T) {
 	tmpDir := t.TempDir()
-	fileset := fileset.New(tmpDir)
+	fileset := fileset.New(vfs.MustNew(tmpDir))
 	testutil.Touch(t, tmpDir, "test1.py")
 	testutil.Touch(t, tmpDir, "test2.py")
 	testutil.Touch(t, tmpDir, "test3.py")
@@ -48,18 +48,11 @@ func TestToSlice(t *testing.T) {
 	require.Len(t, s, 3)
 
 	for _, file := range s {
-		require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Name())
-		require.Contains(t, []string{
-			filepath.Join(tmpDir, "test1.py"),
-			filepath.Join(tmpDir, "test2.py"),
-			filepath.Join(tmpDir, "test3.py"),
-		}, file.Absolute)
-		require.False(t, file.IsDir())
-		require.NotZero(t, file.Type())
-		info, err := file.Info()
-		require.NoError(t, err)
-		require.NotNil(t, info)
-		require.Equal(t, file.Name(), info.Name())
+		require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Relative)
+
+		// If the mtime is not zero we know we produced a valid fs.DirEntry.
+		ts := file.Modified()
+		require.NotZero(t, ts)
 	}
 }
 
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index 42550722b..e5f1bfc9e 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -14,6 +14,7 @@ import (
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/sync"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/spf13/cobra"
 )
 
@@ -46,7 +47,7 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn
 	}
 
 	opts := sync.SyncOptions{
-		LocalPath:    args[0],
+		LocalPath:    vfs.MustNew(args[0]),
 		RemotePath:   args[1],
 		Full:         f.full,
 		PollInterval: f.interval,
diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go
index 026d840f7..b741e7b16 100644
--- a/cmd/sync/sync_test.go
+++ b/cmd/sync/sync_test.go
@@ -31,7 +31,7 @@ func TestSyncOptionsFromBundle(t *testing.T) {
 	f := syncFlags{}
 	opts, err := f.syncOptionsFromBundle(New(), []string{}, b)
 	require.NoError(t, err)
-	assert.Equal(t, tempDir, opts.LocalPath)
+	assert.Equal(t, tempDir, opts.LocalPath.Native())
 	assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath)
 	assert.Equal(t, filepath.Join(tempDir, ".databricks", "bundle", "default"), opts.SnapshotBasePath)
 	assert.NotNil(t, opts.WorkspaceClient)
@@ -49,11 +49,14 @@ func TestSyncOptionsFromArgsRequiredTwoArgs(t *testing.T) {
 }
 
 func TestSyncOptionsFromArgs(t *testing.T) {
+	local := t.TempDir()
+	remote := "/remote"
+
 	f := syncFlags{}
 	cmd := New()
 	cmd.SetContext(root.SetWorkspaceClient(context.Background(), nil))
-	opts, err := f.syncOptionsFromArgs(cmd, []string{"/local", "/remote"})
+	opts, err := f.syncOptionsFromArgs(cmd, []string{local, remote})
 	require.NoError(t, err)
-	assert.Equal(t, "/local", opts.LocalPath)
-	assert.Equal(t, "/remote", opts.RemotePath)
+	assert.Equal(t, local, opts.LocalPath.Native())
+	assert.Equal(t, remote, opts.RemotePath)
 }
diff --git a/internal/sync_test.go b/internal/sync_test.go
index f970a7ce0..4021e6490 100644
--- a/internal/sync_test.go
+++ b/internal/sync_test.go
@@ -313,7 +313,7 @@ func TestAccSyncNestedFolderSync(t *testing.T) {
 	assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"})
 	assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"})
 	assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"})
-	assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("dir1/dir2/dir3/foo.txt")))
+	assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/dir2/dir3/foo.txt"))
 
 	// delete
 	f.Remove(t)
@@ -374,7 +374,7 @@ func TestAccSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) {
 	assertSync.remoteDirContent(ctx, "dir1", []string{"a b+c"})
 	assertSync.remoteDirContent(ctx, "dir1/a b+c", []string{"c+d e"})
 	assertSync.remoteDirContent(ctx, "dir1/a b+c/c+d e", []string{"e+f g#i.txt"})
-	assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("dir1/a b+c/c+d e/e+f g#i.txt")))
+	assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/a b+c/c+d e/e+f g#i.txt"))
 
 	// delete
 	f.Remove(t)
@@ -404,7 +404,7 @@ func TestAccSyncIncrementalFileOverwritesFolder(t *testing.T) {
 	assertSync.waitForCompletionMarker()
 	assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo"))
 	assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"})
-	assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("foo/bar.txt")))
+	assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo/bar.txt"))
 
 	// delete foo/bar.txt
 	f.Remove(t)
diff --git a/libs/fileset/file.go b/libs/fileset/file.go
index 17cae7952..fd846b257 100644
--- a/libs/fileset/file.go
+++ b/libs/fileset/file.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/databricks/cli/libs/notebook"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 type fileType int
@@ -16,40 +17,49 @@ const (
 )
 
 type File struct {
-	fs.DirEntry
-	Absolute, Relative string
-	fileType           fileType
+	// Root path of the fileset.
+	root vfs.Path
+
+	// File entry as returned by the [fs.WalkDir] function.
+	entry fs.DirEntry
+
+	// Type of the file.
+	fileType fileType
+
+	// Relative path within the fileset.
+	// Combine with the [vfs.Path] to interact with the underlying file.
+	Relative string
 }
 
-func NewNotebookFile(entry fs.DirEntry, absolute string, relative string) File {
+func NewNotebookFile(root vfs.Path, entry fs.DirEntry, relative string) File {
 	return File{
-		DirEntry: entry,
-		Absolute: absolute,
-		Relative: relative,
+		root:     root,
+		entry:    entry,
 		fileType: Notebook,
+		Relative: relative,
 	}
 }
 
-func NewFile(entry fs.DirEntry, absolute string, relative string) File {
+func NewFile(root vfs.Path, entry fs.DirEntry, relative string) File {
 	return File{
-		DirEntry: entry,
-		Absolute: absolute,
-		Relative: relative,
+		root:     root,
+		entry:    entry,
 		fileType: Unknown,
+		Relative: relative,
 	}
 }
 
-func NewSourceFile(entry fs.DirEntry, absolute string, relative string) File {
+func NewSourceFile(root vfs.Path, entry fs.DirEntry, relative string) File {
 	return File{
-		DirEntry: entry,
-		Absolute: absolute,
-		Relative: relative,
+		root:     root,
+		entry:    entry,
 		fileType: Source,
+		Relative: relative,
 	}
 }
 
 func (f File) Modified() (ts time.Time) {
-	info, err := f.Info()
+	info, err := f.entry.Info()
 	if err != nil {
 		// return default time, beginning of epoch
 		return ts
@@ -63,7 +73,7 @@ func (f *File) IsNotebook() (bool, error) {
 	}
 
 	// Otherwise, detect the notebook type.
-	isNotebook, _, err := notebook.Detect(f.Absolute)
+	isNotebook, _, err := notebook.DetectWithFS(f.root, f.Relative)
 	if err != nil {
 		return false, err
 	}
diff --git a/libs/fileset/file_test.go b/libs/fileset/file_test.go
index cdfc9ba17..1ce1ff59a 100644
--- a/libs/fileset/file_test.go
+++ b/libs/fileset/file_test.go
@@ -1,22 +1,22 @@
 package fileset
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/databricks/cli/internal/testutil"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNotebookFileIsNotebook(t *testing.T) {
-	f := NewNotebookFile(nil, "", "")
+	f := NewNotebookFile(nil, nil, "")
 	isNotebook, err := f.IsNotebook()
 	require.NoError(t, err)
 	require.True(t, isNotebook)
 }
 
 func TestSourceFileIsNotNotebook(t *testing.T) {
-	f := NewSourceFile(nil, "", "")
+	f := NewSourceFile(nil, nil, "")
 	isNotebook, err := f.IsNotebook()
 	require.NoError(t, err)
 	require.False(t, isNotebook)
@@ -24,18 +24,19 @@ func TestSourceFileIsNotNotebook(t *testing.T) {
 
 func TestUnknownFileDetectsNotebook(t *testing.T) {
 	tmpDir := t.TempDir()
+	root := vfs.MustNew(tmpDir)
 
 	t.Run("file", func(t *testing.T) {
-		path := testutil.Touch(t, tmpDir, "test.py")
-		f := NewFile(nil, path, filepath.Base(path))
+		testutil.Touch(t, tmpDir, "test.py")
+		f := NewFile(root, nil, "test.py")
 		isNotebook, err := f.IsNotebook()
 		require.NoError(t, err)
 		require.False(t, isNotebook)
 	})
 
 	t.Run("notebook", func(t *testing.T) {
-		path := testutil.TouchNotebook(t, tmpDir, "notebook.py")
-		f := NewFile(nil, path, filepath.Base(path))
+		testutil.TouchNotebook(t, tmpDir, "notebook.py")
+		f := NewFile(root, nil, "notebook.py")
 		isNotebook, err := f.IsNotebook()
 		require.NoError(t, err)
 		require.True(t, isNotebook)
diff --git a/libs/fileset/fileset.go b/libs/fileset/fileset.go
index 52463dff3..d0f00f97a 100644
--- a/libs/fileset/fileset.go
+++ b/libs/fileset/fileset.go
@@ -4,20 +4,24 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
-	"path/filepath"
+
+	"github.com/databricks/cli/libs/vfs"
 )
 
 // FileSet facilitates fast recursive file listing of a path.
 // It optionally takes into account ignore rules through the [Ignorer] interface.
 type FileSet struct {
-	root   string
+	// Root path of the fileset.
+	root vfs.Path
+
+	// Ignorer interface to check if a file or directory should be ignored.
 	ignore Ignorer
 }
 
 // New returns a [FileSet] for the given root path.
-func New(root string) *FileSet {
+func New(root vfs.Path) *FileSet {
 	return &FileSet{
-		root:   filepath.Clean(root),
+		root:   root,
 		ignore: nopIgnorer{},
 	}
 }
@@ -32,11 +36,6 @@ func (w *FileSet) SetIgnorer(ignore Ignorer) {
 	w.ignore = ignore
 }
 
-// Return root for fileset.
-func (w *FileSet) Root() string {
-	return w.root
-}
-
 // Return all tracked files for Repo
 func (w *FileSet) All() ([]File, error) {
 	return w.recursiveListFiles()
@@ -46,12 +45,7 @@ func (w *FileSet) All() ([]File, error) {
 // that are being tracked in the FileSet (ie not being ignored for matching one of the
 // patterns in w.ignore)
 func (w *FileSet) recursiveListFiles() (fileList []File, err error) {
-	err = filepath.WalkDir(w.root, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		relPath, err := filepath.Rel(w.root, path)
+	err = fs.WalkDir(w.root, ".", func(name string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -66,25 +60,25 @@ func (w *FileSet) recursiveListFiles() (fileList []File, err error) {
 		}
 
 		if d.IsDir() {
-			ign, err := w.ignore.IgnoreDirectory(relPath)
+			ign, err := w.ignore.IgnoreDirectory(name)
 			if err != nil {
-				return fmt.Errorf("cannot check if %s should be ignored: %w", relPath, err)
+				return fmt.Errorf("cannot check if %s should be ignored: %w", name, err)
 			}
 			if ign {
-				return filepath.SkipDir
+				return fs.SkipDir
 			}
 			return nil
 		}
 
-		ign, err := w.ignore.IgnoreFile(relPath)
+		ign, err := w.ignore.IgnoreFile(name)
 		if err != nil {
-			return fmt.Errorf("cannot check if %s should be ignored: %w", relPath, err)
+			return fmt.Errorf("cannot check if %s should be ignored: %w", name, err)
 		}
 		if ign {
 			return nil
 		}
 
-		fileList = append(fileList, NewFile(d, path, relPath))
+		fileList = append(fileList, NewFile(w.root, d, name))
 		return nil
 	})
 	return
diff --git a/libs/fileset/glob.go b/libs/fileset/glob.go
index 9d8626e54..0a1038472 100644
--- a/libs/fileset/glob.go
+++ b/libs/fileset/glob.go
@@ -1,22 +1,17 @@
 package fileset
 
 import (
-	"path/filepath"
+	"path"
+
+	"github.com/databricks/cli/libs/vfs"
 )
 
-func NewGlobSet(root string, includes []string) (*FileSet, error) {
-	absRoot, err := filepath.Abs(root)
-	if err != nil {
-		return nil, err
-	}
-
+func NewGlobSet(root vfs.Path, includes []string) (*FileSet, error) {
 	for k := range includes {
-		includes[k] = filepath.ToSlash(filepath.Clean(includes[k]))
+		includes[k] = path.Clean(includes[k])
 	}
 
-	fs := &FileSet{
-		absRoot,
-		newIncluder(includes),
-	}
+	fs := New(root)
+	fs.SetIgnorer(newIncluder(includes))
 	return fs, nil
 }
diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go
index e8d3696c4..70b9c444b 100644
--- a/libs/fileset/glob_test.go
+++ b/libs/fileset/glob_test.go
@@ -2,21 +2,26 @@ package fileset
 
 import (
 	"io/fs"
-	"os"
-	"path/filepath"
+	"path"
 	"slices"
 	"strings"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/require"
 )
 
-func TestGlobFileset(t *testing.T) {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
-	root := filepath.Join(cwd, "..", "filer")
+func collectRelativePaths(files []File) []string {
+	relativePaths := make([]string, 0)
+	for _, f := range files {
+		relativePaths = append(relativePaths, f.Relative)
+	}
+	return relativePaths
+}
 
-	entries, err := os.ReadDir(root)
+func TestGlobFileset(t *testing.T) {
+	root := vfs.MustNew("../filer")
+	entries, err := root.ReadDir(".")
 	require.NoError(t, err)
 
 	g, err := NewGlobSet(root, []string{
@@ -30,7 +35,7 @@ func TestGlobFileset(t *testing.T) {
 	require.Equal(t, len(files), len(entries))
 	for _, f := range files {
 		exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool {
-			return de.Name() == f.Name()
+			return de.Name() == path.Base(f.Relative)
 		})
 		require.True(t, exists)
 	}
@@ -46,9 +51,8 @@ func TestGlobFileset(t *testing.T) {
 }
 
 func TestGlobFilesetWithRelativeRoot(t *testing.T) {
-	root := filepath.Join("..", "filer")
-
-	entries, err := os.ReadDir(root)
+	root := vfs.MustNew("../filer")
+	entries, err := root.ReadDir(".")
 	require.NoError(t, err)
 
 	g, err := NewGlobSet(root, []string{
@@ -58,21 +62,14 @@ func TestGlobFilesetWithRelativeRoot(t *testing.T) {
 
 	files, err := g.All()
 	require.NoError(t, err)
-
 	require.Equal(t, len(files), len(entries))
-	for _, f := range files {
-		require.True(t, filepath.IsAbs(f.Absolute))
-	}
 }
 
 func TestGlobFilesetRecursively(t *testing.T) {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
-	root := filepath.Join(cwd, "..", "git")
-
+	root := vfs.MustNew("../git")
 	entries := make([]string, 0)
-	err = filepath.Walk(filepath.Join(root, "testdata"), func(path string, info fs.FileInfo, err error) error {
-		if !info.IsDir() {
+	err := fs.WalkDir(root, "testdata", func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
 			entries = append(entries, path)
 		}
 		return nil
@@ -86,24 +83,14 @@ func TestGlobFilesetRecursively(t *testing.T) {
 
 	files, err := g.All()
 	require.NoError(t, err)
-
-	require.Equal(t, len(files), len(entries))
-	for _, f := range files {
-		exists := slices.ContainsFunc(entries, func(path string) bool {
-			return path == f.Absolute
-		})
-		require.True(t, exists)
-	}
+	require.ElementsMatch(t, entries, collectRelativePaths(files))
 }
 
 func TestGlobFilesetDir(t *testing.T) {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
-	root := filepath.Join(cwd, "..", "git")
-
+	root := vfs.MustNew("../git")
 	entries := make([]string, 0)
-	err = filepath.Walk(filepath.Join(root, "testdata", "a"), func(path string, info fs.FileInfo, err error) error {
-		if !info.IsDir() {
+	err := fs.WalkDir(root, "testdata/a", func(path string, d fs.DirEntry, err error) error {
+		if !d.IsDir() {
 			entries = append(entries, path)
 		}
 		return nil
@@ -117,23 +104,13 @@ func TestGlobFilesetDir(t *testing.T) {
 
 	files, err := g.All()
 	require.NoError(t, err)
-
-	require.Equal(t, len(files), len(entries))
-	for _, f := range files {
-		exists := slices.ContainsFunc(entries, func(path string) bool {
-			return path == f.Absolute
-		})
-		require.True(t, exists)
-	}
+	require.ElementsMatch(t, entries, collectRelativePaths(files))
 }
 
 func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
-	root := filepath.Join(cwd, "..", "git")
-
+	root := vfs.MustNew("../git")
 	entries := make([]string, 0)
-	err = filepath.Walk(filepath.Join(root, "testdata"), func(path string, info fs.FileInfo, err error) error {
+	err := fs.WalkDir(root, "testdata", func(path string, d fs.DirEntry, err error) error {
 		if strings.HasSuffix(path, ".txt") {
 			entries = append(entries, path)
 		}
@@ -148,12 +125,5 @@ func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) {
 
 	files, err := g.All()
 	require.NoError(t, err)
-
-	require.Equal(t, len(files), len(entries))
-	for _, f := range files {
-		exists := slices.ContainsFunc(entries, func(path string) bool {
-			return path == f.Absolute
-		})
-		require.True(t, exists)
-	}
+	require.ElementsMatch(t, entries, collectRelativePaths(files))
 }
diff --git a/libs/git/config.go b/libs/git/config.go
index e83c75b7b..424d453bc 100644
--- a/libs/git/config.go
+++ b/libs/git/config.go
@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/databricks/cli/libs/vfs"
 	"gopkg.in/ini.v1"
 )
 
@@ -87,8 +88,8 @@ func (c config) load(r io.Reader) error {
 	return nil
 }
 
-func (c config) loadFile(path string) error {
-	f, err := os.Open(path)
+func (c config) loadFile(fs vfs.Path, path string) error {
+	f, err := fs.Open(path)
 	if err != nil {
 		// If the file doesn't exist it is ignored.
 		// This is the case for both global and repository specific config files.
@@ -152,8 +153,8 @@ func globalGitConfig() (*config, error) {
 	// > are missing or unreadable they will be ignored.
 	//
 	// We therefore ignore the error return value for the calls below.
-	config.loadFile(filepath.Join(xdgConfigHome, "git/config"))
-	config.loadFile(filepath.Join(config.home, ".gitconfig"))
+	config.loadFile(vfs.MustNew(xdgConfigHome), "git/config")
+	config.loadFile(vfs.MustNew(config.home), ".gitconfig")
 
 	return config, nil
 }
diff --git a/libs/git/fileset.go b/libs/git/fileset.go
index c604ac7fa..f1986aa20 100644
--- a/libs/git/fileset.go
+++ b/libs/git/fileset.go
@@ -2,6 +2,7 @@ package git
 
 import (
 	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 // FileSet is Git repository aware implementation of [fileset.FileSet].
@@ -13,7 +14,7 @@ type FileSet struct {
 }
 
 // NewFileSet returns [FileSet] for the Git repository located at `root`.
-func NewFileSet(root string) (*FileSet, error) {
+func NewFileSet(root vfs.Path) (*FileSet, error) {
 	fs := fileset.New(root)
 	v, err := NewView(root)
 	if err != nil {
@@ -34,10 +35,6 @@ func (f *FileSet) IgnoreDirectory(dir string) (bool, error) {
 	return f.view.IgnoreDirectory(dir)
 }
 
-func (f *FileSet) Root() string {
-	return f.fileset.Root()
-}
-
 func (f *FileSet) All() ([]fileset.File, error) {
 	f.view.repo.taintIgnoreRules()
 	return f.fileset.All()
diff --git a/libs/git/fileset_test.go b/libs/git/fileset_test.go
index 74133f525..4e6172bfd 100644
--- a/libs/git/fileset_test.go
+++ b/libs/git/fileset_test.go
@@ -2,23 +2,25 @@ package git
 
 import (
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func testFileSetAll(t *testing.T, path string) {
-	fileSet, err := NewFileSet(path)
+func testFileSetAll(t *testing.T, root string) {
+	fileSet, err := NewFileSet(vfs.MustNew(root))
 	require.NoError(t, err)
 	files, err := fileSet.All()
 	require.NoError(t, err)
 	require.Len(t, files, 3)
-	assert.Equal(t, filepath.Join("a", "b", "world.txt"), files[0].Relative)
-	assert.Equal(t, filepath.Join("a", "hello.txt"), files[1].Relative)
-	assert.Equal(t, filepath.Join("databricks.yml"), files[2].Relative)
+	assert.Equal(t, path.Join("a", "b", "world.txt"), files[0].Relative)
+	assert.Equal(t, path.Join("a", "hello.txt"), files[1].Relative)
+	assert.Equal(t, path.Join("databricks.yml"), files[2].Relative)
 }
 
 func TestFileSetListAllInRepo(t *testing.T) {
@@ -33,7 +35,7 @@ func TestFileSetNonCleanRoot(t *testing.T) {
 	// Test what happens if the root directory can be simplified.
 	// Path simplification is done by most filepath functions.
 	// This should yield the same result as above test.
-	fileSet, err := NewFileSet("./testdata/../testdata")
+	fileSet, err := NewFileSet(vfs.MustNew("./testdata/../testdata"))
 	require.NoError(t, err)
 	files, err := fileSet.All()
 	require.NoError(t, err)
@@ -42,7 +44,7 @@ func TestFileSetNonCleanRoot(t *testing.T) {
 
 func TestFileSetAddsCacheDirToGitIgnore(t *testing.T) {
 	projectDir := t.TempDir()
-	fileSet, err := NewFileSet(projectDir)
+	fileSet, err := NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	fileSet.EnsureValidGitIgnoreExists()
 
@@ -57,7 +59,7 @@ func TestFileSetDoesNotCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) {
 	projectDir := t.TempDir()
 	gitIgnorePath := filepath.Join(projectDir, ".gitignore")
 
-	fileSet, err := NewFileSet(projectDir)
+	fileSet, err := NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644)
 	require.NoError(t, err)
diff --git a/libs/git/ignore.go b/libs/git/ignore.go
index ec66a2b23..df3a4e919 100644
--- a/libs/git/ignore.go
+++ b/libs/git/ignore.go
@@ -1,9 +1,12 @@
 package git
 
 import (
+	"io/fs"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/databricks/cli/libs/vfs"
 	ignore "github.com/sabhiram/go-gitignore"
 )
 
@@ -21,7 +24,8 @@ type ignoreRules interface {
 // ignoreFile represents a gitignore file backed by a path.
 // If the path doesn't exist (yet), it is treated as an empty file.
 type ignoreFile struct {
-	absPath string
+	root vfs.Path
+	path string
 
 	// Signal a reload of this file.
 	// Set this to call [os.Stat] and a potential reload
@@ -35,9 +39,10 @@ type ignoreFile struct {
 	patterns *ignore.GitIgnore
 }
 
-func newIgnoreFile(absPath string) ignoreRules {
+func newIgnoreFile(root vfs.Path, path string) ignoreRules {
 	return &ignoreFile{
-		absPath:        absPath,
+		root:           root,
+		path:           path,
 		checkForReload: true,
 	}
 }
@@ -67,7 +72,7 @@ func (f *ignoreFile) Taint() {
 func (f *ignoreFile) load() error {
 	// The file must be stat-able.
 	// If it doesn't exist, treat it as an empty file.
-	stat, err := os.Stat(f.absPath)
+	stat, err := fs.Stat(f.root, f.path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
@@ -82,7 +87,7 @@ func (f *ignoreFile) load() error {
 	}
 
 	f.modTime = stat.ModTime()
-	f.patterns, err = ignore.CompileIgnoreFile(f.absPath)
+	f.patterns, err = f.loadGitignore()
 	if err != nil {
 		return err
 	}
@@ -90,6 +95,16 @@ func (f *ignoreFile) load() error {
 	return nil
 }
 
+func (f *ignoreFile) loadGitignore() (*ignore.GitIgnore, error) {
+	data, err := fs.ReadFile(f.root, f.path)
+	if err != nil {
+		return nil, err
+	}
+
+	lines := strings.Split(string(data), "\n")
+	return ignore.CompileIgnoreLines(lines...), nil
+}
+
 // stringIgnoreRules implements the [ignoreRules] interface
 // for a set of in-memory ignore patterns.
 type stringIgnoreRules struct {
diff --git a/libs/git/ignore_test.go b/libs/git/ignore_test.go
index 160f53d7b..057c0cb2e 100644
--- a/libs/git/ignore_test.go
+++ b/libs/git/ignore_test.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -13,7 +14,7 @@ func TestIgnoreFile(t *testing.T) {
 	var ign bool
 	var err error
 
-	f := newIgnoreFile("./testdata/.gitignore")
+	f := newIgnoreFile(vfs.MustNew("testdata"), ".gitignore")
 	ign, err = f.MatchesPath("root.foo")
 	require.NoError(t, err)
 	assert.True(t, ign)
@@ -27,7 +28,7 @@ func TestIgnoreFileDoesntExist(t *testing.T) {
 	var err error
 
 	// Files that don't exist are treated as an empty gitignore file.
-	f := newIgnoreFile("./testdata/thispathdoesntexist")
+	f := newIgnoreFile(vfs.MustNew("testdata"), "thispathdoesntexist")
 	ign, err = f.MatchesPath("i'm included")
 	require.NoError(t, err)
 	assert.False(t, ign)
@@ -41,7 +42,7 @@ func TestIgnoreFileTaint(t *testing.T) {
 	gitIgnorePath := filepath.Join(tempDir, ".gitignore")
 
 	// Files that don't exist are treated as an empty gitignore file.
-	f := newIgnoreFile(gitIgnorePath)
+	f := newIgnoreFile(vfs.MustNew(tempDir), ".gitignore")
 	ign, err = f.MatchesPath("hello")
 	require.NoError(t, err)
 	assert.False(t, ign)
diff --git a/libs/git/reference.go b/libs/git/reference.go
index 4021f2e60..2b4bd3e4d 100644
--- a/libs/git/reference.go
+++ b/libs/git/reference.go
@@ -2,10 +2,12 @@ package git
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/databricks/cli/libs/vfs"
 )
 
 type ReferenceType string
@@ -37,9 +39,9 @@ func isSHA1(s string) bool {
 	return re.MatchString(s)
 }
 
-func LoadReferenceFile(path string) (*Reference, error) {
+func LoadReferenceFile(root vfs.Path, path string) (*Reference, error) {
 	// read reference file content
-	b, err := os.ReadFile(path)
+	b, err := fs.ReadFile(root, path)
 	if os.IsNotExist(err) {
 		return nil, nil
 	}
@@ -73,8 +75,7 @@ func (ref *Reference) ResolvePath() (string, error) {
 	if ref.Type != ReferenceTypePointer {
 		return "", ErrNotAReferencePointer
 	}
-	refPath := strings.TrimPrefix(ref.Content, ReferencePrefix)
-	return filepath.FromSlash(refPath), nil
+	return strings.TrimPrefix(ref.Content, ReferencePrefix), nil
 }
 
 // resolves the name of the current branch from the reference file content. For example
@@ -87,8 +88,6 @@ func (ref *Reference) CurrentBranch() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	// normalize branch ref path to work accross different operating systems
-	branchRefPath = filepath.ToSlash(branchRefPath)
 	if !strings.HasPrefix(branchRefPath, HeadPathPrefix) {
 		return "", fmt.Errorf("reference path %s does not have expected prefix %s", branchRefPath, HeadPathPrefix)
 	}
diff --git a/libs/git/reference_test.go b/libs/git/reference_test.go
index 1b08e989b..194d79333 100644
--- a/libs/git/reference_test.go
+++ b/libs/git/reference_test.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -45,7 +46,7 @@ func TestReferenceReferencePathForReference(t *testing.T) {
 	}
 	path, err := ref.ResolvePath()
 	assert.NoError(t, err)
-	assert.Equal(t, filepath.FromSlash("refs/heads/my-branch"), path)
+	assert.Equal(t, "refs/heads/my-branch", path)
 }
 
 func TestReferenceLoadingForObjectID(t *testing.T) {
@@ -55,7 +56,7 @@ func TestReferenceLoadingForObjectID(t *testing.T) {
 	defer f.Close()
 	f.WriteString(strings.Repeat("e", 40) + "\r\n")
 
-	ref, err := LoadReferenceFile(filepath.Join(tmp, "HEAD"))
+	ref, err := LoadReferenceFile(vfs.MustNew(tmp), "HEAD")
 	assert.NoError(t, err)
 	assert.Equal(t, ReferenceTypeSHA1, ref.Type)
 	assert.Equal(t, strings.Repeat("e", 40), ref.Content)
@@ -68,7 +69,7 @@ func TestReferenceLoadingForReference(t *testing.T) {
 	defer f.Close()
 	f.WriteString("ref: refs/heads/foo\n")
 
-	ref, err := LoadReferenceFile(filepath.Join(tmp, "HEAD"))
+	ref, err := LoadReferenceFile(vfs.MustNew(tmp), "HEAD")
 	assert.NoError(t, err)
 	assert.Equal(t, ReferenceTypePointer, ref.Type)
 	assert.Equal(t, "ref: refs/heads/foo", ref.Content)
@@ -81,7 +82,7 @@ func TestReferenceLoadingFailsForInvalidContent(t *testing.T) {
 	defer f.Close()
 	f.WriteString("abc")
 
-	_, err = LoadReferenceFile(filepath.Join(tmp, "HEAD"))
+	_, err = LoadReferenceFile(vfs.MustNew(tmp), "HEAD")
 	assert.ErrorContains(t, err, "unknown format for git HEAD")
 }
 
diff --git a/libs/git/repository.go b/libs/git/repository.go
index 531fd74e4..6baf26c2e 100644
--- a/libs/git/repository.go
+++ b/libs/git/repository.go
@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/databricks/cli/libs/folders"
+	"github.com/databricks/cli/libs/vfs"
 )
 
 const gitIgnoreFileName = ".gitignore"
@@ -21,8 +21,8 @@ type Repository struct {
 	// directory where we process .gitignore files.
 	real bool
 
-	// rootPath is the absolute path to the repository root.
-	rootPath string
+	// root is the absolute path to the repository root.
+	root vfs.Path
 
 	// ignore contains a list of ignore patterns indexed by the
 	// path prefix relative to the repository root.
@@ -42,12 +42,12 @@ type Repository struct {
 
 // Root returns the absolute path to the repository root.
 func (r *Repository) Root() string {
-	return r.rootPath
+	return r.root.Native()
 }
 
 func (r *Repository) CurrentBranch() (string, error) {
 	// load .git/HEAD
-	ref, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, "HEAD"))
+	ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD"))
 	if err != nil {
 		return "", err
 	}
@@ -64,7 +64,7 @@ func (r *Repository) CurrentBranch() (string, error) {
 
 func (r *Repository) LatestCommit() (string, error) {
 	// load .git/HEAD
-	ref, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, "HEAD"))
+	ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD"))
 	if err != nil {
 		return "", err
 	}
@@ -83,7 +83,7 @@ func (r *Repository) LatestCommit() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	branchHeadRef, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, branchHeadPath))
+	branchHeadRef, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, branchHeadPath))
 	if err != nil {
 		return "", err
 	}
@@ -108,7 +108,7 @@ func (r *Repository) loadConfig() error {
 	if err != nil {
 		return fmt.Errorf("unable to load user specific gitconfig: %w", err)
 	}
-	err = config.loadFile(filepath.Join(r.rootPath, ".git/config"))
+	err = config.loadFile(r.root, ".git/config")
 	if err != nil {
 		return fmt.Errorf("unable to load repository specific gitconfig: %w", err)
 	}
@@ -119,7 +119,7 @@ func (r *Repository) loadConfig() error {
 // newIgnoreFile constructs a new [ignoreRules] implementation backed by
 // a file using the specified path relative to the repository root.
 func (r *Repository) newIgnoreFile(relativeIgnoreFilePath string) ignoreRules {
-	return newIgnoreFile(filepath.Join(r.rootPath, relativeIgnoreFilePath))
+	return newIgnoreFile(r.root, relativeIgnoreFilePath)
 }
 
 // getIgnoreRules returns a slice of [ignoreRules] that apply
@@ -132,7 +132,7 @@ func (r *Repository) getIgnoreRules(prefix string) []ignoreRules {
 		return fs
 	}
 
-	r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(filepath.Join(prefix, gitIgnoreFileName)))
+	r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(path.Join(prefix, gitIgnoreFileName)))
 	return r.ignore[prefix]
 }
 
@@ -149,7 +149,7 @@ func (r *Repository) taintIgnoreRules() {
 // Ignore computes whether to ignore the specified path.
 // The specified path is relative to the repository root path.
 func (r *Repository) Ignore(relPath string) (bool, error) {
-	parts := strings.Split(filepath.ToSlash(relPath), "/")
+	parts := strings.Split(relPath, "/")
 
 	// Retain trailing slash for directory patterns.
 	// We know a trailing slash was present if the last element
@@ -186,14 +186,9 @@ func (r *Repository) Ignore(relPath string) (bool, error) {
 	return false, nil
 }
 
-func NewRepository(path string) (*Repository, error) {
-	path, err := filepath.Abs(path)
-	if err != nil {
-		return nil, err
-	}
-
+func NewRepository(path vfs.Path) (*Repository, error) {
 	real := true
-	rootPath, err := folders.FindDirWithLeaf(path, GitDirectoryName)
+	rootPath, err := vfs.FindLeafInTree(path, GitDirectoryName)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return nil, err
@@ -205,9 +200,9 @@ func NewRepository(path string) (*Repository, error) {
 	}
 
 	repo := &Repository{
-		real:     real,
-		rootPath: rootPath,
-		ignore:   make(map[string][]ignoreRules),
+		real:   real,
+		root:   rootPath,
+		ignore: make(map[string][]ignoreRules),
 	}
 
 	err = repo.loadConfig()
@@ -221,13 +216,21 @@ func NewRepository(path string) (*Repository, error) {
 		return nil, fmt.Errorf("unable to access core excludes file: %w", err)
 	}
 
+	// Load global excludes on this machine.
+	// This is by definition a local path so we create a new [vfs.Path] instance.
+	coreExcludes := newStringIgnoreRules([]string{})
+	if coreExcludesPath != "" {
+		dir := filepath.Dir(coreExcludesPath)
+		base := filepath.Base(coreExcludesPath)
+		coreExcludes = newIgnoreFile(vfs.MustNew(dir), base)
+	}
+
 	// Initialize root ignore rules.
 	// These are special and not lazily initialized because:
 	// 1) we include a hardcoded ignore pattern
 	// 2) we include a gitignore file at a non-standard path
 	repo.ignore["."] = []ignoreRules{
-		// Load global excludes on this machine.
-		newIgnoreFile(coreExcludesPath),
+		coreExcludes,
 		// Always ignore root .git directory.
 		newStringIgnoreRules([]string{
 			".git",
diff --git a/libs/git/repository_test.go b/libs/git/repository_test.go
index fb0e38080..7ddc7ea79 100644
--- a/libs/git/repository_test.go
+++ b/libs/git/repository_test.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -43,7 +44,7 @@ func newTestRepository(t *testing.T) *testRepository {
 	_, err = f2.WriteString(`ref: refs/heads/main`)
 	require.NoError(t, err)
 
-	repo, err := NewRepository(tmp)
+	repo, err := NewRepository(vfs.MustNew(tmp))
 	require.NoError(t, err)
 
 	return &testRepository{
@@ -53,7 +54,7 @@ func newTestRepository(t *testing.T) *testRepository {
 }
 
 func (testRepo *testRepository) checkoutCommit(commitId string) {
-	f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm)
+	f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm)
 	require.NoError(testRepo.t, err)
 	defer f.Close()
 
@@ -63,7 +64,7 @@ func (testRepo *testRepository) checkoutCommit(commitId string) {
 
 func (testRepo *testRepository) addBranch(name string, latestCommit string) {
 	// create dir for branch head reference
-	branchDir := filepath.Join(testRepo.r.rootPath, ".git", "refs", "heads")
+	branchDir := filepath.Join(testRepo.r.Root(), ".git", "refs", "heads")
 	err := os.MkdirAll(branchDir, os.ModePerm)
 	require.NoError(testRepo.t, err)
 
@@ -78,7 +79,7 @@ func (testRepo *testRepository) addBranch(name string, latestCommit string) {
 }
 
 func (testRepo *testRepository) checkoutBranch(name string) {
-	f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm)
+	f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm)
 	require.NoError(testRepo.t, err)
 	defer f.Close()
 
@@ -89,7 +90,7 @@ func (testRepo *testRepository) checkoutBranch(name string) {
 // add remote origin url to test repo
 func (testRepo *testRepository) addOriginUrl(url string) {
 	// open config in append mode
-	f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "config"), os.O_WRONLY|os.O_APPEND, os.ModePerm)
+	f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "config"), os.O_WRONLY|os.O_APPEND, os.ModePerm)
 	require.NoError(testRepo.t, err)
 	defer f.Close()
 
@@ -128,7 +129,7 @@ func (testRepo *testRepository) assertOriginUrl(expected string) {
 
 func TestRepository(t *testing.T) {
 	// Load this repository as test.
-	repo, err := NewRepository("../..")
+	repo, err := NewRepository(vfs.MustNew("../.."))
 	tr := testRepository{t, repo}
 	require.NoError(t, err)
 
@@ -142,7 +143,7 @@ func TestRepository(t *testing.T) {
 	assert.True(t, tr.Ignore("vendor/"))
 
 	// Check that ignores under testdata work.
-	assert.True(t, tr.Ignore(filepath.Join("libs", "git", "testdata", "root.ignoreme")))
+	assert.True(t, tr.Ignore("libs/git/testdata/root.ignoreme"))
 }
 
 func TestRepositoryGitConfigForEmptyRepo(t *testing.T) {
@@ -192,7 +193,7 @@ func TestRepositoryGitConfigForSshUrl(t *testing.T) {
 
 func TestRepositoryGitConfigWhenNotARepo(t *testing.T) {
 	tmp := t.TempDir()
-	repo, err := NewRepository(tmp)
+	repo, err := NewRepository(vfs.MustNew(tmp))
 	require.NoError(t, err)
 
 	branch, err := repo.CurrentBranch()
diff --git a/libs/git/view.go b/libs/git/view.go
index 3cb88d8b1..90eed0bb8 100644
--- a/libs/git/view.go
+++ b/libs/git/view.go
@@ -1,9 +1,13 @@
 package git
 
 import (
+	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
+
+	"github.com/databricks/cli/libs/vfs"
 )
 
 // View represents a view on a directory tree that takes into account
@@ -29,17 +33,15 @@ type View struct {
 
 // Ignore computes whether to ignore the specified path.
 // The specified path is relative to the view's target path.
-func (v *View) Ignore(path string) (bool, error) {
-	path = filepath.ToSlash(path)
-
+func (v *View) Ignore(relPath string) (bool, error) {
 	// Retain trailing slash for directory patterns.
 	// Needs special handling because it is removed by path cleaning.
 	trailingSlash := ""
-	if strings.HasSuffix(path, "/") {
+	if strings.HasSuffix(relPath, "/") {
 		trailingSlash = "/"
 	}
 
-	return v.repo.Ignore(filepath.Join(v.targetPath, path) + trailingSlash)
+	return v.repo.Ignore(path.Join(v.targetPath, relPath) + trailingSlash)
 }
 
 // IgnoreFile returns if the gitignore rules in this fileset
@@ -70,26 +72,27 @@ func (v *View) IgnoreDirectory(dir string) (bool, error) {
 	return v.Ignore(dir + "/")
 }
 
-func NewView(path string) (*View, error) {
-	path, err := filepath.Abs(path)
-	if err != nil {
-		return nil, err
-	}
-
-	repo, err := NewRepository(path)
+func NewView(root vfs.Path) (*View, error) {
+	repo, err := NewRepository(root)
 	if err != nil {
 		return nil, err
 	}
 
 	// Target path must be relative to the repository root path.
-	targetPath, err := filepath.Rel(repo.rootPath, path)
-	if err != nil {
-		return nil, err
+	target := root.Native()
+	prefix := repo.root.Native()
+	if !strings.HasPrefix(target, prefix) {
+		return nil, fmt.Errorf("path %q is not within repository root %q", root.Native(), prefix)
 	}
 
+	// Make target a relative path.
+	target = strings.TrimPrefix(target, prefix)
+	target = strings.TrimPrefix(target, string(os.PathSeparator))
+	target = path.Clean(filepath.ToSlash(target))
+
 	return &View{
 		repo:       repo,
-		targetPath: targetPath,
+		targetPath: target,
 	}, nil
 }
 
diff --git a/libs/git/view_test.go b/libs/git/view_test.go
index 3ecd301b5..76fba3458 100644
--- a/libs/git/view_test.go
+++ b/libs/git/view_test.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -89,19 +90,19 @@ func testViewAtRoot(t *testing.T, tv testView) {
 }
 
 func TestViewRootInBricksRepo(t *testing.T) {
-	v, err := NewView("./testdata")
+	v, err := NewView(vfs.MustNew("./testdata"))
 	require.NoError(t, err)
 	testViewAtRoot(t, testView{t, v})
 }
 
 func TestViewRootInTempRepo(t *testing.T) {
-	v, err := NewView(createFakeRepo(t, "testdata"))
+	v, err := NewView(vfs.MustNew(createFakeRepo(t, "testdata")))
 	require.NoError(t, err)
 	testViewAtRoot(t, testView{t, v})
 }
 
 func TestViewRootInTempDir(t *testing.T) {
-	v, err := NewView(copyTestdata(t, "testdata"))
+	v, err := NewView(vfs.MustNew(copyTestdata(t, "testdata")))
 	require.NoError(t, err)
 	testViewAtRoot(t, testView{t, v})
 }
@@ -124,20 +125,20 @@ func testViewAtA(t *testing.T, tv testView) {
 }
 
 func TestViewAInBricksRepo(t *testing.T) {
-	v, err := NewView("./testdata/a")
+	v, err := NewView(vfs.MustNew("./testdata/a"))
 	require.NoError(t, err)
 	testViewAtA(t, testView{t, v})
 }
 
 func TestViewAInTempRepo(t *testing.T) {
-	v, err := NewView(filepath.Join(createFakeRepo(t, "testdata"), "a"))
+	v, err := NewView(vfs.MustNew(filepath.Join(createFakeRepo(t, "testdata"), "a")))
 	require.NoError(t, err)
 	testViewAtA(t, testView{t, v})
 }
 
 func TestViewAInTempDir(t *testing.T) {
 	// Since this is not a fake repo it should not traverse up the tree.
-	v, err := NewView(filepath.Join(copyTestdata(t, "testdata"), "a"))
+	v, err := NewView(vfs.MustNew(filepath.Join(copyTestdata(t, "testdata"), "a")))
 	require.NoError(t, err)
 	tv := testView{t, v}
 
@@ -174,20 +175,20 @@ func testViewAtAB(t *testing.T, tv testView) {
 }
 
 func TestViewABInBricksRepo(t *testing.T) {
-	v, err := NewView("./testdata/a/b")
+	v, err := NewView(vfs.MustNew("./testdata/a/b"))
 	require.NoError(t, err)
 	testViewAtAB(t, testView{t, v})
 }
 
 func TestViewABInTempRepo(t *testing.T) {
-	v, err := NewView(filepath.Join(createFakeRepo(t, "testdata"), "a", "b"))
+	v, err := NewView(vfs.MustNew(filepath.Join(createFakeRepo(t, "testdata"), "a", "b")))
 	require.NoError(t, err)
 	testViewAtAB(t, testView{t, v})
 }
 
 func TestViewABInTempDir(t *testing.T) {
 	// Since this is not a fake repo it should not traverse up the tree.
-	v, err := NewView(filepath.Join(copyTestdata(t, "testdata"), "a", "b"))
+	v, err := NewView(vfs.MustNew(filepath.Join(copyTestdata(t, "testdata"), "a", "b")))
 	tv := testView{t, v}
 	require.NoError(t, err)
 
@@ -214,7 +215,7 @@ func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredAtRoot(t *testing.T)
 
 	// Since root .gitignore already has .databricks, there should be no edits
 	// to root .gitignore
-	v, err := NewView(repoPath)
+	v, err := NewView(vfs.MustNew(repoPath))
 	require.NoError(t, err)
 
 	err = v.EnsureValidGitIgnoreExists()
@@ -234,7 +235,7 @@ func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredInSubdir(t *testing.T
 
 	// Since root .gitignore already has .databricks, there should be no edits
 	// to a/.gitignore
-	v, err := NewView(filepath.Join(repoPath, "a"))
+	v, err := NewView(vfs.MustNew(filepath.Join(repoPath, "a")))
 	require.NoError(t, err)
 
 	err = v.EnsureValidGitIgnoreExists()
@@ -252,7 +253,7 @@ func TestViewAddsGitignoreWithCacheDir(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Since root .gitignore was deleted, new view adds .databricks to root .gitignore
-	v, err := NewView(repoPath)
+	v, err := NewView(vfs.MustNew(repoPath))
 	require.NoError(t, err)
 
 	err = v.EnsureValidGitIgnoreExists()
@@ -270,7 +271,7 @@ func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) {
 	require.NoError(t, err)
 
 	// Since root .gitignore was deleted, new view adds .databricks to a/.gitignore
-	v, err := NewView(filepath.Join(repoPath, "a"))
+	v, err := NewView(vfs.MustNew(filepath.Join(repoPath, "a")))
 	require.NoError(t, err)
 
 	err = v.EnsureValidGitIgnoreExists()
@@ -287,7 +288,7 @@ func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) {
 func TestViewAlwaysIgnoresCacheDir(t *testing.T) {
 	repoPath := createFakeRepo(t, "testdata")
 
-	v, err := NewView(repoPath)
+	v, err := NewView(vfs.MustNew(repoPath))
 	require.NoError(t, err)
 
 	err = v.EnsureValidGitIgnoreExists()
diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go
index 17685f3bf..0b7c04d6d 100644
--- a/libs/notebook/detect.go
+++ b/libs/notebook/detect.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -15,8 +16,8 @@ import (
 const headerLength = 32
 
 // readHeader reads the first N bytes from a file.
-func readHeader(path string) ([]byte, error) {
-	f, err := os.Open(path)
+func readHeader(fsys fs.FS, name string) ([]byte, error) {
+	f, err := fsys.Open(name)
 	if err != nil {
 		return nil, err
 	}
@@ -36,10 +37,10 @@ func readHeader(path string) ([]byte, error) {
 
 // Detect returns whether the file at path is a Databricks notebook.
 // If it is, it returns the notebook language.
-func Detect(path string) (notebook bool, language workspace.Language, err error) {
+func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) {
 	header := ""
 
-	buf, err := readHeader(path)
+	buf, err := readHeader(fsys, name)
 	if err != nil {
 		return false, "", err
 	}
@@ -48,7 +49,7 @@ func Detect(path string) (notebook bool, language workspace.Language, err error)
 	fileHeader := scanner.Text()
 
 	// Determine which header to expect based on filename extension.
-	ext := strings.ToLower(filepath.Ext(path))
+	ext := strings.ToLower(filepath.Ext(name))
 	switch ext {
 	case ".py":
 		header = `# Databricks notebook source`
@@ -63,7 +64,7 @@ func Detect(path string) (notebook bool, language workspace.Language, err error)
 		header = "-- Databricks notebook source"
 		language = workspace.LanguageSql
 	case ".ipynb":
-		return DetectJupyter(path)
+		return DetectJupyterWithFS(fsys, name)
 	default:
 		return false, "", nil
 	}
@@ -74,3 +75,11 @@ func Detect(path string) (notebook bool, language workspace.Language, err error)
 
 	return true, language, nil
 }
+
+// Detect calls DetectWithFS with the local filesystem.
+// The name argument may be a local relative path or a local absolute path.
+func Detect(name string) (notebook bool, language workspace.Language, err error) {
+	d := filepath.ToSlash(filepath.Dir(name))
+	b := filepath.Base(name)
+	return DetectWithFS(os.DirFS(d), b)
+}
diff --git a/libs/notebook/detect_jupyter.go b/libs/notebook/detect_jupyter.go
index 7d96763cd..f631b5812 100644
--- a/libs/notebook/detect_jupyter.go
+++ b/libs/notebook/detect_jupyter.go
@@ -3,7 +3,9 @@ package notebook
 import (
 	"encoding/json"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
 
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
@@ -56,8 +58,8 @@ func resolveLanguage(nb *jupyter) workspace.Language {
 // DetectJupyter returns whether the file at path is a valid Jupyter notebook.
 // We assume it is valid if we can read it as JSON and see a couple expected fields.
 // If we cannot, importing into the workspace will always fail, so we also return an error.
-func DetectJupyter(path string) (notebook bool, language workspace.Language, err error) {
-	f, err := os.Open(path)
+func DetectJupyterWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) {
+	f, err := fsys.Open(name)
 	if err != nil {
 		return false, "", err
 	}
@@ -68,18 +70,26 @@ func DetectJupyter(path string) (notebook bool, language workspace.Language, err
 	dec := json.NewDecoder(f)
 	err = dec.Decode(&nb)
 	if err != nil {
-		return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", path, err)
+		return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", name, err)
 	}
 
 	// Not a Jupyter notebook if the cells or metadata fields aren't defined.
 	if nb.Cells == nil || nb.Metadata == nil {
-		return false, "", fmt.Errorf("%s: invalid Jupyter notebook file", path)
+		return false, "", fmt.Errorf("%s: invalid Jupyter notebook file", name)
 	}
 
 	// Major version must be at least 4.
 	if nb.NbFormatMajor < 4 {
-		return false, "", fmt.Errorf("%s: unsupported Jupyter notebook version: %d", path, nb.NbFormatMajor)
+		return false, "", fmt.Errorf("%s: unsupported Jupyter notebook version: %d", name, nb.NbFormatMajor)
 	}
 
 	return true, resolveLanguage(&nb), nil
 }
+
+// DetectJupyter calls DetectJupyterWithFS with the local filesystem.
+// The name argument may be a local relative path or a local absolute path.
+func DetectJupyter(name string) (notebook bool, language workspace.Language, err error) {
+	d := filepath.ToSlash(filepath.Dir(name))
+	b := filepath.Base(name)
+	return DetectJupyterWithFS(os.DirFS(d), b)
+}
diff --git a/libs/sync/diff.go b/libs/sync/diff.go
index 074bfc56c..e91f7277e 100644
--- a/libs/sync/diff.go
+++ b/libs/sync/diff.go
@@ -2,7 +2,6 @@ package sync
 
 import (
 	"path"
-	"path/filepath"
 
 	"golang.org/x/exp/maps"
 )
@@ -64,7 +63,7 @@ func (d *diff) addFilesWithRemoteNameChanged(after *SnapshotState, before *Snaps
 func (d *diff) addNewFiles(after *SnapshotState, before *SnapshotState) {
 	for localName := range after.LastModifiedTimes {
 		if _, ok := before.LastModifiedTimes[localName]; !ok {
-			d.put = append(d.put, filepath.ToSlash(localName))
+			d.put = append(d.put, localName)
 		}
 	}
 
@@ -79,7 +78,7 @@ func (d *diff) addUpdatedFiles(after *SnapshotState, before *SnapshotState) {
 	for localName, modTime := range after.LastModifiedTimes {
 		prevModTime, ok := before.LastModifiedTimes[localName]
 		if ok && modTime.After(prevModTime) {
-			d.put = append(d.put, filepath.ToSlash(localName))
+			d.put = append(d.put, localName)
 		}
 	}
 }
diff --git a/libs/sync/dirset.go b/libs/sync/dirset.go
index 3c37c97cf..33b85cb8e 100644
--- a/libs/sync/dirset.go
+++ b/libs/sync/dirset.go
@@ -2,7 +2,6 @@ package sync
 
 import (
 	"path"
-	"path/filepath"
 	"sort"
 )
 
@@ -16,8 +15,8 @@ func MakeDirSet(files []string) DirSet {
 
 	// Iterate over all files.
 	for _, f := range files {
-		// Get the directory of the file in /-separated form.
-		dir := filepath.ToSlash(filepath.Dir(f))
+		// Get the directory of the file.
+		dir := path.Dir(f)
 
 		// Add this directory and its parents until it is either "." or already in the set.
 		for dir != "." {
diff --git a/libs/sync/snapshot.go b/libs/sync/snapshot.go
index a27a8c84f..392e274d4 100644
--- a/libs/sync/snapshot.go
+++ b/libs/sync/snapshot.go
@@ -172,6 +172,11 @@ func loadOrNewSnapshot(ctx context.Context, opts *SyncOptions) (*Snapshot, error
 		return nil, fmt.Errorf("failed to json unmarshal persisted snapshot: %s", err)
 	}
 
+	// Ensure that all paths are slash-separated upon loading
+	// an existing snapshot file. If it was created by an older
+	// CLI version (<= v0.220.0), it may contain backslashes.
+	snapshot.SnapshotState = snapshot.SnapshotState.ToSlash()
+
 	snapshot.New = false
 	return snapshot, nil
 }
diff --git a/libs/sync/snapshot_state.go b/libs/sync/snapshot_state.go
index 10cd34e6d..09bb5b63e 100644
--- a/libs/sync/snapshot_state.go
+++ b/libs/sync/snapshot_state.go
@@ -2,6 +2,7 @@ package sync
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -48,7 +49,7 @@ func NewSnapshotState(localFiles []fileset.File) (*SnapshotState, error) {
 	for k := range localFiles {
 		f := &localFiles[k]
 		// Compute the remote name the file will have in WSFS
-		remoteName := filepath.ToSlash(f.Relative)
+		remoteName := f.Relative
 		isNotebook, err := f.IsNotebook()
 
 		if err != nil {
@@ -57,7 +58,7 @@ func NewSnapshotState(localFiles []fileset.File) (*SnapshotState, error) {
 			continue
 		}
 		if isNotebook {
-			ext := filepath.Ext(remoteName)
+			ext := path.Ext(remoteName)
 			remoteName = strings.TrimSuffix(remoteName, ext)
 		}
 
@@ -119,3 +120,30 @@ func (fs *SnapshotState) validate() error {
 	}
 	return nil
 }
+
+// ToSlash ensures all local paths in the snapshot state
+// are slash-separated. Returns a new snapshot state.
+func (old SnapshotState) ToSlash() *SnapshotState {
+	new := SnapshotState{
+		LastModifiedTimes:  make(map[string]time.Time),
+		LocalToRemoteNames: make(map[string]string),
+		RemoteToLocalNames: make(map[string]string),
+	}
+
+	// Keys are local paths.
+	for k, v := range old.LastModifiedTimes {
+		new.LastModifiedTimes[filepath.ToSlash(k)] = v
+	}
+
+	// Keys are local paths.
+	for k, v := range old.LocalToRemoteNames {
+		new.LocalToRemoteNames[filepath.ToSlash(k)] = v
+	}
+
+	// Values are remote paths.
+	for k, v := range old.RemoteToLocalNames {
+		new.RemoteToLocalNames[k] = filepath.ToSlash(v)
+	}
+
+	return &new
+}
diff --git a/libs/sync/snapshot_state_test.go b/libs/sync/snapshot_state_test.go
index bfcdbef65..92c14e8e0 100644
--- a/libs/sync/snapshot_state_test.go
+++ b/libs/sync/snapshot_state_test.go
@@ -1,25 +1,27 @@
 package sync
 
 import (
+	"runtime"
 	"testing"
 	"time"
 
 	"github.com/databricks/cli/libs/fileset"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestSnapshotState(t *testing.T) {
-	fileSet := fileset.New("./testdata/sync-fileset")
+	fileSet := fileset.New(vfs.MustNew("./testdata/sync-fileset"))
 	files, err := fileSet.All()
 	require.NoError(t, err)
 
 	// Assert initial contents of the fileset
 	assert.Len(t, files, 4)
-	assert.Equal(t, "invalid-nb.ipynb", files[0].Name())
-	assert.Equal(t, "my-nb.py", files[1].Name())
-	assert.Equal(t, "my-script.py", files[2].Name())
-	assert.Equal(t, "valid-nb.ipynb", files[3].Name())
+	assert.Equal(t, "invalid-nb.ipynb", files[0].Relative)
+	assert.Equal(t, "my-nb.py", files[1].Relative)
+	assert.Equal(t, "my-script.py", files[2].Relative)
+	assert.Equal(t, "valid-nb.ipynb", files[3].Relative)
 
 	// Assert snapshot state generated from the fileset. Note that the invalid notebook
 	// has been ignored.
@@ -114,3 +116,30 @@ func TestSnapshotStateValidationErrors(t *testing.T) {
 	}
 	assert.EqualError(t, s.validate(), "invalid sync state representation. Inconsistent values found. Remote file c points to a. Local file a points to b")
 }
+
+func TestSnapshotStateWithBackslashes(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skip("Skipping test on non-Windows platform")
+	}
+
+	now := time.Now()
+	s1 := &SnapshotState{
+		LastModifiedTimes: map[string]time.Time{
+			"foo\\bar.py": now,
+		},
+		LocalToRemoteNames: map[string]string{
+			"foo\\bar.py": "foo/bar",
+		},
+		RemoteToLocalNames: map[string]string{
+			"foo/bar": "foo\\bar.py",
+		},
+	}
+
+	assert.NoError(t, s1.validate())
+
+	s2 := s1.ToSlash()
+	assert.NoError(t, s1.validate())
+	assert.Equal(t, map[string]time.Time{"foo/bar.py": now}, s2.LastModifiedTimes)
+	assert.Equal(t, map[string]string{"foo/bar.py": "foo/bar"}, s2.LocalToRemoteNames)
+	assert.Equal(t, map[string]string{"foo/bar": "foo/bar.py"}, s2.RemoteToLocalNames)
+}
diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go
index d6358d4a1..050b5d965 100644
--- a/libs/sync/snapshot_test.go
+++ b/libs/sync/snapshot_test.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/testfile"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -29,7 +30,7 @@ func TestDiff(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
@@ -93,7 +94,7 @@ func TestSymlinkDiff(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
@@ -124,7 +125,7 @@ func TestFolderDiff(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
@@ -169,7 +170,7 @@ func TestPythonNotebookDiff(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
@@ -244,7 +245,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
@@ -281,7 +282,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) {
 
 	// Create temp project dir
 	projectDir := t.TempDir()
-	fileSet, err := git.NewFileSet(projectDir)
+	fileSet, err := git.NewFileSet(vfs.MustNew(projectDir))
 	require.NoError(t, err)
 	state := Snapshot{
 		SnapshotState: &SnapshotState{
diff --git a/libs/sync/sync.go b/libs/sync/sync.go
index 30b68ccf3..585e8a887 100644
--- a/libs/sync/sync.go
+++ b/libs/sync/sync.go
@@ -10,12 +10,13 @@ import (
 	"github.com/databricks/cli/libs/git"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/set"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/iam"
 )
 
 type SyncOptions struct {
-	LocalPath  string
+	LocalPath  vfs.Path
 	RemotePath string
 	Include    []string
 	Exclude    []string
@@ -54,6 +55,7 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	err = fileSet.EnsureValidGitIgnoreExists()
 	if err != nil {
 		return nil, err
@@ -186,7 +188,7 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) {
 	// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
 	// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
 	all := set.NewSetF(func(f fileset.File) string {
-		return f.Absolute
+		return f.Relative
 	})
 	gitFiles, err := s.fileSet.All()
 	if err != nil {
diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go
index dc220dbf7..292586e8d 100644
--- a/libs/sync/sync_test.go
+++ b/libs/sync/sync_test.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/databricks/cli/libs/fileset"
 	"github.com/databricks/cli/libs/git"
+	"github.com/databricks/cli/libs/vfs"
 	"github.com/stretchr/testify/require"
 )
 
@@ -73,16 +74,17 @@ func TestGetFileSet(t *testing.T) {
 	ctx := context.Background()
 
 	dir := setupFiles(t)
-	fileSet, err := git.NewFileSet(dir)
+	root := vfs.MustNew(dir)
+	fileSet, err := git.NewFileSet(root)
 	require.NoError(t, err)
 
 	err = fileSet.EnsureValidGitIgnoreExists()
 	require.NoError(t, err)
 
-	inc, err := fileset.NewGlobSet(dir, []string{})
+	inc, err := fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
 
-	excl, err := fileset.NewGlobSet(dir, []string{})
+	excl, err := fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
 
 	s := &Sync{
@@ -97,10 +99,10 @@ func TestGetFileSet(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, len(fileList), 9)
 
-	inc, err = fileset.NewGlobSet(dir, []string{})
+	inc, err = fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
 
-	excl, err = fileset.NewGlobSet(dir, []string{"*.go"})
+	excl, err = fileset.NewGlobSet(root, []string{"*.go"})
 	require.NoError(t, err)
 
 	s = &Sync{
@@ -115,10 +117,10 @@ func TestGetFileSet(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, len(fileList), 1)
 
-	inc, err = fileset.NewGlobSet(dir, []string{".databricks/*"})
+	inc, err = fileset.NewGlobSet(root, []string{".databricks/*"})
 	require.NoError(t, err)
 
-	excl, err = fileset.NewGlobSet(dir, []string{})
+	excl, err = fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
 
 	s = &Sync{
@@ -138,16 +140,17 @@ func TestRecursiveExclude(t *testing.T) {
 	ctx := context.Background()
 
 	dir := setupFiles(t)
-	fileSet, err := git.NewFileSet(dir)
+	root := vfs.MustNew(dir)
+	fileSet, err := git.NewFileSet(root)
 	require.NoError(t, err)
 
 	err = fileSet.EnsureValidGitIgnoreExists()
 	require.NoError(t, err)
 
-	inc, err := fileset.NewGlobSet(dir, []string{})
+	inc, err := fileset.NewGlobSet(root, []string{})
 	require.NoError(t, err)
 
-	excl, err := fileset.NewGlobSet(dir, []string{"test/**"})
+	excl, err := fileset.NewGlobSet(root, []string{"test/**"})
 	require.NoError(t, err)
 
 	s := &Sync{
diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go
index b0c96e01c..ca7ec46e9 100644
--- a/libs/sync/watchdog.go
+++ b/libs/sync/watchdog.go
@@ -4,8 +4,6 @@ import (
 	"context"
 	"errors"
 	"io/fs"
-	"os"
-	"path/filepath"
 
 	"github.com/databricks/cli/libs/filer"
 	"github.com/databricks/cli/libs/log"
@@ -59,7 +57,7 @@ func (s *Sync) applyMkdir(ctx context.Context, localName string) error {
 func (s *Sync) applyPut(ctx context.Context, localName string) error {
 	s.notifyProgress(ctx, EventActionPut, localName, 0.0)
 
-	localFile, err := os.Open(filepath.Join(s.LocalPath, localName))
+	localFile, err := s.LocalPath.Open(localName)
 	if err != nil {
 		return err
 	}
diff --git a/libs/vfs/leaf.go b/libs/vfs/leaf.go
new file mode 100644
index 000000000..8c11f9039
--- /dev/null
+++ b/libs/vfs/leaf.go
@@ -0,0 +1,29 @@
+package vfs
+
+import (
+	"errors"
+	"io/fs"
+)
+
+// FindLeafInTree returns the first path that holds `name`,
+// traversing up to the root of the filesystem, starting at `p`.
+func FindLeafInTree(p Path, name string) (Path, error) {
+	for p != nil {
+		_, err := fs.Stat(p, name)
+
+		// No error means we found the leaf in p.
+		if err == nil {
+			return p, nil
+		}
+
+		// ErrNotExist means we continue traversal up the tree.
+		if errors.Is(err, fs.ErrNotExist) {
+			p = p.Parent()
+			continue
+		}
+
+		return nil, err
+	}
+
+	return nil, fs.ErrNotExist
+}
diff --git a/libs/vfs/leaf_test.go b/libs/vfs/leaf_test.go
new file mode 100644
index 000000000..da9412ec0
--- /dev/null
+++ b/libs/vfs/leaf_test.go
@@ -0,0 +1,38 @@
+package vfs
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindLeafInTree(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	root := filepath.Join(wd, "..", "..")
+
+	// Find from working directory should work.
+	{
+		out, err := FindLeafInTree(MustNew(wd), ".git")
+		assert.NoError(t, err)
+		assert.Equal(t, root, out.Native())
+	}
+
+	// Find from project root itself should work.
+	{
+		out, err := FindLeafInTree(MustNew(root), ".git")
+		assert.NoError(t, err)
+		assert.Equal(t, root, out.Native())
+	}
+
+	// Find for something that doesn't exist should work.
+	{
+		out, err := FindLeafInTree(MustNew(root), "this-leaf-doesnt-exist-anywhere")
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		assert.Equal(t, nil, out)
+	}
+}
diff --git a/libs/vfs/os.go b/libs/vfs/os.go
new file mode 100644
index 000000000..26447d830
--- /dev/null
+++ b/libs/vfs/os.go
@@ -0,0 +1,82 @@
+package vfs
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type osPath struct {
+	path string
+
+	openFn     func(name string) (fs.File, error)
+	statFn     func(name string) (fs.FileInfo, error)
+	readDirFn  func(name string) ([]fs.DirEntry, error)
+	readFileFn func(name string) ([]byte, error)
+}
+
+func New(name string) (Path, error) {
+	abs, err := filepath.Abs(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return newOsPath(abs), nil
+}
+
+func MustNew(name string) Path {
+	p, err := New(name)
+	if err != nil {
+		panic(err)
+	}
+
+	return p
+}
+
+func newOsPath(name string) Path {
+	if !filepath.IsAbs(name) {
+		panic("vfs: abs path must be absolute")
+	}
+
+	// [os.DirFS] implements all required interfaces.
+	// We used type assertion below to get the underlying types.
+	dirfs := os.DirFS(name)
+
+	return &osPath{
+		path: name,
+
+		openFn:     dirfs.Open,
+		statFn:     dirfs.(fs.StatFS).Stat,
+		readDirFn:  dirfs.(fs.ReadDirFS).ReadDir,
+		readFileFn: dirfs.(fs.ReadFileFS).ReadFile,
+	}
+}
+
+func (o osPath) Open(name string) (fs.File, error) {
+	return o.openFn(name)
+}
+
+func (o osPath) Stat(name string) (fs.FileInfo, error) {
+	return o.statFn(name)
+}
+
+func (o osPath) ReadDir(name string) ([]fs.DirEntry, error) {
+	return o.readDirFn(name)
+}
+
+func (o osPath) ReadFile(name string) ([]byte, error) {
+	return o.readFileFn(name)
+}
+
+func (o osPath) Parent() Path {
+	dir := filepath.Dir(o.path)
+	if dir == o.path {
+		return nil
+	}
+
+	return newOsPath(dir)
+}
+
+func (o osPath) Native() string {
+	return o.path
+}
diff --git a/libs/vfs/os_test.go b/libs/vfs/os_test.go
new file mode 100644
index 000000000..6199bdc71
--- /dev/null
+++ b/libs/vfs/os_test.go
@@ -0,0 +1,54 @@
+package vfs
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOsNewWithRelativePath(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	p, err := New(".")
+	require.NoError(t, err)
+	require.Equal(t, wd, p.Native())
+}
+
+func TestOsPathParent(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	p := MustNew(wd)
+	require.NotNil(t, p)
+
+	// Traverse all the way to the root.
+	for {
+		q := p.Parent()
+		if q == nil {
+			// Parent returns nil when it is the root.
+			break
+		}
+
+		p = q
+	}
+
+	// We should have reached the root.
+	if runtime.GOOS == "windows" {
+		require.Equal(t, filepath.VolumeName(wd)+`\`, p.Native())
+	} else {
+		require.Equal(t, "/", p.Native())
+	}
+}
+
+func TestOsPathNative(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	p := MustNew(wd)
+	require.NotNil(t, p)
+	require.Equal(t, wd, p.Native())
+}
diff --git a/libs/vfs/path.go b/libs/vfs/path.go
new file mode 100644
index 000000000..19f119d50
--- /dev/null
+++ b/libs/vfs/path.go
@@ -0,0 +1,29 @@
+package vfs
+
+import "io/fs"
+
+// FS combines the fs.FS, fs.StatFS, fs.ReadDirFS, and fs.ReadFileFS interfaces.
+// It mandates that Path implementations must support all these interfaces.
+type FS interface {
+	fs.FS
+	fs.StatFS
+	fs.ReadDirFS
+	fs.ReadFileFS
+}
+
+// Path defines a read-only virtual file system interface for:
+//
+// 1. Intercepting file operations to inject custom logic (e.g., logging, access control).
+// 2. Traversing directories to find specific leaf directories (e.g., .git).
+// 3. Converting virtual paths to OS-native paths.
+//
+// Options 2 and 3 are not possible with the standard fs.FS interface.
+// They are needed such that we can provide an instance to the sync package
+// and still detect the containing .git directory and convert paths to native paths.
+type Path interface {
+	FS
+
+	Parent() Path
+
+	Native() string
+}
diff --git a/libs/vfs/path_test.go b/libs/vfs/path_test.go
new file mode 100644
index 000000000..54c60940e
--- /dev/null
+++ b/libs/vfs/path_test.go
@@ -0,0 +1 @@
+package vfs

From ec33a7c059602d5a0023625d70209b5501dbb2a6 Mon Sep 17 00:00:00 2001
From: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com>
Date: Thu, 30 May 2024 17:29:27 +0530
Subject: [PATCH 39/41] Add `filer.Filer` to read notebooks from WSFS without
 omitting their extension (#1457)

## Changes
This PR adds a filer that'll allow us to read notebooks from the WSFS
using their full paths (with the extension included). The filer relies
on the existing workspace filer (and consequently the workspace
import/export/list APIs).

Using this filer along with a virtual filesystem layer
(https://github.com/databricks/cli/pull/1452/files) will allow us to use
our custom implementation (which preserves the notebook extensions)
rather than the default mount available via DBR when the CLI is run from
DBR.

## Tests
Integration tests.

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
---
 internal/filer_test.go                        | 346 ++++++++++++++++++
 internal/helpers.go                           |  11 +
 .../workspace_files_extensions_client.go      | 345 +++++++++++++++++
 3 files changed, 702 insertions(+)
 create mode 100644 libs/filer/workspace_files_extensions_client.go

diff --git a/internal/filer_test.go b/internal/filer_test.go
index d333a1b70..3361de5bc 100644
--- a/internal/filer_test.go
+++ b/internal/filer_test.go
@@ -3,9 +3,12 @@ package internal
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/fs"
+	"path"
 	"regexp"
 	"strings"
 	"testing"
@@ -37,6 +40,36 @@ func (f filerTest) assertContents(ctx context.Context, name string, contents str
 	assert.Equal(f, contents, body.String())
 }
 
+func (f filerTest) assertContentsJupyter(ctx context.Context, name string) {
+	reader, err := f.Read(ctx, name)
+	if !assert.NoError(f, err) {
+		return
+	}
+
+	defer reader.Close()
+
+	var body bytes.Buffer
+	_, err = io.Copy(&body, reader)
+	if !assert.NoError(f, err) {
+		return
+	}
+
+	var actual map[string]any
+	err = json.Unmarshal(body.Bytes(), &actual)
+	if !assert.NoError(f, err) {
+		return
+	}
+
+	// Since a roundtrip to the workspace changes a Jupyter notebook's payload,
+	// the best we can do is assert that the nbformat is correct.
+	assert.EqualValues(f, 4, actual["nbformat"])
+}
+
+func (f filerTest) assertNotExists(ctx context.Context, name string) {
+	_, err := f.Stat(ctx, name)
+	assert.ErrorIs(f, err, fs.ErrNotExist)
+}
+
 func commonFilerRecursiveDeleteTest(t *testing.T, ctx context.Context, f filer.Filer) {
 	var err error
 
@@ -94,6 +127,7 @@ func TestAccFilerRecursiveDelete(t *testing.T) {
 		{"workspace files", setupWsfsFiler},
 		{"dbfs", setupDbfsFiler},
 		{"files", setupUcVolumesFiler},
+		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
 		tc := testCase
 
@@ -204,6 +238,7 @@ func TestAccFilerReadWrite(t *testing.T) {
 		{"workspace files", setupWsfsFiler},
 		{"dbfs", setupDbfsFiler},
 		{"files", setupUcVolumesFiler},
+		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
 		tc := testCase
 
@@ -312,6 +347,7 @@ func TestAccFilerReadDir(t *testing.T) {
 		{"workspace files", setupWsfsFiler},
 		{"dbfs", setupDbfsFiler},
 		{"files", setupUcVolumesFiler},
+		{"workspace files extensions", setupWsfsExtensionsFiler},
 	} {
 		tc := testCase
 
@@ -374,6 +410,8 @@ var jupyterNotebookContent2 = `
 `
 
 func TestAccFilerWorkspaceNotebookConflict(t *testing.T) {
+	t.Parallel()
+
 	f, _ := setupWsfsFiler(t)
 	ctx := context.Background()
 	var err error
@@ -420,6 +458,8 @@ func TestAccFilerWorkspaceNotebookConflict(t *testing.T) {
 }
 
 func TestAccFilerWorkspaceNotebookWithOverwriteFlag(t *testing.T) {
+	t.Parallel()
+
 	f, _ := setupWsfsFiler(t)
 	ctx := context.Background()
 	var err error
@@ -462,3 +502,309 @@ func TestAccFilerWorkspaceNotebookWithOverwriteFlag(t *testing.T) {
 	filerTest{t, f}.assertContents(ctx, "scalaNb", "// Databricks notebook source\n println(\"second upload\"))")
 	filerTest{t, f}.assertContents(ctx, "jupyterNb", "# Databricks notebook source\nprint(\"Jupyter Notebook Version 2\")")
 }
+
+func TestAccFilerWorkspaceFilesExtensionsReadDir(t *testing.T) {
+	t.Parallel()
+
+	files := []struct {
+		name    string
+		content string
+	}{
+		{"dir1/dir2/dir3/file.txt", "file content"},
+		{"foo.py", "print('foo')"},
+		{"foo.r", "print('foo')"},
+		{"foo.scala", "println('foo')"},
+		{"foo.sql", "SELECT 'foo'"},
+		{"jupyterNb.ipynb", jupyterNotebookContent1},
+		{"jupyterNb2.ipynb", jupyterNotebookContent2},
+		{"pyNb.py", "# Databricks notebook source\nprint('first upload'))"},
+		{"rNb.r", "# Databricks notebook source\nprint('first upload'))"},
+		{"scalaNb.scala", "// Databricks notebook source\n println(\"first upload\"))"},
+		{"sqlNb.sql", "-- Databricks notebook source\n SELECT \"first upload\""},
+	}
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	for _, f := range files {
+		err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories)
+		require.NoError(t, err)
+	}
+
+	// Read entries
+	entries, err := wf.ReadDir(ctx, ".")
+	require.NoError(t, err)
+	assert.Len(t, entries, len(files))
+	names := []string{}
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	assert.Equal(t, []string{
+		"dir1",
+		"foo.py",
+		"foo.r",
+		"foo.scala",
+		"foo.sql",
+		"jupyterNb.ipynb",
+		"jupyterNb2.ipynb",
+		"pyNb.py",
+		"rNb.r",
+		"scalaNb.scala",
+		"sqlNb.sql",
+	}, names)
+}
+
+func setupFilerWithExtensionsTest(t *testing.T) filer.Filer {
+	files := []struct {
+		name    string
+		content string
+	}{
+		{"foo.py", "# Databricks notebook source\nprint('first upload'))"},
+		{"bar.py", "print('foo')"},
+		{"jupyter.ipynb", jupyterNotebookContent1},
+		{"pretender", "not a notebook"},
+		{"dir/file.txt", "file content"},
+		{"scala-notebook.scala", "// Databricks notebook source\nprintln('first upload')"},
+	}
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	for _, f := range files {
+		err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories)
+		require.NoError(t, err)
+	}
+
+	return wf
+}
+
+func TestAccFilerWorkspaceFilesExtensionsRead(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf := setupFilerWithExtensionsTest(t)
+
+	// Read contents of test fixtures as a sanity check.
+	filerTest{t, wf}.assertContents(ctx, "foo.py", "# Databricks notebook source\nprint('first upload'))")
+	filerTest{t, wf}.assertContents(ctx, "bar.py", "print('foo')")
+	filerTest{t, wf}.assertContentsJupyter(ctx, "jupyter.ipynb")
+	filerTest{t, wf}.assertContents(ctx, "dir/file.txt", "file content")
+	filerTest{t, wf}.assertContents(ctx, "scala-notebook.scala", "// Databricks notebook source\nprintln('first upload')")
+	filerTest{t, wf}.assertContents(ctx, "pretender", "not a notebook")
+
+	// Read non-existent file
+	_, err := wf.Read(ctx, "non-existent.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Ensure we do not read a regular file as a notebook
+	_, err = wf.Read(ctx, "pretender.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+	_, err = wf.Read(ctx, "pretender.ipynb")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Read directory
+	_, err = wf.Read(ctx, "dir")
+	assert.ErrorIs(t, err, fs.ErrInvalid)
+
+	// Ensure we do not read a Scala notebook as a Python notebook
+	_, err = wf.Read(ctx, "scala-notebook.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestAccFilerWorkspaceFilesExtensionsDelete(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf := setupFilerWithExtensionsTest(t)
+
+	// Delete notebook
+	err := wf.Delete(ctx, "foo.py")
+	require.NoError(t, err)
+	filerTest{t, wf}.assertNotExists(ctx, "foo.py")
+
+	// Delete file
+	err = wf.Delete(ctx, "bar.py")
+	require.NoError(t, err)
+	filerTest{t, wf}.assertNotExists(ctx, "bar.py")
+
+	// Delete jupyter notebook
+	err = wf.Delete(ctx, "jupyter.ipynb")
+	require.NoError(t, err)
+	filerTest{t, wf}.assertNotExists(ctx, "jupyter.ipynb")
+
+	// Delete non-existent file
+	err = wf.Delete(ctx, "non-existent.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Ensure we do not delete a file as a notebook
+	err = wf.Delete(ctx, "pretender.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Ensure we do not delete a Scala notebook as a Python notebook
+	_, err = wf.Read(ctx, "scala-notebook.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Delete directory
+	err = wf.Delete(ctx, "dir")
+	assert.ErrorIs(t, err, fs.ErrInvalid)
+
+	// Delete directory recursively
+	err = wf.Delete(ctx, "dir", filer.DeleteRecursively)
+	require.NoError(t, err)
+	filerTest{t, wf}.assertNotExists(ctx, "dir")
+}
+
+func TestAccFilerWorkspaceFilesExtensionsStat(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf := setupFilerWithExtensionsTest(t)
+
+	// Stat on a notebook
+	info, err := wf.Stat(ctx, "foo.py")
+	require.NoError(t, err)
+	assert.Equal(t, "foo.py", info.Name())
+	assert.False(t, info.IsDir())
+
+	// Stat on a file
+	info, err = wf.Stat(ctx, "bar.py")
+	require.NoError(t, err)
+	assert.Equal(t, "bar.py", info.Name())
+	assert.False(t, info.IsDir())
+
+	// Stat on a Jupyter notebook
+	info, err = wf.Stat(ctx, "jupyter.ipynb")
+	require.NoError(t, err)
+	assert.Equal(t, "jupyter.ipynb", info.Name())
+	assert.False(t, info.IsDir())
+
+	// Stat on a directory
+	info, err = wf.Stat(ctx, "dir")
+	require.NoError(t, err)
+	assert.Equal(t, "dir", info.Name())
+	assert.True(t, info.IsDir())
+
+	// Stat on a non-existent file
+	_, err = wf.Stat(ctx, "non-existent.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Ensure we do not stat a file as a notebook
+	_, err = wf.Stat(ctx, "pretender.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Ensure we do not stat a Scala notebook as a Python notebook
+	_, err = wf.Stat(ctx, "scala-notebook.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	_, err = wf.Stat(ctx, "pretender.ipynb")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) {
+	t.Parallel()
+
+	tcases := []struct {
+		files []struct{ name, content string }
+		name  string
+	}{
+		{
+			name: "python",
+			files: []struct{ name, content string }{
+				{"foo.py", "print('foo')"},
+				{"foo.py", "# Databricks notebook source\nprint('foo')"},
+			},
+		},
+		{
+			name: "r",
+			files: []struct{ name, content string }{
+				{"foo.r", "print('foo')"},
+				{"foo.r", "# Databricks notebook source\nprint('foo')"},
+			},
+		},
+		{
+			name: "sql",
+			files: []struct{ name, content string }{
+				{"foo.sql", "SELECT 'foo'"},
+				{"foo.sql", "-- Databricks notebook source\nSELECT 'foo'"},
+			},
+		},
+		{
+			name: "scala",
+			files: []struct{ name, content string }{
+				{"foo.scala", "println('foo')"},
+				{"foo.scala", "// Databricks notebook source\nprintln('foo')"},
+			},
+		},
+		// We don't need to test this for ipynb notebooks. The import API
+		// fails when the file extension is .ipynb but the content is not a
+		// valid juptyer notebook.
+	}
+
+	for i := range tcases {
+		tc := tcases[i]
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			wf, tmpDir := setupWsfsExtensionsFiler(t)
+
+			for _, f := range tc.files {
+				err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories)
+				require.NoError(t, err)
+			}
+
+			_, err := wf.ReadDir(ctx, ".")
+			assert.ErrorAs(t, err, &filer.DuplicatePathError{})
+			assert.ErrorContains(t, err, fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at %s and FILE at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", path.Join(tmpDir, "foo"), path.Join(tmpDir, tc.files[0].name), tc.files[0].name))
+		})
+	}
+
+}
+
+func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Create a directory with an extension
+	err := wf.Mkdir(ctx, "foo")
+	require.NoError(t, err)
+
+	// Reading foo.py should fail. foo is a directory, not a notebook.
+	_, err = wf.Read(ctx, "foo.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
+
+func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	wf, _ := setupWsfsExtensionsFiler(t)
+
+	// Case 1: Source Notebook
+	err := wf.Write(ctx, "foo.py", strings.NewReader("# Databricks notebook source\nprint('foo')"))
+	require.NoError(t, err)
+
+	// The source notebook should exist but not the Jupyter notebook
+	filerTest{t, wf}.assertContents(ctx, "foo.py", "# Databricks notebook source\nprint('foo')")
+	_, err = wf.Stat(ctx, "foo.ipynb")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+	_, err = wf.Read(ctx, "foo.ipynb")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+	err = wf.Delete(ctx, "foo.ipynb")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+
+	// Case 2: Jupyter Notebook
+	err = wf.Write(ctx, "bar.ipynb", strings.NewReader(jupyterNotebookContent1))
+	require.NoError(t, err)
+
+	// The Jupyter notebook should exist but not the source notebook
+	filerTest{t, wf}.assertContentsJupyter(ctx, "bar.ipynb")
+	_, err = wf.Stat(ctx, "bar.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+	_, err = wf.Read(ctx, "bar.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+	err = wf.Delete(ctx, "bar.py")
+	assert.ErrorIs(t, err, fs.ErrNotExist)
+}
diff --git a/internal/helpers.go b/internal/helpers.go
index 49dc9f4ca..3923e7e1e 100644
--- a/internal/helpers.go
+++ b/internal/helpers.go
@@ -559,6 +559,17 @@ func setupWsfsFiler(t *testing.T) (filer.Filer, string) {
 	return f, tmpdir
 }
 
+func setupWsfsExtensionsFiler(t *testing.T) (filer.Filer, string) {
+	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	w := databricks.Must(databricks.NewWorkspaceClient())
+	tmpdir := TemporaryWorkspaceDir(t, w)
+	f, err := filer.NewWorkspaceFilesExtensionsClient(w, tmpdir)
+	require.NoError(t, err)
+
+	return f, tmpdir
+}
+
 func setupDbfsFiler(t *testing.T) (filer.Filer, string) {
 	t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
 
diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go
new file mode 100644
index 000000000..bad748b10
--- /dev/null
+++ b/libs/filer/workspace_files_extensions_client.go
@@ -0,0 +1,345 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/notebook"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/client"
+	"github.com/databricks/databricks-sdk-go/marshal"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+)
+
+type workspaceFilesExtensionsClient struct {
+	workspaceClient *databricks.WorkspaceClient
+	apiClient       *client.DatabricksClient
+
+	wsfs Filer
+	root string
+}
+
+var extensionsToLanguages = map[string]workspace.Language{
+	".py":    workspace.LanguagePython,
+	".r":     workspace.LanguageR,
+	".scala": workspace.LanguageScala,
+	".sql":   workspace.LanguageSql,
+	".ipynb": workspace.LanguagePython,
+}
+
+// workspaceFileStatus defines a custom response body for the "/api/2.0/workspace/get-status" API.
+// The "repos_export_format" field is not exposed by the SDK.
+type workspaceFileStatus struct {
+	*workspace.ObjectInfo
+
+	// The export format of the notebook. This is not exposed by the SDK.
+	ReposExportFormat workspace.ExportFormat `json:"repos_export_format,omitempty"`
+
+	// Name of the file to be used in any API calls made using the workspace files
+	// filer. For notebooks this path does not include the extension.
+	nameForWorkspaceAPI string
+}
+
+// A custom unmarsaller for the workspaceFileStatus struct. This is needed because
+// workspaceFileStatus embeds the workspace.ObjectInfo which itself has a custom
+// unmarshaller.
+// If a custom unmarshaller is not provided extra fields like ReposExportFormat
+// will not have values set.
+func (s *workspaceFileStatus) UnmarshalJSON(b []byte) error {
+	return marshal.Unmarshal(b, s)
+}
+
+func (s *workspaceFileStatus) MarshalJSON() ([]byte, error) {
+	return marshal.Marshal(s)
+}
+
+func (w *workspaceFilesExtensionsClient) stat(ctx context.Context, name string) (*workspaceFileStatus, error) {
+	stat := &workspaceFileStatus{
+		nameForWorkspaceAPI: name,
+	}
+
+	// Perform bespoke API call because "return_export_info" is not exposed by the SDK.
+	// We need "repos_export_format" to determine if the file is a py or a ipynb notebook.
+	// This is not exposed by the SDK so we need to make a direct API call.
+	err := w.apiClient.Do(
+		ctx,
+		http.MethodGet,
+		"/api/2.0/workspace/get-status",
+		nil,
+		map[string]string{
+			"path":               path.Join(w.root, name),
+			"return_export_info": "true",
+		},
+		stat,
+	)
+	if err != nil {
+		// If we got an API error we deal with it below.
+		var aerr *apierr.APIError
+		if !errors.As(err, &aerr) {
+			return nil, err
+		}
+
+		// This API returns a 404 if the specified path does not exist.
+		if aerr.StatusCode == http.StatusNotFound {
+			return nil, FileDoesNotExistError{path.Join(w.root, name)}
+		}
+	}
+	return stat, err
+}
+
+// This function returns the stat for the provided notebook. The stat object itself contains the path
+// with the extension since it is meant to be used in the context of a fs.FileInfo.
+func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
+	ext := path.Ext(name)
+	nameWithoutExt := strings.TrimSuffix(name, ext)
+
+	// File name does not have an extension associated with Databricks notebooks, return early.
+	if _, ok := extensionsToLanguages[ext]; !ok {
+		return nil, nil
+	}
+
+	// If the file could be a notebook, check if it is and has the correct language.
+	stat, err := w.stat(ctx, nameWithoutExt)
+	if err != nil {
+		// If the file does not exist, return early.
+		if errors.As(err, &FileDoesNotExistError{}) {
+			return nil, nil
+		}
+		log.Debugf(ctx, "attempting to determine if %s could be a notebook. Failed to fetch the status of object at %s: %s", name, path.Join(w.root, nameWithoutExt), err)
+		return nil, err
+	}
+
+	// Not a notebook. Return early.
+	if stat.ObjectType != workspace.ObjectTypeNotebook {
+		log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found an object at %s but it is not a notebook. It is a %s.", name, path.Join(w.root, nameWithoutExt), stat.ObjectType)
+		return nil, nil
+	}
+
+	// Not the correct language. Return early.
+	if stat.Language != extensionsToLanguages[ext] {
+		log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not of the correct language. Expected %s but found %s.", name, path.Join(w.root, nameWithoutExt), extensionsToLanguages[ext], stat.Language)
+		return nil, nil
+	}
+
+	// When the extension is .py we expect the export format to be source.
+	// If it's not, return early.
+	if ext == ".py" && stat.ReposExportFormat != workspace.ExportFormatSource {
+		log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not exported as a source notebook. Its export format is %s.", name, path.Join(w.root, nameWithoutExt), stat.ReposExportFormat)
+		return nil, nil
+	}
+
+	// When the extension is .ipynb we expect the export format to be Jupyter.
+	// If it's not, return early.
+	if ext == ".ipynb" && stat.ReposExportFormat != workspace.ExportFormatJupyter {
+		log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not exported as a Jupyter notebook. Its export format is %s.", name, path.Join(w.root, nameWithoutExt), stat.ReposExportFormat)
+		return nil, nil
+	}
+
+	// Modify the stat object path to include the extension. This stat object will be used
+	// to return the fs.FileInfo object in the stat method.
+	stat.Path = stat.Path + ext
+	return stat, nil
+}
+
+func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx context.Context, name string) (*workspaceFileStatus, error) {
+	stat, err := w.stat(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	// We expect this internal function to only be called from [ReadDir] when we are sure
+	// that the object is a notebook. Thus, this should never happen.
+	if stat.ObjectType != workspace.ObjectTypeNotebook {
+		return nil, fmt.Errorf("expected object at %s to be a notebook but it is a %s", path.Join(w.root, name), stat.ObjectType)
+	}
+
+	// Get the extension for the notebook.
+	ext := notebook.GetExtensionByLanguage(stat.ObjectInfo)
+
+	// If the notebook was exported as a Jupyter notebook, the extension should be .ipynb.
+	if stat.Language == workspace.LanguagePython && stat.ReposExportFormat == workspace.ExportFormatJupyter {
+		ext = ".ipynb"
+	}
+
+	// Modify the stat object path to include the extension. This stat object will be used
+	// to return the fs.DirEntry object in the ReadDir method.
+	stat.Path = stat.Path + ext
+	return stat, nil
+}
+
+type DuplicatePathError struct {
+	oi1 workspace.ObjectInfo
+	oi2 workspace.ObjectInfo
+
+	commonName string
+}
+
+func (e DuplicatePathError) Error() string {
+	return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName)
+}
+
+// This is a filer for the workspace file system that allows you to pretend the
+// workspace file system is a traditional file system. It allows you to list, read, write,
+// delete, and stat notebooks (and files in general) in the workspace, using their paths
+// with the extension included.
+//
+// The ReadDir method returns a DuplicatePathError if this traditional file system view is
+// not possible. For example, a Python notebook called foo and a Python file called `foo.py`
+// would resolve to the same path `foo.py` in a tradition file system.
+//
+// Users of this filer should be careful when using the Write and Mkdir methods.
+// The underlying import API we use to upload notebooks and files returns opaque internal
+// errors for namespace clashes (e.g. a file and a notebook or a directory and a notebook).
+// Thus users of these methods should be careful to avoid such clashes.
+func NewWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) {
+	apiClient, err := client.New(w.Config)
+	if err != nil {
+		return nil, err
+	}
+
+	filer, err := NewWorkspaceFilesClient(w, root)
+	if err != nil {
+		return nil, err
+	}
+
+	return &workspaceFilesExtensionsClient{
+		workspaceClient: w,
+		apiClient:       apiClient,
+
+		wsfs: filer,
+		root: root,
+	}, nil
+}
+
+func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
+	entries, err := w.wsfs.ReadDir(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	seenPaths := make(map[string]workspace.ObjectInfo)
+	for i := range entries {
+		info, err := entries[i].Info()
+		if err != nil {
+			return nil, err
+		}
+		sysInfo := info.Sys().(workspace.ObjectInfo)
+
+		// If the object is a notebook, include an extension in the entry.
+		if sysInfo.ObjectType == workspace.ObjectTypeNotebook {
+			stat, err := w.getNotebookStatByNameWithoutExt(ctx, entries[i].Name())
+			if err != nil {
+				return nil, err
+			}
+			// Replace the entry with the new entry that includes the extension.
+			entries[i] = wsfsDirEntry{wsfsFileInfo{oi: *stat.ObjectInfo}}
+		}
+
+		// Error if we have seen this path before in the current directory.
+		// If not seen before, add it to the seen paths.
+		if _, ok := seenPaths[entries[i].Name()]; ok {
+			return nil, DuplicatePathError{
+				oi1:        seenPaths[entries[i].Name()],
+				oi2:        sysInfo,
+				commonName: path.Join(name, entries[i].Name()),
+			}
+		}
+		seenPaths[entries[i].Name()] = sysInfo
+	}
+
+	return entries, nil
+}
+
+// Note: The import API returns opaque internal errors for namespace clashes
+// (e.g. a file and a notebook or a directory and a notebook). Thus users of this
+// method should be careful to avoid such clashes.
+func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error {
+	return w.wsfs.Write(ctx, name, reader, mode...)
+}
+
+// Try to read the file as a regular file. If the file is not found, try to read it as a notebook.
+func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
+	r, err := w.wsfs.Read(ctx, name)
+
+	// If the file is not found, it might be a notebook.
+	if errors.As(err, &FileDoesNotExistError{}) {
+		stat, serr := w.getNotebookStatByNameWithExt(ctx, name)
+		if serr != nil {
+			// Unable to stat. Return the stat error.
+			return nil, serr
+		}
+		if stat == nil {
+			// Not a notebook. Return the original error.
+			return nil, err
+		}
+
+		// The workspace files filer performs an additional stat call to make sure
+		// the path is not a directory. We can skip this step since we already have
+		// the stat object and know that the path is a notebook.
+		return w.workspaceClient.Workspace.Download(
+			ctx,
+			path.Join(w.root, stat.nameForWorkspaceAPI),
+			workspace.DownloadFormat(stat.ReposExportFormat),
+		)
+	}
+	return r, err
+}
+
+// Try to delete the file as a regular file. If the file is not found, try to delete it as a notebook.
+func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error {
+	err := w.wsfs.Delete(ctx, name, mode...)
+
+	// If the file is not found, it might be a notebook.
+	if errors.As(err, &FileDoesNotExistError{}) {
+		stat, serr := w.getNotebookStatByNameWithExt(ctx, name)
+		if serr != nil {
+			// Unable to stat. Return the stat error.
+			return serr
+		}
+		if stat == nil {
+			// Not a notebook. Return the original error.
+			return err
+		}
+
+		return w.wsfs.Delete(ctx, stat.nameForWorkspaceAPI, mode...)
+	}
+
+	return err
+}
+
+// Try to stat the file as a regular file. If the file is not found, try to stat it as a notebook.
+func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
+	info, err := w.wsfs.Stat(ctx, name)
+
+	// If the file is not found, it might be a notebook.
+	if errors.As(err, &FileDoesNotExistError{}) {
+		stat, serr := w.getNotebookStatByNameWithExt(ctx, name)
+		if serr != nil {
+			// Unable to stat. Return the stat error.
+			return nil, serr
+		}
+		if stat == nil {
+			// Not a notebook. Return the original error.
+			return nil, err
+		}
+
+		return wsfsFileInfo{oi: *stat.ObjectInfo}, nil
+	}
+
+	return info, err
+}
+
+// Note: The import API returns opaque internal errors for namespace clashes
+// (e.g. a file and a notebook or a directory and a notebook). Thus users of this
+// method should be careful to avoid such clashes.
+func (w *workspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error {
+	return w.wsfs.Mkdir(ctx, name)
+}

From 364a609ea7339ab830c23cf9efc428d36f6c4ce3 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Fri, 31 May 2024 09:13:43 +0200
Subject: [PATCH 40/41] Upgrade TF provider to 1.46.0 (#1460)

## Changes

Release notes in
https://github.com/databricks/terraform-provider-databricks/releases/tag/v1.46.0

Notable changes since 1.43.0:
* The job resource has been migrated to the Go SDK. More fields are now
passed through from DABs into TF.
* Improved zero-value handling.

## Tests

n/a
---
 bundle/internal/tf/codegen/schema/version.go  |   2 +-
 .../internal/tf/schema/data_source_catalog.go |  46 +++
 bundle/internal/tf/schema/data_source_job.go  |  28 +-
 .../schema/data_source_mlflow_experiment.go   |  19 +
 .../internal/tf/schema/data_source_table.go   | 127 ++++++
 bundle/internal/tf/schema/data_sources.go     |   6 +
 ...omatic_cluster_update_workspace_setting.go |  39 ++
 bundle/internal/tf/schema/resource_cluster.go |   6 -
 ...ance_security_profile_workspace_setting.go |  15 +
 ...d_security_monitoring_workspace_setting.go |  14 +
 bundle/internal/tf/schema/resource_job.go     | 363 +++++++++++++-----
 .../tf/schema/resource_model_serving.go       |  16 +-
 .../tf/schema/resource_quality_monitor.go     |  76 ++++
 .../internal/tf/schema/resource_sql_table.go  |   1 +
 .../tf/schema/resource_vector_search_index.go |  11 +-
 bundle/internal/tf/schema/resources.go        | 218 ++++++-----
 bundle/internal/tf/schema/root.go             |   2 +-
 17 files changed, 757 insertions(+), 232 deletions(-)
 create mode 100644 bundle/internal/tf/schema/data_source_catalog.go
 create mode 100644 bundle/internal/tf/schema/data_source_mlflow_experiment.go
 create mode 100644 bundle/internal/tf/schema/data_source_table.go
 create mode 100644 bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go
 create mode 100644 bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go
 create mode 100644 bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go
 create mode 100644 bundle/internal/tf/schema/resource_quality_monitor.go

diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go
index cf98e16e8..f55b6c4f0 100644
--- a/bundle/internal/tf/codegen/schema/version.go
+++ b/bundle/internal/tf/codegen/schema/version.go
@@ -1,3 +1,3 @@
 package schema
 
-const ProviderVersion = "1.43.0"
+const ProviderVersion = "1.46.0"
diff --git a/bundle/internal/tf/schema/data_source_catalog.go b/bundle/internal/tf/schema/data_source_catalog.go
new file mode 100644
index 000000000..6f9237cfa
--- /dev/null
+++ b/bundle/internal/tf/schema/data_source_catalog.go
@@ -0,0 +1,46 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag struct {
+	InheritedFromName string `json:"inherited_from_name,omitempty"`
+	InheritedFromType string `json:"inherited_from_type,omitempty"`
+	Value             string `json:"value"`
+}
+
+type DataSourceCatalogCatalogInfoProvisioningInfo struct {
+	State string `json:"state,omitempty"`
+}
+
+type DataSourceCatalogCatalogInfo struct {
+	BrowseOnly                          bool                                                             `json:"browse_only,omitempty"`
+	CatalogType                         string                                                           `json:"catalog_type,omitempty"`
+	Comment                             string                                                           `json:"comment,omitempty"`
+	ConnectionName                      string                                                           `json:"connection_name,omitempty"`
+	CreatedAt                           int                                                              `json:"created_at,omitempty"`
+	CreatedBy                           string                                                           `json:"created_by,omitempty"`
+	EnablePredictiveOptimization        string                                                           `json:"enable_predictive_optimization,omitempty"`
+	FullName                            string                                                           `json:"full_name,omitempty"`
+	IsolationMode                       string                                                           `json:"isolation_mode,omitempty"`
+	MetastoreId                         string                                                           `json:"metastore_id,omitempty"`
+	Name                                string                                                           `json:"name,omitempty"`
+	Options                             map[string]string                                                `json:"options,omitempty"`
+	Owner                               string                                                           `json:"owner,omitempty"`
+	Properties                          map[string]string                                                `json:"properties,omitempty"`
+	ProviderName                        string                                                           `json:"provider_name,omitempty"`
+	SecurableKind                       string                                                           `json:"securable_kind,omitempty"`
+	SecurableType                       string                                                           `json:"securable_type,omitempty"`
+	ShareName                           string                                                           `json:"share_name,omitempty"`
+	StorageLocation                     string                                                           `json:"storage_location,omitempty"`
+	StorageRoot                         string                                                           `json:"storage_root,omitempty"`
+	UpdatedAt                           int                                                              `json:"updated_at,omitempty"`
+	UpdatedBy                           string                                                           `json:"updated_by,omitempty"`
+	EffectivePredictiveOptimizationFlag *DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"`
+	ProvisioningInfo                    *DataSourceCatalogCatalogInfoProvisioningInfo                    `json:"provisioning_info,omitempty"`
+}
+
+type DataSourceCatalog struct {
+	Id          string                        `json:"id,omitempty"`
+	Name        string                        `json:"name"`
+	CatalogInfo *DataSourceCatalogCatalogInfo `json:"catalog_info,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go
index e5ec5afb7..d517bbe0f 100644
--- a/bundle/internal/tf/schema/data_source_job.go
+++ b/bundle/internal/tf/schema/data_source_job.go
@@ -55,9 +55,9 @@ type DataSourceJobJobSettingsSettingsGitSource struct {
 }
 
 type DataSourceJobJobSettingsSettingsHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type DataSourceJobJobSettingsSettingsHealth struct {
@@ -222,7 +222,7 @@ type DataSourceJobJobSettingsSettingsJobClusterNewCluster struct {
 }
 
 type DataSourceJobJobSettingsSettingsJobCluster struct {
-	JobClusterKey string                                                `json:"job_cluster_key,omitempty"`
+	JobClusterKey string                                                `json:"job_cluster_key"`
 	NewCluster    *DataSourceJobJobSettingsSettingsJobClusterNewCluster `json:"new_cluster,omitempty"`
 }
 
@@ -533,9 +533,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskEmailNotifications struc
 }
 
 type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealth struct {
@@ -805,7 +805,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskQuery struct {
 
 type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTask struct {
 	Parameters  map[string]string                                                    `json:"parameters,omitempty"`
-	WarehouseId string                                                               `json:"warehouse_id,omitempty"`
+	WarehouseId string                                                               `json:"warehouse_id"`
 	Alert       *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskAlert     `json:"alert,omitempty"`
 	Dashboard   *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"`
 	File        *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskFile      `json:"file,omitempty"`
@@ -844,7 +844,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTask struct {
 	MinRetryIntervalMillis int                                                                      `json:"min_retry_interval_millis,omitempty"`
 	RetryOnTimeout         bool                                                                     `json:"retry_on_timeout,omitempty"`
 	RunIf                  string                                                                   `json:"run_if,omitempty"`
-	TaskKey                string                                                                   `json:"task_key,omitempty"`
+	TaskKey                string                                                                   `json:"task_key"`
 	TimeoutSeconds         int                                                                      `json:"timeout_seconds,omitempty"`
 	ConditionTask          *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskConditionTask        `json:"condition_task,omitempty"`
 	DbtTask                *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskDbtTask              `json:"dbt_task,omitempty"`
@@ -872,9 +872,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTask struct {
 }
 
 type DataSourceJobJobSettingsSettingsTaskHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type DataSourceJobJobSettingsSettingsTaskHealth struct {
@@ -1144,7 +1144,7 @@ type DataSourceJobJobSettingsSettingsTaskSqlTaskQuery struct {
 
 type DataSourceJobJobSettingsSettingsTaskSqlTask struct {
 	Parameters  map[string]string                                     `json:"parameters,omitempty"`
-	WarehouseId string                                                `json:"warehouse_id,omitempty"`
+	WarehouseId string                                                `json:"warehouse_id"`
 	Alert       *DataSourceJobJobSettingsSettingsTaskSqlTaskAlert     `json:"alert,omitempty"`
 	Dashboard   *DataSourceJobJobSettingsSettingsTaskSqlTaskDashboard `json:"dashboard,omitempty"`
 	File        *DataSourceJobJobSettingsSettingsTaskSqlTaskFile      `json:"file,omitempty"`
@@ -1183,7 +1183,7 @@ type DataSourceJobJobSettingsSettingsTask struct {
 	MinRetryIntervalMillis int                                                       `json:"min_retry_interval_millis,omitempty"`
 	RetryOnTimeout         bool                                                      `json:"retry_on_timeout,omitempty"`
 	RunIf                  string                                                    `json:"run_if,omitempty"`
-	TaskKey                string                                                    `json:"task_key,omitempty"`
+	TaskKey                string                                                    `json:"task_key"`
 	TimeoutSeconds         int                                                       `json:"timeout_seconds,omitempty"`
 	ConditionTask          *DataSourceJobJobSettingsSettingsTaskConditionTask        `json:"condition_task,omitempty"`
 	DbtTask                *DataSourceJobJobSettingsSettingsTaskDbtTask              `json:"dbt_task,omitempty"`
diff --git a/bundle/internal/tf/schema/data_source_mlflow_experiment.go b/bundle/internal/tf/schema/data_source_mlflow_experiment.go
new file mode 100644
index 000000000..979130c5f
--- /dev/null
+++ b/bundle/internal/tf/schema/data_source_mlflow_experiment.go
@@ -0,0 +1,19 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type DataSourceMlflowExperimentTags struct {
+	Key   string `json:"key,omitempty"`
+	Value string `json:"value,omitempty"`
+}
+
+type DataSourceMlflowExperiment struct {
+	ArtifactLocation string                           `json:"artifact_location,omitempty"`
+	CreationTime     int                              `json:"creation_time,omitempty"`
+	ExperimentId     string                           `json:"experiment_id,omitempty"`
+	Id               string                           `json:"id,omitempty"`
+	LastUpdateTime   int                              `json:"last_update_time,omitempty"`
+	LifecycleStage   string                           `json:"lifecycle_stage,omitempty"`
+	Name             string                           `json:"name,omitempty"`
+	Tags             []DataSourceMlflowExperimentTags `json:"tags,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/data_source_table.go b/bundle/internal/tf/schema/data_source_table.go
new file mode 100644
index 000000000..f59959696
--- /dev/null
+++ b/bundle/internal/tf/schema/data_source_table.go
@@ -0,0 +1,127 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type DataSourceTableTableInfoColumnsMask struct {
+	FunctionName     string   `json:"function_name,omitempty"`
+	UsingColumnNames []string `json:"using_column_names,omitempty"`
+}
+
+type DataSourceTableTableInfoColumns struct {
+	Comment          string                               `json:"comment,omitempty"`
+	Name             string                               `json:"name,omitempty"`
+	Nullable         bool                                 `json:"nullable,omitempty"`
+	PartitionIndex   int                                  `json:"partition_index,omitempty"`
+	Position         int                                  `json:"position,omitempty"`
+	TypeIntervalType string                               `json:"type_interval_type,omitempty"`
+	TypeJson         string                               `json:"type_json,omitempty"`
+	TypeName         string                               `json:"type_name,omitempty"`
+	TypePrecision    int                                  `json:"type_precision,omitempty"`
+	TypeScale        int                                  `json:"type_scale,omitempty"`
+	TypeText         string                               `json:"type_text,omitempty"`
+	Mask             *DataSourceTableTableInfoColumnsMask `json:"mask,omitempty"`
+}
+
+type DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs struct {
+	DeltaRuntimeProperties map[string]string `json:"delta_runtime_properties"`
+}
+
+type DataSourceTableTableInfoEffectivePredictiveOptimizationFlag struct {
+	InheritedFromName string `json:"inherited_from_name,omitempty"`
+	InheritedFromType string `json:"inherited_from_type,omitempty"`
+	Value             string `json:"value"`
+}
+
+type DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails struct {
+	Algorithm    string `json:"algorithm,omitempty"`
+	AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"`
+}
+
+type DataSourceTableTableInfoEncryptionDetails struct {
+	SseEncryptionDetails *DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"`
+}
+
+type DataSourceTableTableInfoRowFilter struct {
+	FunctionName     string   `json:"function_name"`
+	InputColumnNames []string `json:"input_column_names"`
+}
+
+type DataSourceTableTableInfoTableConstraintsForeignKeyConstraint struct {
+	ChildColumns  []string `json:"child_columns"`
+	Name          string   `json:"name"`
+	ParentColumns []string `json:"parent_columns"`
+	ParentTable   string   `json:"parent_table"`
+}
+
+type DataSourceTableTableInfoTableConstraintsNamedTableConstraint struct {
+	Name string `json:"name"`
+}
+
+type DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint struct {
+	ChildColumns []string `json:"child_columns"`
+	Name         string   `json:"name"`
+}
+
+type DataSourceTableTableInfoTableConstraints struct {
+	ForeignKeyConstraint *DataSourceTableTableInfoTableConstraintsForeignKeyConstraint `json:"foreign_key_constraint,omitempty"`
+	NamedTableConstraint *DataSourceTableTableInfoTableConstraintsNamedTableConstraint `json:"named_table_constraint,omitempty"`
+	PrimaryKeyConstraint *DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint `json:"primary_key_constraint,omitempty"`
+}
+
+type DataSourceTableTableInfoViewDependenciesDependenciesFunction struct {
+	FunctionFullName string `json:"function_full_name"`
+}
+
+type DataSourceTableTableInfoViewDependenciesDependenciesTable struct {
+	TableFullName string `json:"table_full_name"`
+}
+
+type DataSourceTableTableInfoViewDependenciesDependencies struct {
+	Function *DataSourceTableTableInfoViewDependenciesDependenciesFunction `json:"function,omitempty"`
+	Table    *DataSourceTableTableInfoViewDependenciesDependenciesTable    `json:"table,omitempty"`
+}
+
+type DataSourceTableTableInfoViewDependencies struct {
+	Dependencies []DataSourceTableTableInfoViewDependenciesDependencies `json:"dependencies,omitempty"`
+}
+
+type DataSourceTableTableInfo struct {
+	AccessPoint                         string                                                       `json:"access_point,omitempty"`
+	BrowseOnly                          bool                                                         `json:"browse_only,omitempty"`
+	CatalogName                         string                                                       `json:"catalog_name,omitempty"`
+	Comment                             string                                                       `json:"comment,omitempty"`
+	CreatedAt                           int                                                          `json:"created_at,omitempty"`
+	CreatedBy                           string                                                       `json:"created_by,omitempty"`
+	DataAccessConfigurationId           string                                                       `json:"data_access_configuration_id,omitempty"`
+	DataSourceFormat                    string                                                       `json:"data_source_format,omitempty"`
+	DeletedAt                           int                                                          `json:"deleted_at,omitempty"`
+	EnablePredictiveOptimization        string                                                       `json:"enable_predictive_optimization,omitempty"`
+	FullName                            string                                                       `json:"full_name,omitempty"`
+	MetastoreId                         string                                                       `json:"metastore_id,omitempty"`
+	Name                                string                                                       `json:"name,omitempty"`
+	Owner                               string                                                       `json:"owner,omitempty"`
+	PipelineId                          string                                                       `json:"pipeline_id,omitempty"`
+	Properties                          map[string]string                                            `json:"properties,omitempty"`
+	SchemaName                          string                                                       `json:"schema_name,omitempty"`
+	SqlPath                             string                                                       `json:"sql_path,omitempty"`
+	StorageCredentialName               string                                                       `json:"storage_credential_name,omitempty"`
+	StorageLocation                     string                                                       `json:"storage_location,omitempty"`
+	TableId                             string                                                       `json:"table_id,omitempty"`
+	TableType                           string                                                       `json:"table_type,omitempty"`
+	UpdatedAt                           int                                                          `json:"updated_at,omitempty"`
+	UpdatedBy                           string                                                       `json:"updated_by,omitempty"`
+	ViewDefinition                      string                                                       `json:"view_definition,omitempty"`
+	Columns                             []DataSourceTableTableInfoColumns                            `json:"columns,omitempty"`
+	DeltaRuntimePropertiesKvpairs       *DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs       `json:"delta_runtime_properties_kvpairs,omitempty"`
+	EffectivePredictiveOptimizationFlag *DataSourceTableTableInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"`
+	EncryptionDetails                   *DataSourceTableTableInfoEncryptionDetails                   `json:"encryption_details,omitempty"`
+	RowFilter                           *DataSourceTableTableInfoRowFilter                           `json:"row_filter,omitempty"`
+	TableConstraints                    []DataSourceTableTableInfoTableConstraints                   `json:"table_constraints,omitempty"`
+	ViewDependencies                    *DataSourceTableTableInfoViewDependencies                    `json:"view_dependencies,omitempty"`
+}
+
+type DataSourceTable struct {
+	Id        string                    `json:"id,omitempty"`
+	Name      string                    `json:"name"`
+	TableInfo *DataSourceTableTableInfo `json:"table_info,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go
index 2e02c4388..c32483db0 100644
--- a/bundle/internal/tf/schema/data_sources.go
+++ b/bundle/internal/tf/schema/data_sources.go
@@ -7,6 +7,7 @@ type DataSources struct {
 	AwsBucketPolicy       map[string]any `json:"databricks_aws_bucket_policy,omitempty"`
 	AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"`
 	AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"`
+	Catalog               map[string]any `json:"databricks_catalog,omitempty"`
 	Catalogs              map[string]any `json:"databricks_catalogs,omitempty"`
 	Cluster               map[string]any `json:"databricks_cluster,omitempty"`
 	ClusterPolicy         map[string]any `json:"databricks_cluster_policy,omitempty"`
@@ -26,6 +27,7 @@ type DataSources struct {
 	Jobs                  map[string]any `json:"databricks_jobs,omitempty"`
 	Metastore             map[string]any `json:"databricks_metastore,omitempty"`
 	Metastores            map[string]any `json:"databricks_metastores,omitempty"`
+	MlflowExperiment      map[string]any `json:"databricks_mlflow_experiment,omitempty"`
 	MlflowModel           map[string]any `json:"databricks_mlflow_model,omitempty"`
 	MwsCredentials        map[string]any `json:"databricks_mws_credentials,omitempty"`
 	MwsWorkspaces         map[string]any `json:"databricks_mws_workspaces,omitempty"`
@@ -43,6 +45,7 @@ type DataSources struct {
 	SqlWarehouses         map[string]any `json:"databricks_sql_warehouses,omitempty"`
 	StorageCredential     map[string]any `json:"databricks_storage_credential,omitempty"`
 	StorageCredentials    map[string]any `json:"databricks_storage_credentials,omitempty"`
+	Table                 map[string]any `json:"databricks_table,omitempty"`
 	Tables                map[string]any `json:"databricks_tables,omitempty"`
 	User                  map[string]any `json:"databricks_user,omitempty"`
 	Views                 map[string]any `json:"databricks_views,omitempty"`
@@ -56,6 +59,7 @@ func NewDataSources() *DataSources {
 		AwsBucketPolicy:       make(map[string]any),
 		AwsCrossaccountPolicy: make(map[string]any),
 		AwsUnityCatalogPolicy: make(map[string]any),
+		Catalog:               make(map[string]any),
 		Catalogs:              make(map[string]any),
 		Cluster:               make(map[string]any),
 		ClusterPolicy:         make(map[string]any),
@@ -75,6 +79,7 @@ func NewDataSources() *DataSources {
 		Jobs:                  make(map[string]any),
 		Metastore:             make(map[string]any),
 		Metastores:            make(map[string]any),
+		MlflowExperiment:      make(map[string]any),
 		MlflowModel:           make(map[string]any),
 		MwsCredentials:        make(map[string]any),
 		MwsWorkspaces:         make(map[string]any),
@@ -92,6 +97,7 @@ func NewDataSources() *DataSources {
 		SqlWarehouses:         make(map[string]any),
 		StorageCredential:     make(map[string]any),
 		StorageCredentials:    make(map[string]any),
+		Table:                 make(map[string]any),
 		Tables:                make(map[string]any),
 		User:                  make(map[string]any),
 		Views:                 make(map[string]any),
diff --git a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go
new file mode 100644
index 000000000..e95639de8
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go
@@ -0,0 +1,39 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct {
+	ForcedForComplianceMode           bool `json:"forced_for_compliance_mode,omitempty"`
+	UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"`
+	UnavailableForNonEnterpriseTier   bool `json:"unavailable_for_non_enterprise_tier,omitempty"`
+}
+
+type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct {
+	Hours   int `json:"hours,omitempty"`
+	Minutes int `json:"minutes,omitempty"`
+}
+
+type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct {
+	DayOfWeek       string                                                                                                                             `json:"day_of_week,omitempty"`
+	Frequency       string                                                                                                                             `json:"frequency,omitempty"`
+	WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"`
+}
+
+type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow struct {
+	WeekDayBasedSchedule *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule `json:"week_day_based_schedule,omitempty"`
+}
+
+type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct {
+	CanToggle                       bool                                                                                            `json:"can_toggle,omitempty"`
+	Enabled                         bool                                                                                            `json:"enabled,omitempty"`
+	RestartEvenIfNoUpdatesAvailable bool                                                                                            `json:"restart_even_if_no_updates_available,omitempty"`
+	EnablementDetails               *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"`
+	MaintenanceWindow               *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"`
+}
+
+type ResourceAutomaticClusterUpdateWorkspaceSetting struct {
+	Etag                            string                                                                         `json:"etag,omitempty"`
+	Id                              string                                                                         `json:"id,omitempty"`
+	SettingName                     string                                                                         `json:"setting_name,omitempty"`
+	AutomaticClusterUpdateWorkspace *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace `json:"automatic_cluster_update_workspace,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go
index 046e0bb43..e4106d049 100644
--- a/bundle/internal/tf/schema/resource_cluster.go
+++ b/bundle/internal/tf/schema/resource_cluster.go
@@ -32,10 +32,6 @@ type ResourceClusterAzureAttributes struct {
 	LogAnalyticsInfo *ResourceClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
 }
 
-type ResourceClusterCloneFrom struct {
-	SourceClusterId string `json:"source_cluster_id"`
-}
-
 type ResourceClusterClusterLogConfDbfs struct {
 	Destination string `json:"destination"`
 }
@@ -169,7 +165,6 @@ type ResourceCluster struct {
 	AutoterminationMinutes    int                               `json:"autotermination_minutes,omitempty"`
 	ClusterId                 string                            `json:"cluster_id,omitempty"`
 	ClusterName               string                            `json:"cluster_name,omitempty"`
-	ClusterSource             string                            `json:"cluster_source,omitempty"`
 	CustomTags                map[string]string                 `json:"custom_tags,omitempty"`
 	DataSecurityMode          string                            `json:"data_security_mode,omitempty"`
 	DefaultTags               map[string]string                 `json:"default_tags,omitempty"`
@@ -195,7 +190,6 @@ type ResourceCluster struct {
 	Autoscale                 *ResourceClusterAutoscale         `json:"autoscale,omitempty"`
 	AwsAttributes             *ResourceClusterAwsAttributes     `json:"aws_attributes,omitempty"`
 	AzureAttributes           *ResourceClusterAzureAttributes   `json:"azure_attributes,omitempty"`
-	CloneFrom                 *ResourceClusterCloneFrom         `json:"clone_from,omitempty"`
 	ClusterLogConf            *ResourceClusterClusterLogConf    `json:"cluster_log_conf,omitempty"`
 	ClusterMountInfo          []ResourceClusterClusterMountInfo `json:"cluster_mount_info,omitempty"`
 	DockerImage               *ResourceClusterDockerImage       `json:"docker_image,omitempty"`
diff --git a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go
new file mode 100644
index 000000000..50815f753
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go
@@ -0,0 +1,15 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct {
+	ComplianceStandards []string `json:"compliance_standards,omitempty"`
+	IsEnabled           bool     `json:"is_enabled,omitempty"`
+}
+
+type ResourceComplianceSecurityProfileWorkspaceSetting struct {
+	Etag                               string                                                                               `json:"etag,omitempty"`
+	Id                                 string                                                                               `json:"id,omitempty"`
+	SettingName                        string                                                                               `json:"setting_name,omitempty"`
+	ComplianceSecurityProfileWorkspace *ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace `json:"compliance_security_profile_workspace,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go
new file mode 100644
index 000000000..2f552402a
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go
@@ -0,0 +1,14 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct {
+	IsEnabled bool `json:"is_enabled,omitempty"`
+}
+
+type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct {
+	Etag                                string                                                                                 `json:"etag,omitempty"`
+	Id                                  string                                                                                 `json:"id,omitempty"`
+	SettingName                         string                                                                                 `json:"setting_name,omitempty"`
+	EnhancedSecurityMonitoringWorkspace *ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace `json:"enhanced_security_monitoring_workspace,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go
index 6958face8..0950073e2 100644
--- a/bundle/internal/tf/schema/resource_job.go
+++ b/bundle/internal/tf/schema/resource_job.go
@@ -39,6 +39,10 @@ type ResourceJobEnvironment struct {
 	Spec           *ResourceJobEnvironmentSpec `json:"spec,omitempty"`
 }
 
+type ResourceJobGitSourceGitSnapshot struct {
+	UsedCommit string `json:"used_commit,omitempty"`
+}
+
 type ResourceJobGitSourceJobSource struct {
 	DirtyState          string `json:"dirty_state,omitempty"`
 	ImportFromGitBranch string `json:"import_from_git_branch"`
@@ -46,18 +50,19 @@ type ResourceJobGitSourceJobSource struct {
 }
 
 type ResourceJobGitSource struct {
-	Branch    string                         `json:"branch,omitempty"`
-	Commit    string                         `json:"commit,omitempty"`
-	Provider  string                         `json:"provider,omitempty"`
-	Tag       string                         `json:"tag,omitempty"`
-	Url       string                         `json:"url"`
-	JobSource *ResourceJobGitSourceJobSource `json:"job_source,omitempty"`
+	Branch      string                           `json:"branch,omitempty"`
+	Commit      string                           `json:"commit,omitempty"`
+	Provider    string                           `json:"provider,omitempty"`
+	Tag         string                           `json:"tag,omitempty"`
+	Url         string                           `json:"url"`
+	GitSnapshot *ResourceJobGitSourceGitSnapshot `json:"git_snapshot,omitempty"`
+	JobSource   *ResourceJobGitSourceJobSource   `json:"job_source,omitempty"`
 }
 
 type ResourceJobHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type ResourceJobHealth struct {
@@ -72,7 +77,9 @@ type ResourceJobJobClusterNewClusterAutoscale struct {
 type ResourceJobJobClusterNewClusterAwsAttributes struct {
 	Availability        string `json:"availability,omitempty"`
 	EbsVolumeCount      int    `json:"ebs_volume_count,omitempty"`
+	EbsVolumeIops       int    `json:"ebs_volume_iops,omitempty"`
 	EbsVolumeSize       int    `json:"ebs_volume_size,omitempty"`
+	EbsVolumeThroughput int    `json:"ebs_volume_throughput,omitempty"`
 	EbsVolumeType       string `json:"ebs_volume_type,omitempty"`
 	FirstOnDemand       int    `json:"first_on_demand,omitempty"`
 	InstanceProfileArn  string `json:"instance_profile_arn,omitempty"`
@@ -80,10 +87,16 @@ type ResourceJobJobClusterNewClusterAwsAttributes struct {
 	ZoneId              string `json:"zone_id,omitempty"`
 }
 
+type ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo struct {
+	LogAnalyticsPrimaryKey  string `json:"log_analytics_primary_key,omitempty"`
+	LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
+}
+
 type ResourceJobJobClusterNewClusterAzureAttributes struct {
-	Availability    string `json:"availability,omitempty"`
-	FirstOnDemand   int    `json:"first_on_demand,omitempty"`
-	SpotBidMaxPrice int    `json:"spot_bid_max_price,omitempty"`
+	Availability     string                                                          `json:"availability,omitempty"`
+	FirstOnDemand    int                                                             `json:"first_on_demand,omitempty"`
+	SpotBidMaxPrice  int                                                             `json:"spot_bid_max_price,omitempty"`
+	LogAnalyticsInfo *ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
 }
 
 type ResourceJobJobClusterNewClusterClusterLogConfDbfs struct {
@@ -179,6 +192,32 @@ type ResourceJobJobClusterNewClusterInitScripts struct {
 	Workspace *ResourceJobJobClusterNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
 }
 
+type ResourceJobJobClusterNewClusterLibraryCran struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobJobClusterNewClusterLibraryMaven struct {
+	Coordinates string   `json:"coordinates"`
+	Exclusions  []string `json:"exclusions,omitempty"`
+	Repo        string   `json:"repo,omitempty"`
+}
+
+type ResourceJobJobClusterNewClusterLibraryPypi struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobJobClusterNewClusterLibrary struct {
+	Egg          string                                       `json:"egg,omitempty"`
+	Jar          string                                       `json:"jar,omitempty"`
+	Requirements string                                       `json:"requirements,omitempty"`
+	Whl          string                                       `json:"whl,omitempty"`
+	Cran         *ResourceJobJobClusterNewClusterLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobJobClusterNewClusterLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobJobClusterNewClusterLibraryPypi  `json:"pypi,omitempty"`
+}
+
 type ResourceJobJobClusterNewClusterWorkloadTypeClients struct {
 	Jobs      bool `json:"jobs,omitempty"`
 	Notebooks bool `json:"notebooks,omitempty"`
@@ -190,7 +229,6 @@ type ResourceJobJobClusterNewClusterWorkloadType struct {
 
 type ResourceJobJobClusterNewCluster struct {
 	ApplyPolicyDefaultValues  bool                                              `json:"apply_policy_default_values,omitempty"`
-	AutoterminationMinutes    int                                               `json:"autotermination_minutes,omitempty"`
 	ClusterId                 string                                            `json:"cluster_id,omitempty"`
 	ClusterName               string                                            `json:"cluster_name,omitempty"`
 	CustomTags                map[string]string                                 `json:"custom_tags,omitempty"`
@@ -218,11 +256,12 @@ type ResourceJobJobClusterNewCluster struct {
 	DockerImage               *ResourceJobJobClusterNewClusterDockerImage       `json:"docker_image,omitempty"`
 	GcpAttributes             *ResourceJobJobClusterNewClusterGcpAttributes     `json:"gcp_attributes,omitempty"`
 	InitScripts               []ResourceJobJobClusterNewClusterInitScripts      `json:"init_scripts,omitempty"`
+	Library                   []ResourceJobJobClusterNewClusterLibrary          `json:"library,omitempty"`
 	WorkloadType              *ResourceJobJobClusterNewClusterWorkloadType      `json:"workload_type,omitempty"`
 }
 
 type ResourceJobJobCluster struct {
-	JobClusterKey string                           `json:"job_cluster_key,omitempty"`
+	JobClusterKey string                           `json:"job_cluster_key"`
 	NewCluster    *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"`
 }
 
@@ -260,7 +299,9 @@ type ResourceJobNewClusterAutoscale struct {
 type ResourceJobNewClusterAwsAttributes struct {
 	Availability        string `json:"availability,omitempty"`
 	EbsVolumeCount      int    `json:"ebs_volume_count,omitempty"`
+	EbsVolumeIops       int    `json:"ebs_volume_iops,omitempty"`
 	EbsVolumeSize       int    `json:"ebs_volume_size,omitempty"`
+	EbsVolumeThroughput int    `json:"ebs_volume_throughput,omitempty"`
 	EbsVolumeType       string `json:"ebs_volume_type,omitempty"`
 	FirstOnDemand       int    `json:"first_on_demand,omitempty"`
 	InstanceProfileArn  string `json:"instance_profile_arn,omitempty"`
@@ -268,10 +309,16 @@ type ResourceJobNewClusterAwsAttributes struct {
 	ZoneId              string `json:"zone_id,omitempty"`
 }
 
+type ResourceJobNewClusterAzureAttributesLogAnalyticsInfo struct {
+	LogAnalyticsPrimaryKey  string `json:"log_analytics_primary_key,omitempty"`
+	LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
+}
+
 type ResourceJobNewClusterAzureAttributes struct {
-	Availability    string `json:"availability,omitempty"`
-	FirstOnDemand   int    `json:"first_on_demand,omitempty"`
-	SpotBidMaxPrice int    `json:"spot_bid_max_price,omitempty"`
+	Availability     string                                                `json:"availability,omitempty"`
+	FirstOnDemand    int                                                   `json:"first_on_demand,omitempty"`
+	SpotBidMaxPrice  int                                                   `json:"spot_bid_max_price,omitempty"`
+	LogAnalyticsInfo *ResourceJobNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
 }
 
 type ResourceJobNewClusterClusterLogConfDbfs struct {
@@ -367,6 +414,32 @@ type ResourceJobNewClusterInitScripts struct {
 	Workspace *ResourceJobNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
 }
 
+type ResourceJobNewClusterLibraryCran struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobNewClusterLibraryMaven struct {
+	Coordinates string   `json:"coordinates"`
+	Exclusions  []string `json:"exclusions,omitempty"`
+	Repo        string   `json:"repo,omitempty"`
+}
+
+type ResourceJobNewClusterLibraryPypi struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobNewClusterLibrary struct {
+	Egg          string                             `json:"egg,omitempty"`
+	Jar          string                             `json:"jar,omitempty"`
+	Requirements string                             `json:"requirements,omitempty"`
+	Whl          string                             `json:"whl,omitempty"`
+	Cran         *ResourceJobNewClusterLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobNewClusterLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobNewClusterLibraryPypi  `json:"pypi,omitempty"`
+}
+
 type ResourceJobNewClusterWorkloadTypeClients struct {
 	Jobs      bool `json:"jobs,omitempty"`
 	Notebooks bool `json:"notebooks,omitempty"`
@@ -378,7 +451,6 @@ type ResourceJobNewClusterWorkloadType struct {
 
 type ResourceJobNewCluster struct {
 	ApplyPolicyDefaultValues  bool                                    `json:"apply_policy_default_values,omitempty"`
-	AutoterminationMinutes    int                                     `json:"autotermination_minutes,omitempty"`
 	ClusterId                 string                                  `json:"cluster_id,omitempty"`
 	ClusterName               string                                  `json:"cluster_name,omitempty"`
 	CustomTags                map[string]string                       `json:"custom_tags,omitempty"`
@@ -406,6 +478,7 @@ type ResourceJobNewCluster struct {
 	DockerImage               *ResourceJobNewClusterDockerImage       `json:"docker_image,omitempty"`
 	GcpAttributes             *ResourceJobNewClusterGcpAttributes     `json:"gcp_attributes,omitempty"`
 	InitScripts               []ResourceJobNewClusterInitScripts      `json:"init_scripts,omitempty"`
+	Library                   []ResourceJobNewClusterLibrary          `json:"library,omitempty"`
 	WorkloadType              *ResourceJobNewClusterWorkloadType      `json:"workload_type,omitempty"`
 }
 
@@ -533,9 +606,9 @@ type ResourceJobTaskForEachTaskTaskEmailNotifications struct {
 }
 
 type ResourceJobTaskForEachTaskTaskHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type ResourceJobTaskForEachTaskTaskHealth struct {
@@ -576,7 +649,9 @@ type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct {
 type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct {
 	Availability        string `json:"availability,omitempty"`
 	EbsVolumeCount      int    `json:"ebs_volume_count,omitempty"`
+	EbsVolumeIops       int    `json:"ebs_volume_iops,omitempty"`
 	EbsVolumeSize       int    `json:"ebs_volume_size,omitempty"`
+	EbsVolumeThroughput int    `json:"ebs_volume_throughput,omitempty"`
 	EbsVolumeType       string `json:"ebs_volume_type,omitempty"`
 	FirstOnDemand       int    `json:"first_on_demand,omitempty"`
 	InstanceProfileArn  string `json:"instance_profile_arn,omitempty"`
@@ -584,10 +659,16 @@ type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct {
 	ZoneId              string `json:"zone_id,omitempty"`
 }
 
+type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo struct {
+	LogAnalyticsPrimaryKey  string `json:"log_analytics_primary_key,omitempty"`
+	LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
+}
+
 type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributes struct {
-	Availability    string `json:"availability,omitempty"`
-	FirstOnDemand   int    `json:"first_on_demand,omitempty"`
-	SpotBidMaxPrice int    `json:"spot_bid_max_price,omitempty"`
+	Availability     string                                                                   `json:"availability,omitempty"`
+	FirstOnDemand    int                                                                      `json:"first_on_demand,omitempty"`
+	SpotBidMaxPrice  int                                                                      `json:"spot_bid_max_price,omitempty"`
+	LogAnalyticsInfo *ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
 }
 
 type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfDbfs struct {
@@ -683,6 +764,32 @@ type ResourceJobTaskForEachTaskTaskNewClusterInitScripts struct {
 	Workspace *ResourceJobTaskForEachTaskTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
 }
 
+type ResourceJobTaskForEachTaskTaskNewClusterLibraryCran struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven struct {
+	Coordinates string   `json:"coordinates"`
+	Exclusions  []string `json:"exclusions,omitempty"`
+	Repo        string   `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskForEachTaskTaskNewClusterLibrary struct {
+	Egg          string                                                `json:"egg,omitempty"`
+	Jar          string                                                `json:"jar,omitempty"`
+	Requirements string                                                `json:"requirements,omitempty"`
+	Whl          string                                                `json:"whl,omitempty"`
+	Cran         *ResourceJobTaskForEachTaskTaskNewClusterLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi  `json:"pypi,omitempty"`
+}
+
 type ResourceJobTaskForEachTaskTaskNewClusterWorkloadTypeClients struct {
 	Jobs      bool `json:"jobs,omitempty"`
 	Notebooks bool `json:"notebooks,omitempty"`
@@ -694,7 +801,6 @@ type ResourceJobTaskForEachTaskTaskNewClusterWorkloadType struct {
 
 type ResourceJobTaskForEachTaskTaskNewCluster struct {
 	ApplyPolicyDefaultValues  bool                                                       `json:"apply_policy_default_values,omitempty"`
-	AutoterminationMinutes    int                                                        `json:"autotermination_minutes,omitempty"`
 	ClusterId                 string                                                     `json:"cluster_id,omitempty"`
 	ClusterName               string                                                     `json:"cluster_name,omitempty"`
 	CustomTags                map[string]string                                          `json:"custom_tags,omitempty"`
@@ -706,7 +812,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct {
 	IdempotencyToken          string                                                     `json:"idempotency_token,omitempty"`
 	InstancePoolId            string                                                     `json:"instance_pool_id,omitempty"`
 	NodeTypeId                string                                                     `json:"node_type_id,omitempty"`
-	NumWorkers                int                                                        `json:"num_workers"`
+	NumWorkers                int                                                        `json:"num_workers,omitempty"`
 	PolicyId                  string                                                     `json:"policy_id,omitempty"`
 	RuntimeEngine             string                                                     `json:"runtime_engine,omitempty"`
 	SingleUserName            string                                                     `json:"single_user_name,omitempty"`
@@ -722,6 +828,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct {
 	DockerImage               *ResourceJobTaskForEachTaskTaskNewClusterDockerImage       `json:"docker_image,omitempty"`
 	GcpAttributes             *ResourceJobTaskForEachTaskTaskNewClusterGcpAttributes     `json:"gcp_attributes,omitempty"`
 	InitScripts               []ResourceJobTaskForEachTaskTaskNewClusterInitScripts      `json:"init_scripts,omitempty"`
+	Library                   []ResourceJobTaskForEachTaskTaskNewClusterLibrary          `json:"library,omitempty"`
 	WorkloadType              *ResourceJobTaskForEachTaskTaskNewClusterWorkloadType      `json:"workload_type,omitempty"`
 }
 
@@ -750,9 +857,21 @@ type ResourceJobTaskForEachTaskTaskPythonWheelTask struct {
 	Parameters      []string          `json:"parameters,omitempty"`
 }
 
+type ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams struct {
+	FullRefresh bool `json:"full_refresh,omitempty"`
+}
+
 type ResourceJobTaskForEachTaskTaskRunJobTask struct {
-	JobId         int               `json:"job_id"`
-	JobParameters map[string]string `json:"job_parameters,omitempty"`
+	DbtCommands       []string                                                `json:"dbt_commands,omitempty"`
+	JarParams         []string                                                `json:"jar_params,omitempty"`
+	JobId             int                                                     `json:"job_id"`
+	JobParameters     map[string]string                                       `json:"job_parameters,omitempty"`
+	NotebookParams    map[string]string                                       `json:"notebook_params,omitempty"`
+	PythonNamedParams map[string]string                                       `json:"python_named_params,omitempty"`
+	PythonParams      []string                                                `json:"python_params,omitempty"`
+	SparkSubmitParams []string                                                `json:"spark_submit_params,omitempty"`
+	SqlParams         map[string]string                                       `json:"sql_params,omitempty"`
+	PipelineParams    *ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"`
 }
 
 type ResourceJobTaskForEachTaskTaskSparkJarTask struct {
@@ -805,7 +924,7 @@ type ResourceJobTaskForEachTaskTaskSqlTaskQuery struct {
 
 type ResourceJobTaskForEachTaskTaskSqlTask struct {
 	Parameters  map[string]string                               `json:"parameters,omitempty"`
-	WarehouseId string                                          `json:"warehouse_id,omitempty"`
+	WarehouseId string                                          `json:"warehouse_id"`
 	Alert       *ResourceJobTaskForEachTaskTaskSqlTaskAlert     `json:"alert,omitempty"`
 	Dashboard   *ResourceJobTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"`
 	File        *ResourceJobTaskForEachTaskTaskSqlTaskFile      `json:"file,omitempty"`
@@ -836,33 +955,34 @@ type ResourceJobTaskForEachTaskTaskWebhookNotifications struct {
 }
 
 type ResourceJobTaskForEachTaskTask struct {
-	Description            string                                              `json:"description,omitempty"`
-	EnvironmentKey         string                                              `json:"environment_key,omitempty"`
-	ExistingClusterId      string                                              `json:"existing_cluster_id,omitempty"`
-	JobClusterKey          string                                              `json:"job_cluster_key,omitempty"`
-	MaxRetries             int                                                 `json:"max_retries,omitempty"`
-	MinRetryIntervalMillis int                                                 `json:"min_retry_interval_millis,omitempty"`
-	RetryOnTimeout         bool                                                `json:"retry_on_timeout,omitempty"`
-	RunIf                  string                                              `json:"run_if,omitempty"`
-	TaskKey                string                                              `json:"task_key,omitempty"`
-	TimeoutSeconds         int                                                 `json:"timeout_seconds,omitempty"`
-	ConditionTask          *ResourceJobTaskForEachTaskTaskConditionTask        `json:"condition_task,omitempty"`
-	DbtTask                *ResourceJobTaskForEachTaskTaskDbtTask              `json:"dbt_task,omitempty"`
-	DependsOn              []ResourceJobTaskForEachTaskTaskDependsOn           `json:"depends_on,omitempty"`
-	EmailNotifications     *ResourceJobTaskForEachTaskTaskEmailNotifications   `json:"email_notifications,omitempty"`
-	Health                 *ResourceJobTaskForEachTaskTaskHealth               `json:"health,omitempty"`
-	Library                []ResourceJobTaskForEachTaskTaskLibrary             `json:"library,omitempty"`
-	NewCluster             *ResourceJobTaskForEachTaskTaskNewCluster           `json:"new_cluster,omitempty"`
-	NotebookTask           *ResourceJobTaskForEachTaskTaskNotebookTask         `json:"notebook_task,omitempty"`
-	NotificationSettings   *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"`
-	PipelineTask           *ResourceJobTaskForEachTaskTaskPipelineTask         `json:"pipeline_task,omitempty"`
-	PythonWheelTask        *ResourceJobTaskForEachTaskTaskPythonWheelTask      `json:"python_wheel_task,omitempty"`
-	RunJobTask             *ResourceJobTaskForEachTaskTaskRunJobTask           `json:"run_job_task,omitempty"`
-	SparkJarTask           *ResourceJobTaskForEachTaskTaskSparkJarTask         `json:"spark_jar_task,omitempty"`
-	SparkPythonTask        *ResourceJobTaskForEachTaskTaskSparkPythonTask      `json:"spark_python_task,omitempty"`
-	SparkSubmitTask        *ResourceJobTaskForEachTaskTaskSparkSubmitTask      `json:"spark_submit_task,omitempty"`
-	SqlTask                *ResourceJobTaskForEachTaskTaskSqlTask              `json:"sql_task,omitempty"`
-	WebhookNotifications   *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
+	Description             string                                              `json:"description,omitempty"`
+	DisableAutoOptimization bool                                                `json:"disable_auto_optimization,omitempty"`
+	EnvironmentKey          string                                              `json:"environment_key,omitempty"`
+	ExistingClusterId       string                                              `json:"existing_cluster_id,omitempty"`
+	JobClusterKey           string                                              `json:"job_cluster_key,omitempty"`
+	MaxRetries              int                                                 `json:"max_retries,omitempty"`
+	MinRetryIntervalMillis  int                                                 `json:"min_retry_interval_millis,omitempty"`
+	RetryOnTimeout          bool                                                `json:"retry_on_timeout,omitempty"`
+	RunIf                   string                                              `json:"run_if,omitempty"`
+	TaskKey                 string                                              `json:"task_key"`
+	TimeoutSeconds          int                                                 `json:"timeout_seconds,omitempty"`
+	ConditionTask           *ResourceJobTaskForEachTaskTaskConditionTask        `json:"condition_task,omitempty"`
+	DbtTask                 *ResourceJobTaskForEachTaskTaskDbtTask              `json:"dbt_task,omitempty"`
+	DependsOn               []ResourceJobTaskForEachTaskTaskDependsOn           `json:"depends_on,omitempty"`
+	EmailNotifications      *ResourceJobTaskForEachTaskTaskEmailNotifications   `json:"email_notifications,omitempty"`
+	Health                  *ResourceJobTaskForEachTaskTaskHealth               `json:"health,omitempty"`
+	Library                 []ResourceJobTaskForEachTaskTaskLibrary             `json:"library,omitempty"`
+	NewCluster              *ResourceJobTaskForEachTaskTaskNewCluster           `json:"new_cluster,omitempty"`
+	NotebookTask            *ResourceJobTaskForEachTaskTaskNotebookTask         `json:"notebook_task,omitempty"`
+	NotificationSettings    *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"`
+	PipelineTask            *ResourceJobTaskForEachTaskTaskPipelineTask         `json:"pipeline_task,omitempty"`
+	PythonWheelTask         *ResourceJobTaskForEachTaskTaskPythonWheelTask      `json:"python_wheel_task,omitempty"`
+	RunJobTask              *ResourceJobTaskForEachTaskTaskRunJobTask           `json:"run_job_task,omitempty"`
+	SparkJarTask            *ResourceJobTaskForEachTaskTaskSparkJarTask         `json:"spark_jar_task,omitempty"`
+	SparkPythonTask         *ResourceJobTaskForEachTaskTaskSparkPythonTask      `json:"spark_python_task,omitempty"`
+	SparkSubmitTask         *ResourceJobTaskForEachTaskTaskSparkSubmitTask      `json:"spark_submit_task,omitempty"`
+	SqlTask                 *ResourceJobTaskForEachTaskTaskSqlTask              `json:"sql_task,omitempty"`
+	WebhookNotifications    *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
 }
 
 type ResourceJobTaskForEachTask struct {
@@ -872,9 +992,9 @@ type ResourceJobTaskForEachTask struct {
 }
 
 type ResourceJobTaskHealthRules struct {
-	Metric string `json:"metric,omitempty"`
-	Op     string `json:"op,omitempty"`
-	Value  int    `json:"value,omitempty"`
+	Metric string `json:"metric"`
+	Op     string `json:"op"`
+	Value  int    `json:"value"`
 }
 
 type ResourceJobTaskHealth struct {
@@ -915,7 +1035,9 @@ type ResourceJobTaskNewClusterAutoscale struct {
 type ResourceJobTaskNewClusterAwsAttributes struct {
 	Availability        string `json:"availability,omitempty"`
 	EbsVolumeCount      int    `json:"ebs_volume_count,omitempty"`
+	EbsVolumeIops       int    `json:"ebs_volume_iops,omitempty"`
 	EbsVolumeSize       int    `json:"ebs_volume_size,omitempty"`
+	EbsVolumeThroughput int    `json:"ebs_volume_throughput,omitempty"`
 	EbsVolumeType       string `json:"ebs_volume_type,omitempty"`
 	FirstOnDemand       int    `json:"first_on_demand,omitempty"`
 	InstanceProfileArn  string `json:"instance_profile_arn,omitempty"`
@@ -923,10 +1045,16 @@ type ResourceJobTaskNewClusterAwsAttributes struct {
 	ZoneId              string `json:"zone_id,omitempty"`
 }
 
+type ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo struct {
+	LogAnalyticsPrimaryKey  string `json:"log_analytics_primary_key,omitempty"`
+	LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
+}
+
 type ResourceJobTaskNewClusterAzureAttributes struct {
-	Availability    string `json:"availability,omitempty"`
-	FirstOnDemand   int    `json:"first_on_demand,omitempty"`
-	SpotBidMaxPrice int    `json:"spot_bid_max_price,omitempty"`
+	Availability     string                                                    `json:"availability,omitempty"`
+	FirstOnDemand    int                                                       `json:"first_on_demand,omitempty"`
+	SpotBidMaxPrice  int                                                       `json:"spot_bid_max_price,omitempty"`
+	LogAnalyticsInfo *ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
 }
 
 type ResourceJobTaskNewClusterClusterLogConfDbfs struct {
@@ -1022,6 +1150,32 @@ type ResourceJobTaskNewClusterInitScripts struct {
 	Workspace *ResourceJobTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
 }
 
+type ResourceJobTaskNewClusterLibraryCran struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskNewClusterLibraryMaven struct {
+	Coordinates string   `json:"coordinates"`
+	Exclusions  []string `json:"exclusions,omitempty"`
+	Repo        string   `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskNewClusterLibraryPypi struct {
+	Package string `json:"package"`
+	Repo    string `json:"repo,omitempty"`
+}
+
+type ResourceJobTaskNewClusterLibrary struct {
+	Egg          string                                 `json:"egg,omitempty"`
+	Jar          string                                 `json:"jar,omitempty"`
+	Requirements string                                 `json:"requirements,omitempty"`
+	Whl          string                                 `json:"whl,omitempty"`
+	Cran         *ResourceJobTaskNewClusterLibraryCran  `json:"cran,omitempty"`
+	Maven        *ResourceJobTaskNewClusterLibraryMaven `json:"maven,omitempty"`
+	Pypi         *ResourceJobTaskNewClusterLibraryPypi  `json:"pypi,omitempty"`
+}
+
 type ResourceJobTaskNewClusterWorkloadTypeClients struct {
 	Jobs      bool `json:"jobs,omitempty"`
 	Notebooks bool `json:"notebooks,omitempty"`
@@ -1033,7 +1187,6 @@ type ResourceJobTaskNewClusterWorkloadType struct {
 
 type ResourceJobTaskNewCluster struct {
 	ApplyPolicyDefaultValues  bool                                        `json:"apply_policy_default_values,omitempty"`
-	AutoterminationMinutes    int                                         `json:"autotermination_minutes,omitempty"`
 	ClusterId                 string                                      `json:"cluster_id,omitempty"`
 	ClusterName               string                                      `json:"cluster_name,omitempty"`
 	CustomTags                map[string]string                           `json:"custom_tags,omitempty"`
@@ -1061,6 +1214,7 @@ type ResourceJobTaskNewCluster struct {
 	DockerImage               *ResourceJobTaskNewClusterDockerImage       `json:"docker_image,omitempty"`
 	GcpAttributes             *ResourceJobTaskNewClusterGcpAttributes     `json:"gcp_attributes,omitempty"`
 	InitScripts               []ResourceJobTaskNewClusterInitScripts      `json:"init_scripts,omitempty"`
+	Library                   []ResourceJobTaskNewClusterLibrary          `json:"library,omitempty"`
 	WorkloadType              *ResourceJobTaskNewClusterWorkloadType      `json:"workload_type,omitempty"`
 }
 
@@ -1089,9 +1243,21 @@ type ResourceJobTaskPythonWheelTask struct {
 	Parameters      []string          `json:"parameters,omitempty"`
 }
 
+type ResourceJobTaskRunJobTaskPipelineParams struct {
+	FullRefresh bool `json:"full_refresh,omitempty"`
+}
+
 type ResourceJobTaskRunJobTask struct {
-	JobId         int               `json:"job_id"`
-	JobParameters map[string]string `json:"job_parameters,omitempty"`
+	DbtCommands       []string                                 `json:"dbt_commands,omitempty"`
+	JarParams         []string                                 `json:"jar_params,omitempty"`
+	JobId             int                                      `json:"job_id"`
+	JobParameters     map[string]string                        `json:"job_parameters,omitempty"`
+	NotebookParams    map[string]string                        `json:"notebook_params,omitempty"`
+	PythonNamedParams map[string]string                        `json:"python_named_params,omitempty"`
+	PythonParams      []string                                 `json:"python_params,omitempty"`
+	SparkSubmitParams []string                                 `json:"spark_submit_params,omitempty"`
+	SqlParams         map[string]string                        `json:"sql_params,omitempty"`
+	PipelineParams    *ResourceJobTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"`
 }
 
 type ResourceJobTaskSparkJarTask struct {
@@ -1144,7 +1310,7 @@ type ResourceJobTaskSqlTaskQuery struct {
 
 type ResourceJobTaskSqlTask struct {
 	Parameters  map[string]string                `json:"parameters,omitempty"`
-	WarehouseId string                           `json:"warehouse_id,omitempty"`
+	WarehouseId string                           `json:"warehouse_id"`
 	Alert       *ResourceJobTaskSqlTaskAlert     `json:"alert,omitempty"`
 	Dashboard   *ResourceJobTaskSqlTaskDashboard `json:"dashboard,omitempty"`
 	File        *ResourceJobTaskSqlTaskFile      `json:"file,omitempty"`
@@ -1175,34 +1341,35 @@ type ResourceJobTaskWebhookNotifications struct {
 }
 
 type ResourceJobTask struct {
-	Description            string                               `json:"description,omitempty"`
-	EnvironmentKey         string                               `json:"environment_key,omitempty"`
-	ExistingClusterId      string                               `json:"existing_cluster_id,omitempty"`
-	JobClusterKey          string                               `json:"job_cluster_key,omitempty"`
-	MaxRetries             int                                  `json:"max_retries,omitempty"`
-	MinRetryIntervalMillis int                                  `json:"min_retry_interval_millis,omitempty"`
-	RetryOnTimeout         bool                                 `json:"retry_on_timeout,omitempty"`
-	RunIf                  string                               `json:"run_if,omitempty"`
-	TaskKey                string                               `json:"task_key,omitempty"`
-	TimeoutSeconds         int                                  `json:"timeout_seconds,omitempty"`
-	ConditionTask          *ResourceJobTaskConditionTask        `json:"condition_task,omitempty"`
-	DbtTask                *ResourceJobTaskDbtTask              `json:"dbt_task,omitempty"`
-	DependsOn              []ResourceJobTaskDependsOn           `json:"depends_on,omitempty"`
-	EmailNotifications     *ResourceJobTaskEmailNotifications   `json:"email_notifications,omitempty"`
-	ForEachTask            *ResourceJobTaskForEachTask          `json:"for_each_task,omitempty"`
-	Health                 *ResourceJobTaskHealth               `json:"health,omitempty"`
-	Library                []ResourceJobTaskLibrary             `json:"library,omitempty"`
-	NewCluster             *ResourceJobTaskNewCluster           `json:"new_cluster,omitempty"`
-	NotebookTask           *ResourceJobTaskNotebookTask         `json:"notebook_task,omitempty"`
-	NotificationSettings   *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"`
-	PipelineTask           *ResourceJobTaskPipelineTask         `json:"pipeline_task,omitempty"`
-	PythonWheelTask        *ResourceJobTaskPythonWheelTask      `json:"python_wheel_task,omitempty"`
-	RunJobTask             *ResourceJobTaskRunJobTask           `json:"run_job_task,omitempty"`
-	SparkJarTask           *ResourceJobTaskSparkJarTask         `json:"spark_jar_task,omitempty"`
-	SparkPythonTask        *ResourceJobTaskSparkPythonTask      `json:"spark_python_task,omitempty"`
-	SparkSubmitTask        *ResourceJobTaskSparkSubmitTask      `json:"spark_submit_task,omitempty"`
-	SqlTask                *ResourceJobTaskSqlTask              `json:"sql_task,omitempty"`
-	WebhookNotifications   *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
+	Description             string                               `json:"description,omitempty"`
+	DisableAutoOptimization bool                                 `json:"disable_auto_optimization,omitempty"`
+	EnvironmentKey          string                               `json:"environment_key,omitempty"`
+	ExistingClusterId       string                               `json:"existing_cluster_id,omitempty"`
+	JobClusterKey           string                               `json:"job_cluster_key,omitempty"`
+	MaxRetries              int                                  `json:"max_retries,omitempty"`
+	MinRetryIntervalMillis  int                                  `json:"min_retry_interval_millis,omitempty"`
+	RetryOnTimeout          bool                                 `json:"retry_on_timeout,omitempty"`
+	RunIf                   string                               `json:"run_if,omitempty"`
+	TaskKey                 string                               `json:"task_key"`
+	TimeoutSeconds          int                                  `json:"timeout_seconds,omitempty"`
+	ConditionTask           *ResourceJobTaskConditionTask        `json:"condition_task,omitempty"`
+	DbtTask                 *ResourceJobTaskDbtTask              `json:"dbt_task,omitempty"`
+	DependsOn               []ResourceJobTaskDependsOn           `json:"depends_on,omitempty"`
+	EmailNotifications      *ResourceJobTaskEmailNotifications   `json:"email_notifications,omitempty"`
+	ForEachTask             *ResourceJobTaskForEachTask          `json:"for_each_task,omitempty"`
+	Health                  *ResourceJobTaskHealth               `json:"health,omitempty"`
+	Library                 []ResourceJobTaskLibrary             `json:"library,omitempty"`
+	NewCluster              *ResourceJobTaskNewCluster           `json:"new_cluster,omitempty"`
+	NotebookTask            *ResourceJobTaskNotebookTask         `json:"notebook_task,omitempty"`
+	NotificationSettings    *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"`
+	PipelineTask            *ResourceJobTaskPipelineTask         `json:"pipeline_task,omitempty"`
+	PythonWheelTask         *ResourceJobTaskPythonWheelTask      `json:"python_wheel_task,omitempty"`
+	RunJobTask              *ResourceJobTaskRunJobTask           `json:"run_job_task,omitempty"`
+	SparkJarTask            *ResourceJobTaskSparkJarTask         `json:"spark_jar_task,omitempty"`
+	SparkPythonTask         *ResourceJobTaskSparkPythonTask      `json:"spark_python_task,omitempty"`
+	SparkSubmitTask         *ResourceJobTaskSparkSubmitTask      `json:"spark_submit_task,omitempty"`
+	SqlTask                 *ResourceJobTaskSqlTask              `json:"sql_task,omitempty"`
+	WebhookNotifications    *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
 }
 
 type ResourceJobTriggerFileArrival struct {
@@ -1211,6 +1378,13 @@ type ResourceJobTriggerFileArrival struct {
 	WaitAfterLastChangeSeconds    int    `json:"wait_after_last_change_seconds,omitempty"`
 }
 
+type ResourceJobTriggerTable struct {
+	Condition                     string   `json:"condition,omitempty"`
+	MinTimeBetweenTriggersSeconds int      `json:"min_time_between_triggers_seconds,omitempty"`
+	TableNames                    []string `json:"table_names,omitempty"`
+	WaitAfterLastChangeSeconds    int      `json:"wait_after_last_change_seconds,omitempty"`
+}
+
 type ResourceJobTriggerTableUpdate struct {
 	Condition                     string   `json:"condition,omitempty"`
 	MinTimeBetweenTriggersSeconds int      `json:"min_time_between_triggers_seconds,omitempty"`
@@ -1221,6 +1395,7 @@ type ResourceJobTriggerTableUpdate struct {
 type ResourceJobTrigger struct {
 	PauseStatus string                         `json:"pause_status,omitempty"`
 	FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"`
+	Table       *ResourceJobTriggerTable       `json:"table,omitempty"`
 	TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"`
 }
 
diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go
index a74a544ed..f5ffbbe5e 100644
--- a/bundle/internal/tf/schema/resource_model_serving.go
+++ b/bundle/internal/tf/schema/resource_model_serving.go
@@ -34,12 +34,15 @@ type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServing
 }
 
 type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct {
-	OpenaiApiBase        string `json:"openai_api_base,omitempty"`
-	OpenaiApiKey         string `json:"openai_api_key"`
-	OpenaiApiType        string `json:"openai_api_type,omitempty"`
-	OpenaiApiVersion     string `json:"openai_api_version,omitempty"`
-	OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"`
-	OpenaiOrganization   string `json:"openai_organization,omitempty"`
+	MicrosoftEntraClientId     string `json:"microsoft_entra_client_id,omitempty"`
+	MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"`
+	MicrosoftEntraTenantId     string `json:"microsoft_entra_tenant_id,omitempty"`
+	OpenaiApiBase              string `json:"openai_api_base,omitempty"`
+	OpenaiApiKey               string `json:"openai_api_key,omitempty"`
+	OpenaiApiType              string `json:"openai_api_type,omitempty"`
+	OpenaiApiVersion           string `json:"openai_api_version,omitempty"`
+	OpenaiDeploymentName       string `json:"openai_deployment_name,omitempty"`
+	OpenaiOrganization         string `json:"openai_organization,omitempty"`
 }
 
 type ResourceModelServingConfigServedEntitiesExternalModelPalmConfig struct {
@@ -114,6 +117,7 @@ type ResourceModelServingTags struct {
 type ResourceModelServing struct {
 	Id                string                           `json:"id,omitempty"`
 	Name              string                           `json:"name"`
+	RouteOptimized    bool                             `json:"route_optimized,omitempty"`
 	ServingEndpointId string                           `json:"serving_endpoint_id,omitempty"`
 	Config            *ResourceModelServingConfig      `json:"config,omitempty"`
 	RateLimits        []ResourceModelServingRateLimits `json:"rate_limits,omitempty"`
diff --git a/bundle/internal/tf/schema/resource_quality_monitor.go b/bundle/internal/tf/schema/resource_quality_monitor.go
new file mode 100644
index 000000000..0fc2abd66
--- /dev/null
+++ b/bundle/internal/tf/schema/resource_quality_monitor.go
@@ -0,0 +1,76 @@
+// Generated from Databricks Terraform provider schema. DO NOT EDIT.
+
+package schema
+
+type ResourceQualityMonitorCustomMetrics struct {
+	Definition     string   `json:"definition"`
+	InputColumns   []string `json:"input_columns"`
+	Name           string   `json:"name"`
+	OutputDataType string   `json:"output_data_type"`
+	Type           string   `json:"type"`
+}
+
+type ResourceQualityMonitorDataClassificationConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type ResourceQualityMonitorInferenceLog struct {
+	Granularities      []string `json:"granularities"`
+	LabelCol           string   `json:"label_col,omitempty"`
+	ModelIdCol         string   `json:"model_id_col"`
+	PredictionCol      string   `json:"prediction_col"`
+	PredictionProbaCol string   `json:"prediction_proba_col,omitempty"`
+	ProblemType        string   `json:"problem_type"`
+	TimestampCol       string   `json:"timestamp_col"`
+}
+
+type ResourceQualityMonitorNotificationsOnFailure struct {
+	EmailAddresses []string `json:"email_addresses,omitempty"`
+}
+
+type ResourceQualityMonitorNotificationsOnNewClassificationTagDetected struct {
+	EmailAddresses []string `json:"email_addresses,omitempty"`
+}
+
+type ResourceQualityMonitorNotifications struct {
+	OnFailure                      *ResourceQualityMonitorNotificationsOnFailure                      `json:"on_failure,omitempty"`
+	OnNewClassificationTagDetected *ResourceQualityMonitorNotificationsOnNewClassificationTagDetected `json:"on_new_classification_tag_detected,omitempty"`
+}
+
+type ResourceQualityMonitorSchedule struct {
+	PauseStatus          string `json:"pause_status,omitempty"`
+	QuartzCronExpression string `json:"quartz_cron_expression"`
+	TimezoneId           string `json:"timezone_id"`
+}
+
+type ResourceQualityMonitorSnapshot struct {
+}
+
+type ResourceQualityMonitorTimeSeries struct {
+	Granularities []string `json:"granularities"`
+	TimestampCol  string   `json:"timestamp_col"`
+}
+
+type ResourceQualityMonitor struct {
+	AssetsDir                string                                          `json:"assets_dir"`
+	BaselineTableName        string                                          `json:"baseline_table_name,omitempty"`
+	DashboardId              string                                          `json:"dashboard_id,omitempty"`
+	DriftMetricsTableName    string                                          `json:"drift_metrics_table_name,omitempty"`
+	Id                       string                                          `json:"id,omitempty"`
+	LatestMonitorFailureMsg  string                                          `json:"latest_monitor_failure_msg,omitempty"`
+	MonitorVersion           string                                          `json:"monitor_version,omitempty"`
+	OutputSchemaName         string                                          `json:"output_schema_name"`
+	ProfileMetricsTableName  string                                          `json:"profile_metrics_table_name,omitempty"`
+	SkipBuiltinDashboard     bool                                            `json:"skip_builtin_dashboard,omitempty"`
+	SlicingExprs             []string                                        `json:"slicing_exprs,omitempty"`
+	Status                   string                                          `json:"status,omitempty"`
+	TableName                string                                          `json:"table_name"`
+	WarehouseId              string                                          `json:"warehouse_id,omitempty"`
+	CustomMetrics            []ResourceQualityMonitorCustomMetrics           `json:"custom_metrics,omitempty"`
+	DataClassificationConfig *ResourceQualityMonitorDataClassificationConfig `json:"data_classification_config,omitempty"`
+	InferenceLog             *ResourceQualityMonitorInferenceLog             `json:"inference_log,omitempty"`
+	Notifications            *ResourceQualityMonitorNotifications            `json:"notifications,omitempty"`
+	Schedule                 *ResourceQualityMonitorSchedule                 `json:"schedule,omitempty"`
+	Snapshot                 *ResourceQualityMonitorSnapshot                 `json:"snapshot,omitempty"`
+	TimeSeries               *ResourceQualityMonitorTimeSeries               `json:"time_series,omitempty"`
+}
diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go
index 97a8977bc..51fb3bc0d 100644
--- a/bundle/internal/tf/schema/resource_sql_table.go
+++ b/bundle/internal/tf/schema/resource_sql_table.go
@@ -18,6 +18,7 @@ type ResourceSqlTable struct {
 	Id                    string                   `json:"id,omitempty"`
 	Name                  string                   `json:"name"`
 	Options               map[string]string        `json:"options,omitempty"`
+	Owner                 string                   `json:"owner,omitempty"`
 	Partitions            []string                 `json:"partitions,omitempty"`
 	Properties            map[string]string        `json:"properties,omitempty"`
 	SchemaName            string                   `json:"schema_name"`
diff --git a/bundle/internal/tf/schema/resource_vector_search_index.go b/bundle/internal/tf/schema/resource_vector_search_index.go
index 06f666656..2ce51576d 100644
--- a/bundle/internal/tf/schema/resource_vector_search_index.go
+++ b/bundle/internal/tf/schema/resource_vector_search_index.go
@@ -13,11 +13,12 @@ type ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns struct {
 }
 
 type ResourceVectorSearchIndexDeltaSyncIndexSpec struct {
-	PipelineId             string                                                              `json:"pipeline_id,omitempty"`
-	PipelineType           string                                                              `json:"pipeline_type,omitempty"`
-	SourceTable            string                                                              `json:"source_table,omitempty"`
-	EmbeddingSourceColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"`
-	EmbeddingVectorColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"`
+	EmbeddingWritebackTable string                                                              `json:"embedding_writeback_table,omitempty"`
+	PipelineId              string                                                              `json:"pipeline_id,omitempty"`
+	PipelineType            string                                                              `json:"pipeline_type,omitempty"`
+	SourceTable             string                                                              `json:"source_table,omitempty"`
+	EmbeddingSourceColumns  []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"`
+	EmbeddingVectorColumns  []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"`
 }
 
 type ResourceVectorSearchIndexDirectAccessIndexSpecEmbeddingSourceColumns struct {
diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go
index e5eacc867..79d71a65f 100644
--- a/bundle/internal/tf/schema/resources.go
+++ b/bundle/internal/tf/schema/resources.go
@@ -3,115 +3,122 @@
 package schema
 
 type Resources struct {
-	AccessControlRuleSet           map[string]any `json:"databricks_access_control_rule_set,omitempty"`
-	ArtifactAllowlist              map[string]any `json:"databricks_artifact_allowlist,omitempty"`
-	AwsS3Mount                     map[string]any `json:"databricks_aws_s3_mount,omitempty"`
-	AzureAdlsGen1Mount             map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
-	AzureAdlsGen2Mount             map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
-	AzureBlobMount                 map[string]any `json:"databricks_azure_blob_mount,omitempty"`
-	Catalog                        map[string]any `json:"databricks_catalog,omitempty"`
-	CatalogWorkspaceBinding        map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
-	Cluster                        map[string]any `json:"databricks_cluster,omitempty"`
-	ClusterPolicy                  map[string]any `json:"databricks_cluster_policy,omitempty"`
-	Connection                     map[string]any `json:"databricks_connection,omitempty"`
-	DbfsFile                       map[string]any `json:"databricks_dbfs_file,omitempty"`
-	DefaultNamespaceSetting        map[string]any `json:"databricks_default_namespace_setting,omitempty"`
-	Directory                      map[string]any `json:"databricks_directory,omitempty"`
-	Entitlements                   map[string]any `json:"databricks_entitlements,omitempty"`
-	ExternalLocation               map[string]any `json:"databricks_external_location,omitempty"`
-	File                           map[string]any `json:"databricks_file,omitempty"`
-	GitCredential                  map[string]any `json:"databricks_git_credential,omitempty"`
-	GlobalInitScript               map[string]any `json:"databricks_global_init_script,omitempty"`
-	Grant                          map[string]any `json:"databricks_grant,omitempty"`
-	Grants                         map[string]any `json:"databricks_grants,omitempty"`
-	Group                          map[string]any `json:"databricks_group,omitempty"`
-	GroupInstanceProfile           map[string]any `json:"databricks_group_instance_profile,omitempty"`
-	GroupMember                    map[string]any `json:"databricks_group_member,omitempty"`
-	GroupRole                      map[string]any `json:"databricks_group_role,omitempty"`
-	InstancePool                   map[string]any `json:"databricks_instance_pool,omitempty"`
-	InstanceProfile                map[string]any `json:"databricks_instance_profile,omitempty"`
-	IpAccessList                   map[string]any `json:"databricks_ip_access_list,omitempty"`
-	Job                            map[string]any `json:"databricks_job,omitempty"`
-	LakehouseMonitor               map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
-	Library                        map[string]any `json:"databricks_library,omitempty"`
-	Metastore                      map[string]any `json:"databricks_metastore,omitempty"`
-	MetastoreAssignment            map[string]any `json:"databricks_metastore_assignment,omitempty"`
-	MetastoreDataAccess            map[string]any `json:"databricks_metastore_data_access,omitempty"`
-	MlflowExperiment               map[string]any `json:"databricks_mlflow_experiment,omitempty"`
-	MlflowModel                    map[string]any `json:"databricks_mlflow_model,omitempty"`
-	MlflowWebhook                  map[string]any `json:"databricks_mlflow_webhook,omitempty"`
-	ModelServing                   map[string]any `json:"databricks_model_serving,omitempty"`
-	Mount                          map[string]any `json:"databricks_mount,omitempty"`
-	MwsCredentials                 map[string]any `json:"databricks_mws_credentials,omitempty"`
-	MwsCustomerManagedKeys         map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
-	MwsLogDelivery                 map[string]any `json:"databricks_mws_log_delivery,omitempty"`
-	MwsNccBinding                  map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
-	MwsNccPrivateEndpointRule      map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
-	MwsNetworkConnectivityConfig   map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
-	MwsNetworks                    map[string]any `json:"databricks_mws_networks,omitempty"`
-	MwsPermissionAssignment        map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
-	MwsPrivateAccessSettings       map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
-	MwsStorageConfigurations       map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
-	MwsVpcEndpoint                 map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
-	MwsWorkspaces                  map[string]any `json:"databricks_mws_workspaces,omitempty"`
-	Notebook                       map[string]any `json:"databricks_notebook,omitempty"`
-	OboToken                       map[string]any `json:"databricks_obo_token,omitempty"`
-	OnlineTable                    map[string]any `json:"databricks_online_table,omitempty"`
-	PermissionAssignment           map[string]any `json:"databricks_permission_assignment,omitempty"`
-	Permissions                    map[string]any `json:"databricks_permissions,omitempty"`
-	Pipeline                       map[string]any `json:"databricks_pipeline,omitempty"`
-	Provider                       map[string]any `json:"databricks_provider,omitempty"`
-	Recipient                      map[string]any `json:"databricks_recipient,omitempty"`
-	RegisteredModel                map[string]any `json:"databricks_registered_model,omitempty"`
-	Repo                           map[string]any `json:"databricks_repo,omitempty"`
-	RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
-	Schema                         map[string]any `json:"databricks_schema,omitempty"`
-	Secret                         map[string]any `json:"databricks_secret,omitempty"`
-	SecretAcl                      map[string]any `json:"databricks_secret_acl,omitempty"`
-	SecretScope                    map[string]any `json:"databricks_secret_scope,omitempty"`
-	ServicePrincipal               map[string]any `json:"databricks_service_principal,omitempty"`
-	ServicePrincipalRole           map[string]any `json:"databricks_service_principal_role,omitempty"`
-	ServicePrincipalSecret         map[string]any `json:"databricks_service_principal_secret,omitempty"`
-	Share                          map[string]any `json:"databricks_share,omitempty"`
-	SqlAlert                       map[string]any `json:"databricks_sql_alert,omitempty"`
-	SqlDashboard                   map[string]any `json:"databricks_sql_dashboard,omitempty"`
-	SqlEndpoint                    map[string]any `json:"databricks_sql_endpoint,omitempty"`
-	SqlGlobalConfig                map[string]any `json:"databricks_sql_global_config,omitempty"`
-	SqlPermissions                 map[string]any `json:"databricks_sql_permissions,omitempty"`
-	SqlQuery                       map[string]any `json:"databricks_sql_query,omitempty"`
-	SqlTable                       map[string]any `json:"databricks_sql_table,omitempty"`
-	SqlVisualization               map[string]any `json:"databricks_sql_visualization,omitempty"`
-	SqlWidget                      map[string]any `json:"databricks_sql_widget,omitempty"`
-	StorageCredential              map[string]any `json:"databricks_storage_credential,omitempty"`
-	SystemSchema                   map[string]any `json:"databricks_system_schema,omitempty"`
-	Table                          map[string]any `json:"databricks_table,omitempty"`
-	Token                          map[string]any `json:"databricks_token,omitempty"`
-	User                           map[string]any `json:"databricks_user,omitempty"`
-	UserInstanceProfile            map[string]any `json:"databricks_user_instance_profile,omitempty"`
-	UserRole                       map[string]any `json:"databricks_user_role,omitempty"`
-	VectorSearchEndpoint           map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
-	VectorSearchIndex              map[string]any `json:"databricks_vector_search_index,omitempty"`
-	Volume                         map[string]any `json:"databricks_volume,omitempty"`
-	WorkspaceConf                  map[string]any `json:"databricks_workspace_conf,omitempty"`
-	WorkspaceFile                  map[string]any `json:"databricks_workspace_file,omitempty"`
+	AccessControlRuleSet                       map[string]any `json:"databricks_access_control_rule_set,omitempty"`
+	ArtifactAllowlist                          map[string]any `json:"databricks_artifact_allowlist,omitempty"`
+	AutomaticClusterUpdateWorkspaceSetting     map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"`
+	AwsS3Mount                                 map[string]any `json:"databricks_aws_s3_mount,omitempty"`
+	AzureAdlsGen1Mount                         map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
+	AzureAdlsGen2Mount                         map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
+	AzureBlobMount                             map[string]any `json:"databricks_azure_blob_mount,omitempty"`
+	Catalog                                    map[string]any `json:"databricks_catalog,omitempty"`
+	CatalogWorkspaceBinding                    map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
+	Cluster                                    map[string]any `json:"databricks_cluster,omitempty"`
+	ClusterPolicy                              map[string]any `json:"databricks_cluster_policy,omitempty"`
+	ComplianceSecurityProfileWorkspaceSetting  map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"`
+	Connection                                 map[string]any `json:"databricks_connection,omitempty"`
+	DbfsFile                                   map[string]any `json:"databricks_dbfs_file,omitempty"`
+	DefaultNamespaceSetting                    map[string]any `json:"databricks_default_namespace_setting,omitempty"`
+	Directory                                  map[string]any `json:"databricks_directory,omitempty"`
+	EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"`
+	Entitlements                               map[string]any `json:"databricks_entitlements,omitempty"`
+	ExternalLocation                           map[string]any `json:"databricks_external_location,omitempty"`
+	File                                       map[string]any `json:"databricks_file,omitempty"`
+	GitCredential                              map[string]any `json:"databricks_git_credential,omitempty"`
+	GlobalInitScript                           map[string]any `json:"databricks_global_init_script,omitempty"`
+	Grant                                      map[string]any `json:"databricks_grant,omitempty"`
+	Grants                                     map[string]any `json:"databricks_grants,omitempty"`
+	Group                                      map[string]any `json:"databricks_group,omitempty"`
+	GroupInstanceProfile                       map[string]any `json:"databricks_group_instance_profile,omitempty"`
+	GroupMember                                map[string]any `json:"databricks_group_member,omitempty"`
+	GroupRole                                  map[string]any `json:"databricks_group_role,omitempty"`
+	InstancePool                               map[string]any `json:"databricks_instance_pool,omitempty"`
+	InstanceProfile                            map[string]any `json:"databricks_instance_profile,omitempty"`
+	IpAccessList                               map[string]any `json:"databricks_ip_access_list,omitempty"`
+	Job                                        map[string]any `json:"databricks_job,omitempty"`
+	LakehouseMonitor                           map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
+	Library                                    map[string]any `json:"databricks_library,omitempty"`
+	Metastore                                  map[string]any `json:"databricks_metastore,omitempty"`
+	MetastoreAssignment                        map[string]any `json:"databricks_metastore_assignment,omitempty"`
+	MetastoreDataAccess                        map[string]any `json:"databricks_metastore_data_access,omitempty"`
+	MlflowExperiment                           map[string]any `json:"databricks_mlflow_experiment,omitempty"`
+	MlflowModel                                map[string]any `json:"databricks_mlflow_model,omitempty"`
+	MlflowWebhook                              map[string]any `json:"databricks_mlflow_webhook,omitempty"`
+	ModelServing                               map[string]any `json:"databricks_model_serving,omitempty"`
+	Mount                                      map[string]any `json:"databricks_mount,omitempty"`
+	MwsCredentials                             map[string]any `json:"databricks_mws_credentials,omitempty"`
+	MwsCustomerManagedKeys                     map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
+	MwsLogDelivery                             map[string]any `json:"databricks_mws_log_delivery,omitempty"`
+	MwsNccBinding                              map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
+	MwsNccPrivateEndpointRule                  map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
+	MwsNetworkConnectivityConfig               map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
+	MwsNetworks                                map[string]any `json:"databricks_mws_networks,omitempty"`
+	MwsPermissionAssignment                    map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
+	MwsPrivateAccessSettings                   map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
+	MwsStorageConfigurations                   map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
+	MwsVpcEndpoint                             map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
+	MwsWorkspaces                              map[string]any `json:"databricks_mws_workspaces,omitempty"`
+	Notebook                                   map[string]any `json:"databricks_notebook,omitempty"`
+	OboToken                                   map[string]any `json:"databricks_obo_token,omitempty"`
+	OnlineTable                                map[string]any `json:"databricks_online_table,omitempty"`
+	PermissionAssignment                       map[string]any `json:"databricks_permission_assignment,omitempty"`
+	Permissions                                map[string]any `json:"databricks_permissions,omitempty"`
+	Pipeline                                   map[string]any `json:"databricks_pipeline,omitempty"`
+	Provider                                   map[string]any `json:"databricks_provider,omitempty"`
+	QualityMonitor                             map[string]any `json:"databricks_quality_monitor,omitempty"`
+	Recipient                                  map[string]any `json:"databricks_recipient,omitempty"`
+	RegisteredModel                            map[string]any `json:"databricks_registered_model,omitempty"`
+	Repo                                       map[string]any `json:"databricks_repo,omitempty"`
+	RestrictWorkspaceAdminsSetting             map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
+	Schema                                     map[string]any `json:"databricks_schema,omitempty"`
+	Secret                                     map[string]any `json:"databricks_secret,omitempty"`
+	SecretAcl                                  map[string]any `json:"databricks_secret_acl,omitempty"`
+	SecretScope                                map[string]any `json:"databricks_secret_scope,omitempty"`
+	ServicePrincipal                           map[string]any `json:"databricks_service_principal,omitempty"`
+	ServicePrincipalRole                       map[string]any `json:"databricks_service_principal_role,omitempty"`
+	ServicePrincipalSecret                     map[string]any `json:"databricks_service_principal_secret,omitempty"`
+	Share                                      map[string]any `json:"databricks_share,omitempty"`
+	SqlAlert                                   map[string]any `json:"databricks_sql_alert,omitempty"`
+	SqlDashboard                               map[string]any `json:"databricks_sql_dashboard,omitempty"`
+	SqlEndpoint                                map[string]any `json:"databricks_sql_endpoint,omitempty"`
+	SqlGlobalConfig                            map[string]any `json:"databricks_sql_global_config,omitempty"`
+	SqlPermissions                             map[string]any `json:"databricks_sql_permissions,omitempty"`
+	SqlQuery                                   map[string]any `json:"databricks_sql_query,omitempty"`
+	SqlTable                                   map[string]any `json:"databricks_sql_table,omitempty"`
+	SqlVisualization                           map[string]any `json:"databricks_sql_visualization,omitempty"`
+	SqlWidget                                  map[string]any `json:"databricks_sql_widget,omitempty"`
+	StorageCredential                          map[string]any `json:"databricks_storage_credential,omitempty"`
+	SystemSchema                               map[string]any `json:"databricks_system_schema,omitempty"`
+	Table                                      map[string]any `json:"databricks_table,omitempty"`
+	Token                                      map[string]any `json:"databricks_token,omitempty"`
+	User                                       map[string]any `json:"databricks_user,omitempty"`
+	UserInstanceProfile                        map[string]any `json:"databricks_user_instance_profile,omitempty"`
+	UserRole                                   map[string]any `json:"databricks_user_role,omitempty"`
+	VectorSearchEndpoint                       map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
+	VectorSearchIndex                          map[string]any `json:"databricks_vector_search_index,omitempty"`
+	Volume                                     map[string]any `json:"databricks_volume,omitempty"`
+	WorkspaceConf                              map[string]any `json:"databricks_workspace_conf,omitempty"`
+	WorkspaceFile                              map[string]any `json:"databricks_workspace_file,omitempty"`
 }
 
 func NewResources() *Resources {
 	return &Resources{
-		AccessControlRuleSet:           make(map[string]any),
-		ArtifactAllowlist:              make(map[string]any),
-		AwsS3Mount:                     make(map[string]any),
-		AzureAdlsGen1Mount:             make(map[string]any),
-		AzureAdlsGen2Mount:             make(map[string]any),
-		AzureBlobMount:                 make(map[string]any),
-		Catalog:                        make(map[string]any),
-		CatalogWorkspaceBinding:        make(map[string]any),
-		Cluster:                        make(map[string]any),
-		ClusterPolicy:                  make(map[string]any),
-		Connection:                     make(map[string]any),
-		DbfsFile:                       make(map[string]any),
-		DefaultNamespaceSetting:        make(map[string]any),
-		Directory:                      make(map[string]any),
+		AccessControlRuleSet:                   make(map[string]any),
+		ArtifactAllowlist:                      make(map[string]any),
+		AutomaticClusterUpdateWorkspaceSetting: make(map[string]any),
+		AwsS3Mount:                             make(map[string]any),
+		AzureAdlsGen1Mount:                     make(map[string]any),
+		AzureAdlsGen2Mount:                     make(map[string]any),
+		AzureBlobMount:                         make(map[string]any),
+		Catalog:                                make(map[string]any),
+		CatalogWorkspaceBinding:                make(map[string]any),
+		Cluster:                                make(map[string]any),
+		ClusterPolicy:                          make(map[string]any),
+		ComplianceSecurityProfileWorkspaceSetting: make(map[string]any),
+		Connection:              make(map[string]any),
+		DbfsFile:                make(map[string]any),
+		DefaultNamespaceSetting: make(map[string]any),
+		Directory:               make(map[string]any),
+		EnhancedSecurityMonitoringWorkspaceSetting: make(map[string]any),
 		Entitlements:                   make(map[string]any),
 		ExternalLocation:               make(map[string]any),
 		File:                           make(map[string]any),
@@ -156,6 +163,7 @@ func NewResources() *Resources {
 		Permissions:                    make(map[string]any),
 		Pipeline:                       make(map[string]any),
 		Provider:                       make(map[string]any),
+		QualityMonitor:                 make(map[string]any),
 		Recipient:                      make(map[string]any),
 		RegisteredModel:                make(map[string]any),
 		Repo:                           make(map[string]any),
diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go
index b1fed9424..e4ca67740 100644
--- a/bundle/internal/tf/schema/root.go
+++ b/bundle/internal/tf/schema/root.go
@@ -21,7 +21,7 @@ type Root struct {
 
 const ProviderHost = "registry.terraform.io"
 const ProviderSource = "databricks/databricks"
-const ProviderVersion = "1.43.0"
+const ProviderVersion = "1.46.0"
 
 func NewRoot() *Root {
 	return &Root{

From a33d0c8bf9f19fb594d33c478adb5f925076dfe6 Mon Sep 17 00:00:00 2001
From: Aravind Segu <aravind.segu@databricks.com>
Date: Fri, 31 May 2024 02:42:25 -0700
Subject: [PATCH 41/41] Add support for Lakehouse monitoring in bundles (#1307)

## Changes

This change adds support for Lakehouse monitoring in bundles.

The associated resource type name is "quality monitor".

## Testing

Unit tests.

---------

Co-authored-by: Pieter Noordhuis <pcnoordhuis@gmail.com>
Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Co-authored-by: Arpit Jasapara <87999496+arpitjasa-db@users.noreply.github.com>
---
 .../mutator/process_target_mode_test.go       |  8 +++
 bundle/config/mutator/run_as.go               | 10 ++++
 bundle/config/mutator/run_as_test.go          |  1 +
 bundle/config/resources.go                    | 20 +++++++
 bundle/config/resources/quality_monitor.go    | 60 +++++++++++++++++++
 bundle/deploy/terraform/convert.go            | 22 +++++++
 bundle/deploy/terraform/convert_test.go       | 55 +++++++++++++++++
 bundle/deploy/terraform/interpolate.go        |  2 +
 .../tfdyn/convert_quality_monitor.go          | 37 ++++++++++++
 .../tfdyn/convert_quality_monitor_test.go     | 46 ++++++++++++++
 bundle/tests/quality_monitor/databricks.yml   | 40 +++++++++++++
 bundle/tests/quality_monitor_test.go          | 59 ++++++++++++++++++
 libs/dyn/convert/struct_info.go               |  9 +++
 libs/textutil/case.go                         | 14 +++++
 libs/textutil/case_test.go                    | 40 +++++++++++++
 15 files changed, 423 insertions(+)
 create mode 100644 bundle/config/resources/quality_monitor.go
 create mode 100644 bundle/deploy/terraform/tfdyn/convert_quality_monitor.go
 create mode 100644 bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go
 create mode 100644 bundle/tests/quality_monitor/databricks.yml
 create mode 100644 bundle/tests/quality_monitor_test.go
 create mode 100644 libs/textutil/case.go
 create mode 100644 libs/textutil/case_test.go

diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go
index 583efcfe5..cf8229bfe 100644
--- a/bundle/config/mutator/process_target_mode_test.go
+++ b/bundle/config/mutator/process_target_mode_test.go
@@ -97,6 +97,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 				RegisteredModels: map[string]*resources.RegisteredModel{
 					"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
 				},
+				QualityMonitors: map[string]*resources.QualityMonitor{
+					"qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}},
+				},
 			},
 		},
 		// Use AWS implementation for testing.
@@ -145,6 +148,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
 
 	// Registered model 1
 	assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
+
+	// Quality Monitor 1
+	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
 }
 
 func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) {
@@ -200,6 +206,7 @@ func TestProcessTargetModeDefault(t *testing.T) {
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
+	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
 }
 
 func TestProcessTargetModeProduction(t *testing.T) {
@@ -240,6 +247,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
+	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
 }
 
 func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {
diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go
index c5b294b27..aecd1d17e 100644
--- a/bundle/config/mutator/run_as.go
+++ b/bundle/config/mutator/run_as.go
@@ -100,6 +100,16 @@ func validateRunAs(b *bundle.Bundle) error {
 		}
 	}
 
+	// Monitors do not support run_as in the API.
+	if len(b.Config.Resources.QualityMonitors) > 0 {
+		return errUnsupportedResourceTypeForRunAs{
+			resourceType:     "quality_monitors",
+			resourceLocation: b.Config.GetLocation("resources.quality_monitors"),
+			currentUser:      b.Config.Workspace.CurrentUser.UserName,
+			runAsUser:        identity,
+		}
+	}
+
 	return nil
 }
 
diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go
index d6fb2939f..c57de847b 100644
--- a/bundle/config/mutator/run_as_test.go
+++ b/bundle/config/mutator/run_as_test.go
@@ -37,6 +37,7 @@ func allResourceTypes(t *testing.T) []string {
 		"model_serving_endpoints",
 		"models",
 		"pipelines",
+		"quality_monitors",
 		"registered_models",
 	},
 		resourceTypes,
diff --git a/bundle/config/resources.go b/bundle/config/resources.go
index 41ffc25cd..f70052ec0 100644
--- a/bundle/config/resources.go
+++ b/bundle/config/resources.go
@@ -17,6 +17,7 @@ type Resources struct {
 	Experiments           map[string]*resources.MlflowExperiment     `json:"experiments,omitempty"`
 	ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"`
 	RegisteredModels      map[string]*resources.RegisteredModel      `json:"registered_models,omitempty"`
+	QualityMonitors       map[string]*resources.QualityMonitor       `json:"quality_monitors,omitempty"`
 }
 
 type UniqueResourceIdTracker struct {
@@ -123,6 +124,19 @@ func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker,
 		tracker.Type[k] = "registered_model"
 		tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath
 	}
+	for k := range r.QualityMonitors {
+		if _, ok := tracker.Type[k]; ok {
+			return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)",
+				k,
+				tracker.Type[k],
+				tracker.ConfigPath[k],
+				"quality_monitor",
+				r.QualityMonitors[k].ConfigFilePath,
+			)
+		}
+		tracker.Type[k] = "quality_monitor"
+		tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath
+	}
 	return tracker, nil
 }
 
@@ -152,6 +166,9 @@ func (r *Resources) allResources() []resource {
 	for k, e := range r.RegisteredModels {
 		all = append(all, resource{resource_type: "registered model", resource: e, key: k})
 	}
+	for k, e := range r.QualityMonitors {
+		all = append(all, resource{resource_type: "quality monitor", resource: e, key: k})
+	}
 	return all
 }
 
@@ -189,6 +206,9 @@ func (r *Resources) ConfigureConfigFilePath() {
 	for _, e := range r.RegisteredModels {
 		e.ConfigureConfigFilePath()
 	}
+	for _, e := range r.QualityMonitors {
+		e.ConfigureConfigFilePath()
+	}
 }
 
 type ConfigResource interface {
diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go
new file mode 100644
index 000000000..0d13e58fa
--- /dev/null
+++ b/bundle/config/resources/quality_monitor.go
@@ -0,0 +1,60 @@
+package resources
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/marshal"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+)
+
+type QualityMonitor struct {
+	// Represents the Input Arguments for Terraform and will get
+	// converted to a HCL representation for CRUD
+	*catalog.CreateMonitor
+
+	// This represents the id which is the full name of the monitor
+	// (catalog_name.schema_name.table_name) that can be used
+	// as a reference in other resources. This value is returned by terraform.
+	ID string `json:"id,omitempty" bundle:"readonly"`
+
+	// Path to config file where the resource is defined. All bundle resources
+	// include this for interpolation purposes.
+	paths.Paths
+
+	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
+}
+
+func (s *QualityMonitor) UnmarshalJSON(b []byte) error {
+	return marshal.Unmarshal(b, s)
+}
+
+func (s QualityMonitor) MarshalJSON() ([]byte, error) {
+	return marshal.Marshal(s)
+}
+
+func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	_, err := w.QualityMonitors.Get(ctx, catalog.GetQualityMonitorRequest{
+		TableName: id,
+	})
+	if err != nil {
+		log.Debugf(ctx, "quality monitor %s does not exist", id)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *QualityMonitor) TerraformResourceName() string {
+	return "databricks_quality_monitor"
+}
+
+func (s *QualityMonitor) Validate() error {
+	if s == nil || !s.DynamicValue.IsValid() {
+		return fmt.Errorf("quality monitor is not defined")
+	}
+
+	return nil
+}
diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go
index d0b633582..a6ec04d9a 100644
--- a/bundle/deploy/terraform/convert.go
+++ b/bundle/deploy/terraform/convert.go
@@ -222,6 +222,13 @@ func BundleToTerraform(config *config.Root) *schema.Root {
 		}
 	}
 
+	for k, src := range config.Resources.QualityMonitors {
+		noResources = false
+		var dst schema.ResourceQualityMonitor
+		conv(src, &dst)
+		tfroot.Resource.QualityMonitor[k] = &dst
+	}
+
 	// We explicitly set "resource" to nil to omit it from a JSON encoding.
 	// This is required because the terraform CLI requires >= 1 resources defined
 	// if the "resource" property is used in a .tf.json file.
@@ -365,6 +372,16 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 				}
 				cur.ID = instance.Attributes.ID
 				config.Resources.RegisteredModels[resource.Name] = cur
+			case "databricks_quality_monitor":
+				if config.Resources.QualityMonitors == nil {
+					config.Resources.QualityMonitors = make(map[string]*resources.QualityMonitor)
+				}
+				cur := config.Resources.QualityMonitors[resource.Name]
+				if cur == nil {
+					cur = &resources.QualityMonitor{ModifiedStatus: resources.ModifiedStatusDeleted}
+				}
+				cur.ID = instance.Attributes.ID
+				config.Resources.QualityMonitors[resource.Name] = cur
 			case "databricks_permissions":
 			case "databricks_grants":
 				// Ignore; no need to pull these back into the configuration.
@@ -404,6 +421,11 @@ func TerraformToBundle(state *resourcesState, config *config.Root) error {
 			src.ModifiedStatus = resources.ModifiedStatusCreated
 		}
 	}
+	for _, src := range config.Resources.QualityMonitors {
+		if src.ModifiedStatus == "" && src.ID == "" {
+			src.ModifiedStatus = resources.ModifiedStatusCreated
+		}
+	}
 
 	return nil
 }
diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go
index 58523bb49..e1f73be28 100644
--- a/bundle/deploy/terraform/convert_test.go
+++ b/bundle/deploy/terraform/convert_test.go
@@ -629,6 +629,14 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "1"}},
 				},
 			},
+			{
+				Type: "databricks_quality_monitor",
+				Mode: "managed",
+				Name: "test_monitor",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "1"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -652,6 +660,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
 	assert.Equal(t, "1", config.Resources.RegisteredModels["test_registered_model"].ID)
 	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus)
 
+	assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -700,6 +711,13 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 					},
 				},
 			},
+			QualityMonitors: map[string]*resources.QualityMonitor{
+				"test_monitor": {
+					CreateMonitor: &catalog.CreateMonitor{
+						TableName: "test_monitor",
+					},
+				},
+			},
 		},
 	}
 	var tfState = resourcesState{
@@ -726,6 +744,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus)
 
+	assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
+
 	AssertFullResourceCoverage(t, &config)
 }
 
@@ -804,6 +825,18 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					},
 				},
 			},
+			QualityMonitors: map[string]*resources.QualityMonitor{
+				"test_monitor": {
+					CreateMonitor: &catalog.CreateMonitor{
+						TableName: "test_monitor",
+					},
+				},
+				"test_monitor_new": {
+					CreateMonitor: &catalog.CreateMonitor{
+						TableName: "test_monitor_new",
+					},
+				},
+			},
 		},
 	}
 	var tfState = resourcesState{
@@ -904,6 +937,22 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 					{Attributes: stateInstanceAttributes{ID: "2"}},
 				},
 			},
+			{
+				Type: "databricks_quality_monitor",
+				Mode: "managed",
+				Name: "test_monitor",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "test_monitor"}},
+				},
+			},
+			{
+				Type: "databricks_quality_monitor",
+				Mode: "managed",
+				Name: "test_monitor_old",
+				Instances: []stateResourceInstance{
+					{Attributes: stateInstanceAttributes{ID: "test_monitor_old"}},
+				},
+			},
 		},
 	}
 	err := TerraformToBundle(&tfState, &config)
@@ -951,6 +1000,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 	assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving_new"].ID)
 	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServingEndpoints["test_model_serving_new"].ModifiedStatus)
 
+	assert.Equal(t, "test_monitor", config.Resources.QualityMonitors["test_monitor"].ID)
+	assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
+	assert.Equal(t, "test_monitor_old", config.Resources.QualityMonitors["test_monitor_old"].ID)
+	assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus)
+	assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID)
+	assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus)
 	AssertFullResourceCoverage(t, &config)
 }
 
diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go
index 358279a7a..608f1c795 100644
--- a/bundle/deploy/terraform/interpolate.go
+++ b/bundle/deploy/terraform/interpolate.go
@@ -54,6 +54,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
 				path = dyn.NewPath(dyn.Key("databricks_model_serving")).Append(path[2:]...)
 			case dyn.Key("registered_models"):
 				path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...)
+			case dyn.Key("quality_monitors"):
+				path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...)
 			default:
 				// Trigger "key not found" for unknown resource types.
 				return dyn.GetByPath(root, path)
diff --git a/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go b/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go
new file mode 100644
index 000000000..341df7c22
--- /dev/null
+++ b/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go
@@ -0,0 +1,37 @@
+package tfdyn
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+)
+
+func convertQualityMonitorResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
+	// Normalize the output value to the target schema.
+	vout, diags := convert.Normalize(schema.ResourceQualityMonitor{}, vin)
+	for _, diag := range diags {
+		log.Debugf(ctx, "monitor normalization diagnostic: %s", diag.Summary)
+	}
+	return vout, nil
+}
+
+type qualityMonitorConverter struct{}
+
+func (qualityMonitorConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
+	vout, err := convertQualityMonitorResource(ctx, vin)
+	if err != nil {
+		return err
+	}
+
+	// Add the converted resource to the output.
+	out.QualityMonitor[key] = vout.AsAny()
+
+	return nil
+}
+
+func init() {
+	registerConverter("quality_monitors", qualityMonitorConverter{})
+}
diff --git a/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go b/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go
new file mode 100644
index 000000000..50bfce7a0
--- /dev/null
+++ b/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go
@@ -0,0 +1,46 @@
+package tfdyn
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertQualityMonitor(t *testing.T) {
+	var src = resources.QualityMonitor{
+		CreateMonitor: &catalog.CreateMonitor{
+			TableName:        "test_table_name",
+			AssetsDir:        "assets_dir",
+			OutputSchemaName: "output_schema_name",
+			InferenceLog: &catalog.MonitorInferenceLog{
+				ModelIdCol:    "model_id",
+				PredictionCol: "test_prediction_col",
+				ProblemType:   "PROBLEM_TYPE_CLASSIFICATION",
+			},
+		},
+	}
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = qualityMonitorConverter{}.Convert(ctx, "my_monitor", vin, out)
+
+	require.NoError(t, err)
+	assert.Equal(t, map[string]any{
+		"assets_dir":         "assets_dir",
+		"output_schema_name": "output_schema_name",
+		"table_name":         "test_table_name",
+		"inference_log": map[string]any{
+			"model_id_col":   "model_id",
+			"prediction_col": "test_prediction_col",
+			"problem_type":   "PROBLEM_TYPE_CLASSIFICATION",
+		},
+	}, out.QualityMonitor["my_monitor"])
+}
diff --git a/bundle/tests/quality_monitor/databricks.yml b/bundle/tests/quality_monitor/databricks.yml
new file mode 100644
index 000000000..3abcdfdda
--- /dev/null
+++ b/bundle/tests/quality_monitor/databricks.yml
@@ -0,0 +1,40 @@
+resources:
+  quality_monitors:
+    my_monitor:
+      table_name: "main.test.thing1"
+      assets_dir: "/Shared/provider-test/databricks_monitoring/main.test.thing1"
+      output_schema_name: "test"
+      inference_log:
+        granularities: ["1 day"]
+        timestamp_col:  "timestamp"
+        prediction_col: "prediction"
+        model_id_col:  "model_id"
+        problem_type: "PROBLEM_TYPE_REGRESSION"
+
+targets:
+  development:
+    mode: development
+    resources:
+      quality_monitors:
+        my_monitor:
+          table_name: "main.test.dev"
+
+  staging:
+    resources:
+      quality_monitors:
+        my_monitor:
+          table_name: "main.test.staging"
+          output_schema_name: "staging"
+
+  production:
+    resources:
+      quality_monitors:
+        my_monitor:
+          table_name: "main.test.prod"
+          output_schema_name: "prod"
+          inference_log:
+            granularities: ["1 hour"]
+            timestamp_col:  "timestamp_prod"
+            prediction_col: "prediction_prod"
+            model_id_col:  "model_id_prod"
+            problem_type: "PROBLEM_TYPE_REGRESSION"
diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go
new file mode 100644
index 000000000..d5db05196
--- /dev/null
+++ b/bundle/tests/quality_monitor_test.go
@@ -0,0 +1,59 @@
+package config_tests
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/catalog"
+	"github.com/stretchr/testify/assert"
+)
+
+func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) {
+	assert.Equal(t, "timestamp", p.InferenceLog.TimestampCol)
+	assert.Equal(t, "prediction", p.InferenceLog.PredictionCol)
+	assert.Equal(t, "model_id", p.InferenceLog.ModelIdCol)
+	assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
+}
+
+func TestMonitorTableNames(t *testing.T) {
+	b := loadTarget(t, "./quality_monitor", "development")
+	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
+	assert.Equal(t, b.Config.Bundle.Mode, config.Development)
+
+	p := b.Config.Resources.QualityMonitors["my_monitor"]
+	assert.Equal(t, "main.test.dev", p.TableName)
+	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
+	assert.Equal(t, "test", p.OutputSchemaName)
+
+	assertExpectedMonitor(t, p)
+}
+
+func TestMonitorStaging(t *testing.T) {
+	b := loadTarget(t, "./quality_monitor", "staging")
+	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
+
+	p := b.Config.Resources.QualityMonitors["my_monitor"]
+	assert.Equal(t, "main.test.staging", p.TableName)
+	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
+	assert.Equal(t, "staging", p.OutputSchemaName)
+
+	assertExpectedMonitor(t, p)
+}
+
+func TestMonitorProduction(t *testing.T) {
+	b := loadTarget(t, "./quality_monitor", "production")
+	assert.Len(t, b.Config.Resources.QualityMonitors, 1)
+
+	p := b.Config.Resources.QualityMonitors["my_monitor"]
+	assert.Equal(t, "main.test.prod", p.TableName)
+	assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
+	assert.Equal(t, "prod", p.OutputSchemaName)
+
+	inferenceLog := p.InferenceLog
+	assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities)
+	assert.Equal(t, "timestamp_prod", p.InferenceLog.TimestampCol)
+	assert.Equal(t, "prediction_prod", p.InferenceLog.PredictionCol)
+	assert.Equal(t, "model_id_prod", p.InferenceLog.ModelIdCol)
+	assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
+}
diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go
index dc3ed4da4..595e52edd 100644
--- a/libs/dyn/convert/struct_info.go
+++ b/libs/dyn/convert/struct_info.go
@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/textutil"
 )
 
 // structInfo holds the type information we need to efficiently
@@ -84,6 +85,14 @@ func buildStructInfo(typ reflect.Type) structInfo {
 			}
 
 			name, _, _ := strings.Cut(sf.Tag.Get("json"), ",")
+			if typ.Name() == "QualityMonitor" && name == "-" {
+				urlName, _, _ := strings.Cut(sf.Tag.Get("url"), ",")
+				if urlName == "" || urlName == "-" {
+					name = textutil.CamelToSnakeCase(sf.Name)
+				} else {
+					name = urlName
+				}
+			}
 			if name == "" || name == "-" {
 				continue
 			}
diff --git a/libs/textutil/case.go b/libs/textutil/case.go
new file mode 100644
index 000000000..a8c780591
--- /dev/null
+++ b/libs/textutil/case.go
@@ -0,0 +1,14 @@
+package textutil
+
+import "unicode"
+
+func CamelToSnakeCase(name string) string {
+	var out []rune = make([]rune, 0, len(name)*2)
+	for i, r := range name {
+		if i > 0 && unicode.IsUpper(r) {
+			out = append(out, '_')
+		}
+		out = append(out, unicode.ToLower(r))
+	}
+	return string(out)
+}
diff --git a/libs/textutil/case_test.go b/libs/textutil/case_test.go
new file mode 100644
index 000000000..77b3e0679
--- /dev/null
+++ b/libs/textutil/case_test.go
@@ -0,0 +1,40 @@
+package textutil
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCamelToSnakeCase(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "test",
+			expected: "test",
+		},
+		{
+			input:    "testTest",
+			expected: "test_test",
+		},
+		{
+			input:    "testTestTest",
+			expected: "test_test_test",
+		},
+		{
+			input:    "TestTest",
+			expected: "test_test",
+		},
+		{
+			input:    "TestTestTest",
+			expected: "test_test_test",
+		},
+	}
+
+	for _, c := range cases {
+		output := CamelToSnakeCase(c.input)
+		assert.Equal(t, c.expected, output)
+	}
+}