diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha
index 9a95107e8..562b72fcc 100644
--- a/.codegen/_openapi_sha
+++ b/.codegen/_openapi_sha
@@ -1 +1 @@
-c72c58f97b950fcb924a90ef164bcb10cfcd5ece
\ No newline at end of file
+99f644e72261ef5ecf8d74db20f4b7a1e09723cc
\ No newline at end of file
diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl
index 2f4987b13..33833dfa1 100644
--- a/.codegen/service.go.tmpl
+++ b/.codegen/service.go.tmpl
@@ -179,7 +179,7 @@ func new{{.PascalName}}() *cobra.Command {
 	{{- $wait := and .Wait (and (not .IsCrudRead) (not (eq .SnakeName "get_run"))) -}}
 	{{- $hasRequiredArgs :=  and (not $hasIdPrompt) $hasPosArgs -}}
 	{{- $hasSingleRequiredRequestBodyFieldWithPrompt := and (and $hasIdPrompt $request) (eq 1 (len $request.RequiredRequestBodyFields))  -}}
-	{{- $onlyPathArgsRequiredAsPositionalArguments := and $request (eq (len .RequiredPositionalArguments) (len $request.RequiredPathFields)) -}}
+	{{- $onlyPathArgsRequiredAsPositionalArguments := and .Request (eq (len .RequiredPositionalArguments) (len .Request.RequiredPathFields)) -}}
 	{{- $hasDifferentArgsWithJsonFlag := and (not $onlyPathArgsRequiredAsPositionalArguments) (and $canUseJson (or $request.HasRequiredRequestBodyFields )) -}}
 	{{- $hasCustomArgHandler := or $hasRequiredArgs $hasDifferentArgsWithJsonFlag -}}
 
@@ -218,12 +218,12 @@ func new{{.PascalName}}() *cobra.Command {
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
 		{{- if $hasDifferentArgsWithJsonFlag }}
 		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs({{len $request.RequiredPathFields}})(cmd, args)
+			err := root.ExactArgs({{len .Request.RequiredPathFields}})(cmd, args)
 			if err != nil {
-			{{- if eq 0 (len $request.RequiredPathFields) }}
+			{{- if eq 0 (len .Request.RequiredPathFields) }}
 				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide{{- range $index, $field := $request.RequiredFields}}{{if $index}},{{end}} '{{$field.Name}}'{{end}} in your JSON input")
 			{{- else }}
-				return fmt.Errorf("when --json flag is specified, provide only{{- range $index, $field := $request.RequiredPathFields}}{{if $index}},{{end}} {{$field.ConstantName}}{{end}} as positional arguments. Provide{{- range $index, $field := $request.RequiredRequestBodyFields}}{{if $index}},{{end}} '{{$field.Name}}'{{end}} in your JSON input")
+				return fmt.Errorf("when --json flag is specified, provide only{{- range $index, $field := .Request.RequiredPathFields}}{{if $index}},{{end}} {{$field.ConstantName}}{{end}} as positional arguments. Provide{{- range $index, $field := $request.RequiredRequestBodyFields}}{{if $index}},{{end}} '{{$field.Name}}'{{end}} in your JSON input")
 			{{- end }}
 			}
 			return nil
diff --git a/.gitignore b/.gitignore
index 35aef1764..2f6d0ad8e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,6 +25,7 @@ coverage-acceptance.txt
 __pycache__
 *.pyc
 
+.idea
 .vscode/launch.json
 .vscode/tasks.json
 
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index c0fa960b6..066a84299 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -217,8 +217,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	}
 
 	cloudEnv := os.Getenv("CLOUD_ENV")
-	if config.LocalOnly && cloudEnv != "" {
-		t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	if !isTruePtr(config.Local) && cloudEnv == "" {
+		t.Skipf("Disabled via Local setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	}
+
+	if !isTruePtr(config.Cloud) && cloudEnv != "" {
+		t.Skipf("Disabled via Cloud setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
 	}
 
 	var tmpDir string
@@ -263,9 +267,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 		databricksLocalHost := os.Getenv("DATABRICKS_DEFAULT_HOST")
 
-		if len(config.Server) > 0 || config.RecordRequests {
+		if len(config.Server) > 0 || isTruePtr(config.RecordRequests) {
 			server = testserver.New(t)
-			if config.RecordRequests {
+			if isTruePtr(config.RecordRequests) {
 				requestsPath := filepath.Join(tmpDir, "out.requests.txt")
 				server.RecordRequestsCallback = func(request *testserver.Request) {
 					req := getLoggedRequest(request, config.IncludeRequestHeaders)
@@ -703,3 +707,7 @@ func filterHeaders(h http.Header, includedHeaders []string) http.Header {
 	}
 	return headers
 }
+
+func isTruePtr(value *bool) bool {
+	return value != nil && *value
+}
diff --git a/acceptance/auth/bundle_and_profile/output.txt b/acceptance/auth/bundle_and_profile/output.txt
index 8d2584622..f32d5ba22 100644
--- a/acceptance/auth/bundle_and_profile/output.txt
+++ b/acceptance/auth/bundle_and_profile/output.txt
@@ -11,9 +11,9 @@
 >>> errcode [CLI] current-user me -t dev -p DEFAULT
 "[USERNAME]"
 
-=== Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail
+=== Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle.
 >>> errcode [CLI] current-user me -p profile_name
-Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET]
+Error: Get "https://non-existing-subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted)
 
 Exit code: 1
 
@@ -23,6 +23,65 @@ Error: cannot resolve bundle auth configuration: config host mismatch: profile u
 
 Exit code: 1
 
+=== Bundle commands load bundle configuration when no flags, validation OK
+>>> errcode [CLI] bundle validate
+Name: test-auth
+Target: dev
+Workspace:
+  Host: [DATABRICKS_TARGET]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-auth/dev
+
+Validation OK!
+
+=== Bundle commands load bundle configuration with -t flag, validation OK
+>>> errcode [CLI] bundle validate -t dev
+Name: test-auth
+Target: dev
+Workspace:
+  Host: [DATABRICKS_TARGET]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-auth/dev
+
+Validation OK!
+
+=== Bundle commands load bundle configuration with -p flag, validation not OK (profile host don't match bundle host)
+>>> errcode [CLI] bundle validate -p profile_name
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host https://non-existing-subdomain.databricks.com, but CLI configured to use [DATABRICKS_TARGET]
+
+Name: test-auth
+Target: dev
+Workspace:
+  Host: [DATABRICKS_TARGET]
+
+Found 1 error
+
+Exit code: 1
+
+=== Bundle commands load bundle configuration with -t and -p flag, validation OK (profile host match bundle host)
+>>> errcode [CLI] bundle validate -t dev -p DEFAULT
+Name: test-auth
+Target: dev
+Workspace:
+  Host: [DATABRICKS_TARGET]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-auth/dev
+
+Validation OK!
+
+=== Bundle commands load bundle configuration with -t and -p flag, validation not OK (profile host don't match bundle host)
+>>> errcode [CLI] bundle validate -t prod -p DEFAULT
+Error: cannot resolve bundle auth configuration: config host mismatch: profile uses host [DATABRICKS_TARGET], but CLI configured to use https://bar.com
+
+Name: test-auth
+Target: prod
+Workspace:
+  Host: https://bar.com
+
+Found 1 error
+
+Exit code: 1
+
 === Outside the bundle, no flags
 >>> errcode [CLI] current-user me
 "[USERNAME]"
diff --git a/acceptance/auth/bundle_and_profile/script b/acceptance/auth/bundle_and_profile/script
index b37f5e01d..c078d5316 100644
--- a/acceptance/auth/bundle_and_profile/script
+++ b/acceptance/auth/bundle_and_profile/script
@@ -15,12 +15,27 @@ trace errcode $CLI current-user me -t dev | jq .userName
 title "Inside the bundle, target and matching profile"
 trace errcode $CLI current-user me -t dev -p DEFAULT | jq .userName
 
-title "Inside the bundle, profile flag not matching bundle host. Badness: should use profile from flag instead and not fail"
+title "Inside the bundle, profile flag not matching bundle host. Should use profile from the flag and not the bundle."
 trace errcode $CLI current-user me -p profile_name | jq .userName
 
 title "Inside the bundle, target and not matching profile"
 trace errcode $CLI current-user me -t dev -p profile_name
 
+title "Bundle commands load bundle configuration when no flags, validation OK"
+trace errcode $CLI bundle validate
+
+title "Bundle commands load bundle configuration with -t flag, validation OK"
+trace errcode $CLI bundle validate -t dev
+
+title "Bundle commands load bundle configuration with -p flag, validation not OK (profile host don't match bundle host)"
+trace errcode $CLI bundle validate -p profile_name
+
+title "Bundle commands load bundle configuration with -t and -p flag, validation OK (profile host match bundle host)"
+trace errcode $CLI bundle validate -t dev -p DEFAULT
+
+title "Bundle commands load bundle configuration with -t and -p flag, validation not OK (profile host don't match bundle host)"
+trace errcode $CLI bundle validate -t prod -p DEFAULT
+
 cd ..
 export DATABRICKS_HOST=$host
 title "Outside the bundle, no flags"
diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml
index 1a611ed95..697281ee5 100644
--- a/acceptance/auth/bundle_and_profile/test.toml
+++ b/acceptance/auth/bundle_and_profile/test.toml
@@ -1,5 +1,3 @@
-Badness = "When -p flag is used inside the bundle folder for any CLI commands, CLI use bundle host anyway instead of profile one"
-
 # Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix
 # In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST
 # This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST
@@ -10,3 +8,7 @@ New='DATABRICKS_TARGET'
 [[Repls]]
 Old='DATABRICKS_URL'
 New='DATABRICKS_TARGET'
+
+[[Repls]]
+Old='Get "https://non-existing-subdomain.databricks.com/api/2.0/preview/scim/v2/Me": .*'
+New='Get "https://non-existing-subdomain.databricks.com/api/2.0/preview/scim/v2/Me": (redacted)'
diff --git a/acceptance/auth/credentials/test.toml b/acceptance/auth/credentials/test.toml
index 89438f43a..dc775ea62 100644
--- a/acceptance/auth/credentials/test.toml
+++ b/acceptance/auth/credentials/test.toml
@@ -1,5 +1,3 @@
-LocalOnly = true
-
 RecordRequests = true
 IncludeRequestHeaders = ["Authorization", "User-Agent"]
 
diff --git a/acceptance/bin/diff.py b/acceptance/bin/diff.py
index 0a91d57ce..c1b59655a 100755
--- a/acceptance/bin/diff.py
+++ b/acceptance/bin/diff.py
@@ -43,8 +43,8 @@ def main():
         elif f not in set1:
             print(f"Only in {d2}: {f}")
         else:
-            a = [replaceAll(patterns, x) for x in p1.read_text().splitlines(True)]
-            b = [replaceAll(patterns, x) for x in p2.read_text().splitlines(True)]
+            a = replaceAll(patterns, p1.read_text()).splitlines(True)
+            b = replaceAll(patterns, p2.read_text()).splitlines(True)
             if a != b:
                 p1_str = p1.as_posix()
                 p2_str = p2.as_posix()
diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml
index bb0fcb395..79d1b9ee6 100644
--- a/acceptance/bundle/debug/test.toml
+++ b/acceptance/bundle/debug/test.toml
@@ -1,4 +1,4 @@
-LocalOnly = true
+Cloud = false
 
 [[Repls]]
 # The keys are unsorted and also vary per OS
diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml
index 28b473245..fce46071a 100644
--- a/acceptance/bundle/generate/git_job/test.toml
+++ b/acceptance/bundle/generate/git_job/test.toml
@@ -1,4 +1,4 @@
-LocalOnly = true  # This test needs to run against stubbed Databricks API
+Cloud = false  # This test needs to run against stubbed Databricks API
 
 [[Server]]
 Pattern = "GET /api/2.1/jobs/get"
diff --git a/acceptance/bundle/help/test.toml b/acceptance/bundle/help/test.toml
new file mode 100644
index 000000000..18b1a8841
--- /dev/null
+++ b/acceptance/bundle/help/test.toml
@@ -0,0 +1 @@
+Cloud = false
diff --git a/acceptance/bundle/libraries/maven/.gitignore b/acceptance/bundle/libraries/maven/.gitignore
new file mode 100644
index 000000000..15bcc6dd0
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/.gitignore
@@ -0,0 +1 @@
+.databricks
diff --git a/acceptance/bundle/libraries/maven/databricks.yml b/acceptance/bundle/libraries/maven/databricks.yml
new file mode 100644
index 000000000..785142626
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/databricks.yml
@@ -0,0 +1,27 @@
+bundle:
+  name: maven
+
+
+resources:
+  jobs:
+    testjob:
+      name: test-job
+      tasks:
+        - task_key: dbt
+          spark_jar_task:
+            main_class_name: com.databricks.example.Main
+
+          libraries:
+          - maven:
+              coordinates: org.jsoup:jsoup:1.7.2
+
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            data_security_mode: SINGLE_USER
+            num_workers: 0
+            spark_conf:
+                spark.master: "local[*, 4]"
+                spark.databricks.cluster.profile: singleNode
+            custom_tags:
+              ResourceClass: SingleNode
diff --git a/acceptance/bundle/libraries/maven/out.job.libraries.txt b/acceptance/bundle/libraries/maven/out.job.libraries.txt
new file mode 100644
index 000000000..2b4a0d5f5
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/out.job.libraries.txt
@@ -0,0 +1,7 @@
+[
+  {
+    "maven": {
+      "coordinates": "org.jsoup:jsoup:1.7.2"
+    }
+  }
+]
diff --git a/acceptance/bundle/libraries/maven/output.txt b/acceptance/bundle/libraries/maven/output.txt
new file mode 100644
index 000000000..fd72d8d14
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/output.txt
@@ -0,0 +1,15 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "maven": {
+      "coordinates": "org.jsoup:jsoup:1.7.2"
+    }
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/maven/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
diff --git a/acceptance/bundle/libraries/maven/script b/acceptance/bundle/libraries/maven/script
new file mode 100644
index 000000000..06d1b6409
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/script
@@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json | jq '.resources.jobs.testjob.tasks[0].libraries'
+trace $CLI bundle deploy
+cat out.requests.txt | jq 'select(.path == "/api/2.1/jobs/create")' | jq '.body.tasks[0].libraries'  > out.job.libraries.txt
+rm out.requests.txt
diff --git a/acceptance/bundle/libraries/maven/test.toml b/acceptance/bundle/libraries/maven/test.toml
new file mode 100644
index 000000000..62ba36982
--- /dev/null
+++ b/acceptance/bundle/libraries/maven/test.toml
@@ -0,0 +1,5 @@
+# We run this test only locally for now because we need to figure out how to do
+# bundle destroy on script.cleanup first.
+Cloud = false
+
+RecordRequests = true
diff --git a/acceptance/bundle/libraries/pypi/.gitignore b/acceptance/bundle/libraries/pypi/.gitignore
new file mode 100644
index 000000000..15bcc6dd0
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/.gitignore
@@ -0,0 +1 @@
+.databricks
diff --git a/acceptance/bundle/libraries/pypi/databricks.yml b/acceptance/bundle/libraries/pypi/databricks.yml
new file mode 100644
index 000000000..67f3da254
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/databricks.yml
@@ -0,0 +1,32 @@
+bundle:
+  name: pypi
+
+
+resources:
+  jobs:
+    testjob:
+      name: test-job
+      tasks:
+        - task_key: dbt
+          dbt_task:
+            project_directory: ./
+            profiles_directory: dbt_profiles/
+            commands:
+            - 'dbt deps --target=${bundle.target}'
+            - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
+            - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"'
+
+          libraries:
+          - pypi:
+              package: dbt-databricks>=1.8.0,<2.0.0
+
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            data_security_mode: SINGLE_USER
+            num_workers: 0
+            spark_conf:
+                spark.master: "local[*, 4]"
+                spark.databricks.cluster.profile: singleNode
+            custom_tags:
+              ResourceClass: SingleNode
diff --git a/acceptance/bundle/libraries/pypi/out.job.libraries.txt b/acceptance/bundle/libraries/pypi/out.job.libraries.txt
new file mode 100644
index 000000000..ddc7d84a5
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/out.job.libraries.txt
@@ -0,0 +1,7 @@
+[
+  {
+    "pypi": {
+      "package": "dbt-databricks>=1.8.0,<2.0.0"
+    }
+  }
+]
diff --git a/acceptance/bundle/libraries/pypi/output.txt b/acceptance/bundle/libraries/pypi/output.txt
new file mode 100644
index 000000000..002677d64
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/output.txt
@@ -0,0 +1,15 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "pypi": {
+      "package": "dbt-databricks>=1.8.0,<2.0.0"
+    }
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pypi/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
diff --git a/acceptance/bundle/libraries/pypi/script b/acceptance/bundle/libraries/pypi/script
new file mode 100644
index 000000000..06d1b6409
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/script
@@ -0,0 +1,4 @@
+trace $CLI bundle validate -o json | jq '.resources.jobs.testjob.tasks[0].libraries'
+trace $CLI bundle deploy
+cat out.requests.txt | jq 'select(.path == "/api/2.1/jobs/create")' | jq '.body.tasks[0].libraries'  > out.job.libraries.txt
+rm out.requests.txt
diff --git a/acceptance/bundle/libraries/pypi/test.toml b/acceptance/bundle/libraries/pypi/test.toml
new file mode 100644
index 000000000..62ba36982
--- /dev/null
+++ b/acceptance/bundle/libraries/pypi/test.toml
@@ -0,0 +1,5 @@
+# We run this test only locally for now because we need to figure out how to do
+# bundle destroy on script.cleanup first.
+Cloud = false
+
+RecordRequests = true
diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml
index 77f4ed94b..3839635db 100644
--- a/acceptance/bundle/templates-machinery/helpers-error/test.toml
+++ b/acceptance/bundle/templates-machinery/helpers-error/test.toml
@@ -1,5 +1,4 @@
 Badness = '''(minor) error message is not great: executing "" at <user_name>: error calling user_name:'''
-LocalOnly = true
 
 [[Server]]
 Pattern = "GET /api/2.0/preview/scim/v2/Me"
diff --git a/acceptance/bundle/templates-machinery/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml
deleted file mode 100644
index b76e712fb..000000000
--- a/acceptance/bundle/templates-machinery/helpers/test.toml
+++ /dev/null
@@ -1 +0,0 @@
-LocalOnly = true
diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml
index 9083ecd1b..18b1a8841 100644
--- a/acceptance/bundle/templates-machinery/test.toml
+++ b/acceptance/bundle/templates-machinery/test.toml
@@ -1,2 +1 @@
-# Testing template machinery, by default there is no need to check against cloud.
-LocalOnly = true
+Cloud = false
diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore
index de811f118..231162918 100644
--- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore
+++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/out.gitignore
@@ -1,2 +1,15 @@
+# DABs
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
 
-.databricks
+# dbt
+target/
+dbt_packages/
+dbt_modules/
+logs/
diff --git a/acceptance/bundle/templates/default-python/input.json b/acceptance/bundle/templates/default-python/classic/input.json
similarity index 66%
rename from acceptance/bundle/templates/default-python/input.json
rename to acceptance/bundle/templates/default-python/classic/input.json
index 3e1d79c68..2c4416c00 100644
--- a/acceptance/bundle/templates/default-python/input.json
+++ b/acceptance/bundle/templates/default-python/classic/input.json
@@ -2,5 +2,6 @@
     "project_name": "my_default_python",
     "include_notebook": "yes",
     "include_dlt": "yes",
-    "include_python": "yes"
+    "include_python": "yes",
+    "serverless": "no"
 }
diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff
new file mode 100644
index 000000000..6890badf0
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff
@@ -0,0 +1,54 @@
+--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.job.yml
++++ output/my_default_python/resources/my_default_python.job.yml
+@@ -17,4 +17,5 @@
+       tasks:
+         - task_key: notebook_task
++          job_cluster_key: job_cluster
+           notebook_task:
+             notebook_path: ../src/notebook.ipynb
+@@ -29,17 +30,21 @@
+           depends_on:
+             - task_key: refresh_pipeline
+-          environment_key: default
++          job_cluster_key: job_cluster
+           python_wheel_task:
+             package_name: my_default_python
+             entry_point: main
++          libraries:
++            # By default we just include the .whl file generated for the my_default_python package.
++            # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
++            # for more information on how to add other libraries.
++            - whl: ../dist/*.whl
+ 
+-      # A list of task execution environment specifications that can be referenced by tasks of this job.
+-      environments:
+-        - environment_key: default
+-
+-          # Full documentation of this spec can be found at:
+-          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+-          spec:
+-            client: "1"
+-            dependencies:
+-              - ../dist/*.whl
++      job_clusters:
++        - job_cluster_key: job_cluster
++          new_cluster:
++            spark_version: 15.4.x-scala2.12
++            node_type_id: i3.xlarge
++            data_security_mode: SINGLE_USER
++            autoscale:
++                min_workers: 1
++                max_workers: 4
+--- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml
++++ output/my_default_python/resources/my_default_python.pipeline.yml
+@@ -4,8 +4,7 @@
+     my_default_python_pipeline:
+       name: my_default_python_pipeline
+-      ## Catalog is required for serverless compute
+-      catalog: main
++      ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
++      # catalog: catalog_name
+       target: my_default_python_${bundle.target}
+-      serverless: true
+       libraries:
+         - notebook:
diff --git a/acceptance/bundle/templates/default-python/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt
similarity index 100%
rename from acceptance/bundle/templates/default-python/output.txt
rename to acceptance/bundle/templates/default-python/classic/output.txt
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/__builtins__.pyi
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/__builtins__.pyi
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/__builtins__.pyi
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/extensions.json
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/extensions.json
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/.vscode/settings.json
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/.vscode/settings.json
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/README.md
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/README.md
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/databricks.yml
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/databricks.yml
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/fixtures/.gitkeep
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/fixtures/.gitkeep
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/out.gitignore
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/out.gitignore
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini b/acceptance/bundle/templates/default-python/classic/output/my_default_python/pytest.ini
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/pytest.ini
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/pytest.ini
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt b/acceptance/bundle/templates/default-python/classic/output/my_default_python/requirements-dev.txt
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/requirements-dev.txt
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/requirements-dev.txt
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml
similarity index 97%
rename from acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml
index d9e31691a..7c11e143f 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml
+++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.job.yml
@@ -44,6 +44,7 @@ resources:
           new_cluster:
             spark_version: 15.4.x-scala2.12
             node_type_id: i3.xlarge
+            data_security_mode: SINGLE_USER
             autoscale:
                 min_workers: 1
                 max_workers: 4
diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml
new file mode 100644
index 000000000..4176f765d
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml
@@ -0,0 +1,14 @@
+# The main pipeline for my_default_python
+resources:
+  pipelines:
+    my_default_python_pipeline:
+      name: my_default_python_pipeline
+      ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
+      # catalog: catalog_name
+      target: my_default_python_${bundle.target}
+      libraries:
+        - notebook:
+            path: ../src/dlt_pipeline.ipynb
+
+      configuration:
+        bundle.sourcePath: ${workspace.file_path}/src
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/scratch/README.md
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/README.md
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/scratch/exploration.ipynb
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/scratch/exploration.ipynb
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/setup.py
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/setup.py
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/setup.py
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/dlt_pipeline.ipynb
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/src/dlt_pipeline.ipynb
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/dlt_pipeline.ipynb
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/__init__.py
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/__init__.py
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/__init__.py
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/src/my_default_python/main.py
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/my_default_python/main.py
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/classic/output/my_default_python/src/notebook.ipynb
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/src/notebook.ipynb
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/src/notebook.ipynb
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py
similarity index 100%
rename from acceptance/bundle/templates/default-python/output/my_default_python/tests/main_test.py
rename to acceptance/bundle/templates/default-python/classic/output/my_default_python/tests/main_test.py
diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script
new file mode 100644
index 000000000..7e5524065
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/classic/script
@@ -0,0 +1,13 @@
+trace $CLI bundle init default-python --config-file ./input.json --output-dir output
+
+cd output/my_default_python
+trace $CLI bundle validate -t dev
+trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore
+
+cd ../../
+
+# Calculate the difference from the serverless template
+diff.py $TESTDIR/../serverless/output output/ > out.compare-vs-serverless.diff
diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt
new file mode 100644
index 000000000..30726013b
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt
@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle init default-python --config-file [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> diff.py [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output output/
+--- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python.pipeline.yml
++++ output/my_default_python/resources/my_default_python.pipeline.yml
+@@ -4,6 +4,5 @@
+     my_default_python_pipeline:
+       name: my_default_python_pipeline
+-      ## Catalog is required for serverless compute
+-      catalog: main
++      catalog: customcatalog
+       target: my_default_python_${bundle.target}
+       serverless: true
diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/script b/acceptance/bundle/templates/default-python/serverless-customcatalog/script
new file mode 100644
index 000000000..2d1597c81
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/script
@@ -0,0 +1,4 @@
+trace $CLI bundle init default-python --config-file $TESTDIR/../serverless/input.json --output-dir output
+mv output/my_default_python/.gitignore output/my_default_python/out.gitignore
+trace diff.py $TESTDIR/../serverless/output output/
+rm -fr output
diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/test.toml
new file mode 100644
index 000000000..4029057be
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/test.toml
@@ -0,0 +1,8 @@
+[[Server]]
+Pattern = "GET /api/2.1/unity-catalog/current-metastore-assignment"
+Response.Body = '{"default_catalog_name": "customcatalog"}'
+
+[[Repls]]
+# windows fix
+Old = '\\'
+New = '/'
diff --git a/acceptance/bundle/templates/default-python/serverless/input.json b/acceptance/bundle/templates/default-python/serverless/input.json
new file mode 100644
index 000000000..b1ae9a2ba
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/input.json
@@ -0,0 +1,7 @@
+{
+    "project_name": "my_default_python",
+    "include_notebook": "yes",
+    "include_dlt": "yes",
+    "include_python": "yes",
+    "serverless": "yes"
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt
new file mode 100644
index 000000000..930e756de
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output.txt
@@ -0,0 +1,30 @@
+
+>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output
+
+Welcome to the default Python template for Databricks Asset Bundles!
+Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL]
+
+✨ Your new project has been created in the 'my_default_python' directory!
+
+Please refer to the README.md file for "getting started" instructions.
+See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.
+
+>>> [CLI] bundle validate -t dev
+Name: my_default_python
+Target: dev
+Workspace:
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev
+
+Validation OK!
+
+>>> [CLI] bundle validate -t prod
+Name: my_default_python
+Target: prod
+Workspace:
+  Host: [DATABRICKS_URL]
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod
+
+Validation OK!
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi
new file mode 100644
index 000000000..0edd5181b
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/__builtins__.pyi
@@ -0,0 +1,3 @@
+# Typings for Pylance in Visual Studio Code
+# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md
+from databricks.sdk.runtime import *
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json
new file mode 100644
index 000000000..5d15eba36
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/extensions.json
@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "databricks.databricks",
+        "ms-python.vscode-pylance",
+        "redhat.vscode-yaml"
+    ]
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json
new file mode 100644
index 000000000..8ee87c30d
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.vscode/settings.json
@@ -0,0 +1,16 @@
+{
+    "python.analysis.stubPath": ".vscode",
+    "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+    "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["src"],
+    "files.exclude": {
+        "**/*.egg-info": true,
+        "**/__pycache__": true,
+        ".pytest_cache": true,
+    },
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md
new file mode 100644
index 000000000..10f570bf4
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/README.md
@@ -0,0 +1,49 @@
+# my_default_python
+
+The 'my_default_python' project was generated by using the default-python template.
+
+## Getting started
+
+1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html
+
+2. Authenticate to your Databricks workspace, if you have not done so already:
+    ```
+    $ databricks configure
+    ```
+
+3. To deploy a development copy of this project, type:
+    ```
+    $ databricks bundle deploy --target dev
+    ```
+    (Note that "dev" is the default target, so the `--target` parameter
+    is optional here.)
+
+    This deploys everything that's defined for this project.
+    For example, the default template would deploy a job called
+    `[dev yourname] my_default_python_job` to your workspace.
+    You can find that job by opening your workpace and clicking on **Workflows**.
+
+4. Similarly, to deploy a production copy, type:
+   ```
+   $ databricks bundle deploy --target prod
+   ```
+
+   Note that the default job from the template has a schedule that runs every day
+   (defined in resources/my_default_python.job.yml). The schedule
+   is paused when deploying in development mode (see
+   https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).
+
+5. To run a job or pipeline, use the "run" command:
+   ```
+   $ databricks bundle run
+   ```
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
+
+7. For documentation on the Databricks asset bundles format used
+   for this project, and for CI/CD configuration, see
+   https://docs.databricks.com/dev-tools/bundles/index.html.
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml
new file mode 100644
index 000000000..6080a368f
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/databricks.yml
@@ -0,0 +1,29 @@
+# This is a Databricks asset bundle definition for my_default_python.
+# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.
+bundle:
+  name: my_default_python
+  uuid: [UUID]
+
+include:
+  - resources/*.yml
+
+targets:
+  dev:
+    # The default target uses 'mode: development' to create a development copy.
+    # - Deployed resources get prefixed with '[dev my_user_name]'
+    # - Any job schedules and triggers are paused by default.
+    # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.
+    mode: development
+    default: true
+    workspace:
+      host: [DATABRICKS_URL]
+
+  prod:
+    mode: production
+    workspace:
+      host: [DATABRICKS_URL]
+      # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.
+      root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}
+    permissions:
+      - user_name: [USERNAME]
+        level: CAN_MANAGE
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep
new file mode 100644
index 000000000..fa25d2745
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/fixtures/.gitkeep
@@ -0,0 +1,22 @@
+# Fixtures
+
+This folder is reserved for fixtures, such as CSV files.
+
+Below is an example of how to load fixtures as a data frame:
+
+```
+import pandas as pd
+import os
+
+def get_absolute_path(*relative_parts):
+    if 'dbutils' in globals():
+        base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore
+        path = os.path.normpath(os.path.join(base_dir, *relative_parts))
+        return path if path.startswith("/Workspace") else "/Workspace" + path
+    else:
+        return os.path.join(*relative_parts)
+
+csv_file = get_absolute_path("..", "fixtures", "mycsv.csv")
+df = pd.read_csv(csv_file)
+display(df)
+```
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore
new file mode 100644
index 000000000..0dab7f499
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/out.gitignore
@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini
new file mode 100644
index 000000000..80432c220
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+pythonpath = src
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt
new file mode 100644
index 000000000..0ffbf6aed
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/requirements-dev.txt
@@ -0,0 +1,29 @@
+## requirements-dev.txt: dependencies for local development.
+##
+## For defining dependencies used by jobs in Databricks Workflows, see
+## https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+
+## Add code completion support for DLT
+databricks-dlt
+
+## pytest is the default package used for testing
+pytest
+
+## Dependencies for building wheel files
+setuptools
+wheel
+
+## databricks-connect can be used to run parts of this project locally.
+## See https://docs.databricks.com/dev-tools/databricks-connect.html.
+##
+## databricks-connect is automatically installed if you're using Databricks
+## extension for Visual Studio Code
+## (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html).
+##
+## To manually install databricks-connect, either follow the instructions
+## at https://docs.databricks.com/dev-tools/databricks-connect.html
+## to install the package system-wide. Or uncomment the line below to install a
+## version of db-connect that corresponds to the Databricks Runtime version used
+## for this project.
+#
+# databricks-connect>=15.4,<15.5
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml
new file mode 100644
index 000000000..cc5aeb71c
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.job.yml
@@ -0,0 +1,45 @@
+# The main job for my_default_python.
+resources:
+  jobs:
+    my_default_python_job:
+      name: my_default_python_job
+
+      trigger:
+        # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+        periodic:
+          interval: 1
+          unit: DAYS
+
+      email_notifications:
+        on_failure:
+          - [USERNAME]
+
+      tasks:
+        - task_key: notebook_task
+          notebook_task:
+            notebook_path: ../src/notebook.ipynb
+        
+        - task_key: refresh_pipeline
+          depends_on:
+            - task_key: notebook_task
+          pipeline_task:
+            pipeline_id: ${resources.pipelines.my_default_python_pipeline.id}
+        
+        - task_key: main_task
+          depends_on:
+            - task_key: refresh_pipeline
+          environment_key: default
+          python_wheel_task:
+            package_name: my_default_python
+            entry_point: main
+
+      # A list of task execution environment specifications that can be referenced by tasks of this job.
+      environments:
+        - environment_key: default
+
+          # Full documentation of this spec can be found at:
+          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+          spec:
+            client: "1"
+            dependencies:
+              - ../dist/*.whl
diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml
similarity index 82%
rename from acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml
rename to acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml
index f9e083f4f..6dac62ded 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/resources/my_default_python.pipeline.yml
@@ -3,8 +3,10 @@ resources:
   pipelines:
     my_default_python_pipeline:
       name: my_default_python_pipeline
+      ## Catalog is required for serverless compute
       catalog: main
       target: my_default_python_${bundle.target}
+      serverless: true
       libraries:
         - notebook:
             path: ../src/dlt_pipeline.ipynb
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md
new file mode 100644
index 000000000..e6cfb81b4
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/README.md
@@ -0,0 +1,4 @@
+# scratch
+
+This folder is reserved for personal, exploratory notebooks.
+By default these are not committed to Git, as 'scratch' is listed in .gitignore.
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb
new file mode 100644
index 000000000..a12773d4e
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/scratch/exploration.ipynb
@@ -0,0 +1,61 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "\n",
+    "sys.path.append(\"../src\")\n",
+    "from my_default_python import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "ipynb-notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/setup.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/setup.py
new file mode 100644
index 000000000..548f1035e
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/setup.py
@@ -0,0 +1,41 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import datetime
+import my_default_python
+
+local_version = datetime.datetime.utcnow().strftime("%Y%m%d.%H%M%S")
+
+setup(
+    name="my_default_python",
+    # We use timestamp as Local version identifier (https://peps.python.org/pep-0440/#local-version-identifiers.)
+    # to ensure that changes to wheel package are picked up when used on all-purpose clusters
+    version=my_default_python.__version__ + "+" + local_version,
+    url="https://databricks.com",
+    author="[USERNAME]",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/dlt_pipeline.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/dlt_pipeline.ipynb
new file mode 100644
index 000000000..8a02183e7
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/dlt_pipeline.ipynb
@@ -0,0 +1,90 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# DLT pipeline\n",
+    "\n",
+    "This Delta Live Tables (DLT) definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Import DLT and src/my_default_python\n",
+    "import dlt\n",
+    "import sys\n",
+    "\n",
+    "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n",
+    "from pyspark.sql.functions import expr\n",
+    "from my_default_python import main"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "@dlt.view\n",
+    "def taxi_raw():\n",
+    "    return main.get_taxis(spark)\n",
+    "\n",
+    "\n",
+    "@dlt.table\n",
+    "def filtered_taxis():\n",
+    "    return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "dlt_pipeline",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/__init__.py
new file mode 100644
index 000000000..f102a9cad
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.0.1"
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py
new file mode 100644
index 000000000..5ae344c7e
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/my_default_python/main.py
@@ -0,0 +1,25 @@
+from pyspark.sql import SparkSession, DataFrame
+
+
+def get_taxis(spark: SparkSession) -> DataFrame:
+    return spark.read.table("samples.nyctaxi.trips")
+
+
+# Create a new Databricks Connect session. If this fails,
+# check that you have configured Databricks Connect correctly.
+# See https://docs.databricks.com/dev-tools/databricks-connect.html.
+def get_spark() -> SparkSession:
+    try:
+        from databricks.connect import DatabricksSession
+
+        return DatabricksSession.builder.getOrCreate()
+    except ImportError:
+        return SparkSession.builder.getOrCreate()
+
+
+def main():
+    get_taxis(get_spark()).show(5)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb
new file mode 100644
index 000000000..472ccb219
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/src/notebook.ipynb
@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {},
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "source": [
+    "# Default notebook\n",
+    "\n",
+    "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 0,
+   "metadata": {
+    "application/vnd.databricks.v1+cell": {
+     "cellMetadata": {
+      "byteLimit": 2048000,
+      "rowLimit": 10000
+     },
+     "inputWidgets": {},
+     "nuid": "[UUID]",
+     "showTitle": false,
+     "title": ""
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from my_default_python import main\n",
+    "\n",
+    "main.get_taxis(spark).show(10)"
+   ]
+  }
+ ],
+ "metadata": {
+  "application/vnd.databricks.v1+notebook": {
+   "dashboards": [],
+   "language": "python",
+   "notebookMetadata": {
+    "pythonIndentUnit": 2
+   },
+   "notebookName": "notebook",
+   "widgets": {}
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py
new file mode 100644
index 000000000..dc449154a
--- /dev/null
+++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/tests/main_test.py
@@ -0,0 +1,6 @@
+from my_default_python.main import get_taxis, get_spark
+
+
+def test_main():
+    taxis = get_taxis(get_spark())
+    assert taxis.count() > 5
diff --git a/acceptance/bundle/templates/default-python/script b/acceptance/bundle/templates/default-python/serverless/script
similarity index 100%
rename from acceptance/bundle/templates/default-python/script
rename to acceptance/bundle/templates/default-python/serverless/script
diff --git a/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore
index de811f118..0dab7f499 100644
--- a/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore
+++ b/acceptance/bundle/templates/default-sql/output/my_default_sql/out.gitignore
@@ -1,2 +1,8 @@
-
-.databricks
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
index 984dad604..2099dd498 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
@@ -34,6 +34,7 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi
               "max_workers": 4,
               "min_workers": 1
             },
+            "data_security_mode": "SINGLE_USER",
             "node_type_id": "i3.xlarge",
             "spark_version": "15.4.x-scala2.12"
           }
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
index e8406fd7b..be7254b80 100644
--- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
+++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py
@@ -56,6 +56,7 @@ my_jobs_as_code_job = Job.from_dict(
                 "new_cluster": {
                     "spark_version": "15.4.x-scala2.12",
                     "node_type_id": "i3.xlarge",
+                    "data_security_mode": "SINGLE_USER",
                     "autoscale": {
                         "min_workers": 1,
                         "max_workers": 4,
diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml
index 90539263d..d0d289b5c 100644
--- a/acceptance/bundle/templates/test.toml
+++ b/acceptance/bundle/templates/test.toml
@@ -1,2 +1,2 @@
 # At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on.
-LocalOnly = true
+Cloud = false
diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml
new file mode 100644
index 000000000..0e8c8a384
--- /dev/null
+++ b/acceptance/bundle/test.toml
@@ -0,0 +1,2 @@
+Local = true
+Cloud = true
diff --git a/acceptance/bundle/trampoline/warning_message/databricks.yml b/acceptance/bundle/trampoline/warning_message/databricks.yml
new file mode 100644
index 000000000..c6125f5f0
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message/databricks.yml
@@ -0,0 +1,37 @@
+bundle:
+  name: trampoline_warning_message
+
+targets:
+  dev:
+    mode: development
+    default: true
+
+  prod:
+    resources:
+      clusters:
+        interactive_cluster:
+          spark_version: 14.2.x-cpu-ml-scala2.12
+
+
+resources:
+  clusters:
+    interactive_cluster:
+      cluster_name: jobs-as-code-all-purpose-cluster
+      spark_version: 12.2.x-cpu-ml-scala2.12
+      node_type_id: r5d.8xlarge
+      autotermination_minutes: 30
+      autoscale:
+        min_workers: 1
+        max_workers: 1
+      driver_node_type_id: r5d.8xlarge
+  jobs:
+    whl:
+      name: "wheel-job"
+      tasks:
+        - task_key: test_task
+          python_wheel_task:
+            package_name: my_package
+            entry_point: my_module.my_function
+          existing_cluster_id: ${resources.clusters.interactive_cluster.id}
+          libraries:
+            - whl: ./dist/*.whl
diff --git a/acceptance/bundle/trampoline/warning_message/output.txt b/acceptance/bundle/trampoline/warning_message/output.txt
new file mode 100644
index 000000000..2f7d69e1f
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message/output.txt
@@ -0,0 +1,22 @@
+
+>>> errcode [CLI] bundle validate -t dev
+Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.
+
+Name: trampoline_warning_message
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/dev
+
+Found 1 error
+
+Exit code: 1
+
+>>> errcode [CLI] bundle validate -t prod
+Name: trampoline_warning_message
+Target: prod
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/prod
+
+Validation OK!
diff --git a/acceptance/bundle/trampoline/warning_message/script b/acceptance/bundle/trampoline/warning_message/script
new file mode 100644
index 000000000..ffc151840
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message/script
@@ -0,0 +1,2 @@
+trace errcode $CLI bundle validate -t dev
+trace errcode $CLI bundle validate -t prod
diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/databricks.yml b/acceptance/bundle/trampoline/warning_message_with_new_spark/databricks.yml
new file mode 100644
index 000000000..fa1a05dfb
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/databricks.yml
@@ -0,0 +1,20 @@
+bundle:
+  name: trampoline_warning_message_with_new_spark
+
+targets:
+  dev:
+    mode: development
+    default: true
+
+resources:
+  jobs:
+    whl:
+      name: "wheel-job"
+      tasks:
+        - task_key: test_task
+          python_wheel_task:
+            package_name: my_package
+            entry_point: my_module.my_function
+          existing_cluster_id: "some-test-cluster-id"
+          libraries:
+            - whl: ./dist/*.whl
diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/output.txt b/acceptance/bundle/trampoline/warning_message_with_new_spark/output.txt
new file mode 100644
index 000000000..e311ab9fd
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/output.txt
@@ -0,0 +1,9 @@
+
+>>> errcode [CLI] bundle validate
+Name: trampoline_warning_message_with_new_spark
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message_with_new_spark/dev
+
+Validation OK!
diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/script b/acceptance/bundle/trampoline/warning_message_with_new_spark/script
new file mode 100644
index 000000000..9ecda517f
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/script
@@ -0,0 +1 @@
+trace errcode $CLI bundle validate
diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml
new file mode 100644
index 000000000..4e52dbb5e
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml
@@ -0,0 +1,16 @@
+# Since we use existing cluster id value which is not available in cloud envs, we need to stub the request
+# and run this test only locally
+Cloud = false
+
+[[Server]]
+Pattern = "GET /api/2.1/clusters/get"
+Response.Body = '''
+{
+    "cluster_id": "some-cluster-id",
+    "state": "RUNNING",
+    "spark_version": "13.3.x-scala2.12",
+    "node_type_id": "Standard_DS3_v2",
+    "driver_node_type_id": "Standard_DS3_v2",
+    "cluster_name": "some-cluster-name",
+    "spark_context_id": 12345
+}'''
diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/databricks.yml b/acceptance/bundle/trampoline/warning_message_with_old_spark/databricks.yml
new file mode 100644
index 000000000..864c0f3fe
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/databricks.yml
@@ -0,0 +1,20 @@
+bundle:
+  name: trampoline_warning_message_with_old_spark
+
+targets:
+  dev:
+    mode: development
+    default: true
+
+resources:
+  jobs:
+    whl:
+      name: "wheel-job"
+      tasks:
+        - task_key: test_task
+          python_wheel_task:
+            package_name: my_package
+            entry_point: my_module.my_function
+          existing_cluster_id: "some-test-cluster-id"
+          libraries:
+            - whl: ./dist/*.whl
diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt b/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt
new file mode 100644
index 000000000..551cd17bc
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt
@@ -0,0 +1,13 @@
+
+>>> errcode [CLI] bundle validate
+Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.
+
+Name: trampoline_warning_message_with_old_spark
+Target: dev
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message_with_old_spark/dev
+
+Found 1 error
+
+Exit code: 1
diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/script b/acceptance/bundle/trampoline/warning_message_with_old_spark/script
new file mode 100644
index 000000000..9ecda517f
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/script
@@ -0,0 +1 @@
+trace errcode $CLI bundle validate
diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml
new file mode 100644
index 000000000..09021bfc0
--- /dev/null
+++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml
@@ -0,0 +1,16 @@
+# Since we use existing cluster id value which is not available in cloud envs, we need to stub the request
+# and run this test only locally
+Cloud = false
+
+[[Server]]
+Pattern = "GET /api/2.1/clusters/get"
+Response.Body = '''
+{
+    "cluster_id": "some-cluster-id",
+    "state": "RUNNING",
+    "spark_version": "7.3.x-scala2.12",
+    "node_type_id": "Standard_DS3_v2",
+    "driver_node_type_id": "Standard_DS3_v2",
+    "cluster_name": "some-cluster-name",
+    "spark_context_id": 12345
+}'''
diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml
index 32398e828..8ed716ad0 100644
--- a/acceptance/bundle/variables/test.toml
+++ b/acceptance/bundle/variables/test.toml
@@ -1,3 +1,3 @@
 # The tests here intend to test variable interpolation via "bundle validate".
 # Even though "bundle validate" does a few API calls, that's not the focus there.
-LocalOnly = true
+Cloud = false
diff --git a/acceptance/cmd/workspace/apps/input.json b/acceptance/cmd/workspace/apps/input.json
new file mode 100644
index 000000000..76f3e589c
--- /dev/null
+++ b/acceptance/cmd/workspace/apps/input.json
@@ -0,0 +1,14 @@
+{
+    "description": "My app description.",
+    "resources": [
+        {
+            "name": "api-key",
+            "description": "API key for external service.",
+            "secret": {
+                "scope": "my-scope",
+                "key": "my-key",
+                "permission": "READ"
+            }
+        }
+    ]
+}
diff --git a/acceptance/cmd/workspace/apps/out.requests.txt b/acceptance/cmd/workspace/apps/out.requests.txt
new file mode 100644
index 000000000..04891dc74
--- /dev/null
+++ b/acceptance/cmd/workspace/apps/out.requests.txt
@@ -0,0 +1,19 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.0/apps/test-name",
+  "body": {
+    "description": "My app description.",
+    "name": "",
+    "resources": [
+      {
+        "description": "API key for external service.",
+        "name": "api-key",
+        "secret": {
+          "key": "my-key",
+          "permission": "READ",
+          "scope": "my-scope"
+        }
+      }
+    ]
+  }
+}
diff --git a/acceptance/cmd/workspace/apps/output.txt b/acceptance/cmd/workspace/apps/output.txt
new file mode 100644
index 000000000..4d9f80f44
--- /dev/null
+++ b/acceptance/cmd/workspace/apps/output.txt
@@ -0,0 +1,49 @@
+
+=== Apps update with correct input
+>>> [CLI] apps update test-name --json @input.json
+{
+  "app_status": {
+    "message":"Application is running.",
+    "state":"DEPLOYING"
+  },
+  "compute_status": {
+    "message":"App compute is active.",
+    "state":"ERROR"
+  },
+  "description":"My app description.",
+  "id":"12345",
+  "name":"test-name",
+  "resources": [
+    {
+      "description":"API key for external service.",
+      "name":"api-key",
+      "secret": {
+        "key":"my-key",
+        "permission":"READ",
+        "scope":"my-scope"
+      }
+    }
+  ],
+  "url":"test-name-123.cloud.databricksapps.com"
+}
+
+=== Apps update with missing parameter
+>>> [CLI] apps update --json @input.json
+Error: accepts 1 arg(s), received 0
+
+Usage:
+  databricks apps update NAME [flags]
+
+Flags:
+      --description string   The description of the app.
+  -h, --help                 help for update
+      --json JSON            either inline JSON string or @path/to/file.json with request body (default JSON (0 bytes))
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+
+
+Exit code: 1
diff --git a/acceptance/cmd/workspace/apps/script b/acceptance/cmd/workspace/apps/script
new file mode 100644
index 000000000..221ffc4c0
--- /dev/null
+++ b/acceptance/cmd/workspace/apps/script
@@ -0,0 +1,5 @@
+title "Apps update with correct input"
+trace $CLI apps update test-name --json @input.json
+
+title "Apps update with missing parameter"
+trace $CLI apps update --json @input.json
diff --git a/acceptance/cmd/workspace/apps/test.toml b/acceptance/cmd/workspace/apps/test.toml
new file mode 100644
index 000000000..972ae1c50
--- /dev/null
+++ b/acceptance/cmd/workspace/apps/test.toml
@@ -0,0 +1,30 @@
+RecordRequests = true
+
+[[Server]]
+Pattern = "PATCH /api/2.0/apps/test-name"
+Response.Body = '''
+{
+  "name": "test-name",
+  "description": "My app description.",
+  "compute_status": {
+    "state": "ERROR",
+    "message": "App compute is active."
+  },
+  "app_status": {
+    "state": "DEPLOYING",
+    "message": "Application is running."
+  },
+  "url": "test-name-123.cloud.databricksapps.com",
+  "resources": [
+    {
+      "name": "api-key",
+      "description": "API key for external service.",
+      "secret": {
+        "scope": "my-scope",
+        "key": "my-key",
+        "permission": "READ"
+      }
+    }
+  ],
+  "id": "12345"
+}'''
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index ec0d1baee..4edfee69d 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -18,14 +18,17 @@ const configFilename = "test.toml"
 
 type TestConfig struct {
 	// Place to describe what's wrong with this test. Does not affect how the test is run.
-	Badness string
+	Badness *string
 
 	// Which OSes the test is enabled on. Each string is compared against runtime.GOOS.
 	// If absent, default to true.
 	GOOS map[string]bool
 
-	// If true, do not run this test against cloud environment
-	LocalOnly bool
+	// If true, run this test when running locally with a testserver
+	Local *bool
+
+	// If true, run this test when running with cloud env configured
+	Cloud *bool
 
 	// List of additional replacements to apply on this test.
 	// Old is a regexp, New is a replacement expression.
@@ -44,7 +47,7 @@ type TestConfig struct {
 
 	// Record the requests made to the server and write them as output to
 	// out.requests.txt
-	RecordRequests bool
+	RecordRequests *bool
 
 	// List of request headers to include when recording requests.
 	IncludeRequestHeaders []string
@@ -102,7 +105,7 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
 
 	for _, cfgName := range configs[1:] {
 		cfg := DoLoadConfig(t, cfgName)
-		err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice)
+		err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithoutDereference, mergo.WithAppendSlice)
 		if err != nil {
 			t.Fatalf("Error during config merge: %s: %s", cfgName, err)
 		}
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
index 43ad1e85b..8fc7b3cac 100644
--- a/acceptance/selftest/server/test.toml
+++ b/acceptance/selftest/server/test.toml
@@ -1,4 +1,3 @@
-LocalOnly = true
 RecordRequests = true
 
 [[Server]]
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
deleted file mode 100644
index b76e712fb..000000000
--- a/acceptance/selftest/test.toml
+++ /dev/null
@@ -1 +0,0 @@
-LocalOnly = true
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index 4fc3108d2..402e3ca5f 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -20,6 +20,12 @@ var testUser = iam.User{
 	UserName: "tester@databricks.com",
 }
 
+var testMetastore = catalog.MetastoreAssignment{
+	DefaultCatalogName: "hive_metastore",
+	MetastoreId:        "120efa64-9b68-46ba-be38-f319458430d2",
+	WorkspaceId:        470123456789500,
+}
+
 func AddHandlers(server *testserver.Server) {
 	server.Handle("GET", "/api/2.0/policies/clusters/list", func(req testserver.Request) any {
 		return compute.ListPoliciesResponse{
@@ -106,9 +112,7 @@ func AddHandlers(server *testserver.Server) {
 	})
 
 	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(req testserver.Request) any {
-		return catalog.MetastoreAssignment{
-			DefaultCatalogName: "main",
-		}
+		return testMetastore
 	})
 
 	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(req testserver.Request) any {
diff --git a/acceptance/terraform/test.toml b/acceptance/terraform/test.toml
index a6849e30f..9fbd70943 100644
--- a/acceptance/terraform/test.toml
+++ b/acceptance/terraform/test.toml
@@ -1,3 +1,6 @@
+Local = true
+Cloud = true
+
 [[Repls]]
 Old = 'Read complete after [^\s]+'
 New = 'Read complete after (redacted)'
diff --git a/acceptance/test.toml b/acceptance/test.toml
new file mode 100644
index 000000000..0a009f397
--- /dev/null
+++ b/acceptance/test.toml
@@ -0,0 +1,3 @@
+# Default settings that apply to all tests unless overriden by test.toml files in inner directories.
+Local = true
+Cloud = false
diff --git a/acceptance/workspace/jobs/create-error/test.toml b/acceptance/workspace/jobs/create-error/test.toml
index b45bf77e5..a7b86accb 100644
--- a/acceptance/workspace/jobs/create-error/test.toml
+++ b/acceptance/workspace/jobs/create-error/test.toml
@@ -1,4 +1,3 @@
-LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 
 [[Server]]
diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml
index 1fd9b3cec..f08bc0e63 100644
--- a/acceptance/workspace/jobs/create/test.toml
+++ b/acceptance/workspace/jobs/create/test.toml
@@ -1,4 +1,3 @@
-LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 IncludeRequestHeaders = ["Authorization", "User-Agent"]
 
diff --git a/bundle/artifacts/autodetect.go b/bundle/artifacts/autodetect.go
deleted file mode 100644
index c8d235616..000000000
--- a/bundle/artifacts/autodetect.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package artifacts
-
-import (
-	"context"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/artifacts/whl"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/log"
-)
-
-func DetectPackages() bundle.Mutator {
-	return &autodetect{}
-}
-
-type autodetect struct{}
-
-func (m *autodetect) Name() string {
-	return "artifacts.DetectPackages"
-}
-
-func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	// If artifacts section explicitly defined, do not try to auto detect packages
-	if b.Config.Artifacts != nil {
-		log.Debugf(ctx, "artifacts block is defined, skipping auto-detecting")
-		return nil
-	}
-
-	return bundle.Apply(ctx, b, bundle.Seq(
-		whl.DetectPackage(),
-	))
-}
diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go
index 202ea12bc..9eead83b7 100644
--- a/bundle/artifacts/whl/autodetect.go
+++ b/bundle/artifacts/whl/autodetect.go
@@ -2,11 +2,8 @@ package whl
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"time"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
@@ -26,11 +23,17 @@ func (m *detectPkg) Name() string {
 }
 
 func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	if b.Config.Artifacts != nil {
+		log.Debugf(ctx, "artifacts block is defined, skipping auto-detecting")
+		return nil
+	}
+
 	tasks := libraries.FindTasksWithLocalLibraries(b)
 	if len(tasks) == 0 {
 		log.Infof(ctx, "No local tasks in databricks.yml config, skipping auto detect")
 		return nil
 	}
+
 	log.Infof(ctx, "Detecting Python wheel project...")
 
 	// checking if there is setup.py in the bundle root
@@ -42,39 +45,18 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic
 	}
 
 	log.Infof(ctx, "Found Python wheel project at %s", b.BundleRootPath)
-	module := extractModuleName(setupPy)
-
-	if b.Config.Artifacts == nil {
-		b.Config.Artifacts = make(map[string]*config.Artifact)
-	}
 
 	pkgPath, err := filepath.Abs(b.BundleRootPath)
 	if err != nil {
 		return diag.FromErr(err)
 	}
-	b.Config.Artifacts[module] = &config.Artifact{
+
+	b.Config.Artifacts = make(map[string]*config.Artifact)
+	b.Config.Artifacts["python_artifact"] = &config.Artifact{
 		Path: pkgPath,
 		Type: config.ArtifactPythonWheel,
+		// BuildCommand will be set by bundle/artifacts/whl/infer.go to "python3 setup.py bdist_wheel"
 	}
 
 	return nil
 }
-
-func extractModuleName(setupPy string) string {
-	bytes, err := os.ReadFile(setupPy)
-	if err != nil {
-		return randomName()
-	}
-
-	content := string(bytes)
-	r := regexp.MustCompile(`name=['"](.*)['"]`)
-	matches := r.FindStringSubmatch(content)
-	if len(matches) == 0 {
-		return randomName()
-	}
-	return matches[1]
-}
-
-func randomName() string {
-	return fmt.Sprintf("artifact%d", time.Now().Unix())
-}
diff --git a/bundle/artifacts/whl/autodetect_test.go b/bundle/artifacts/whl/autodetect_test.go
deleted file mode 100644
index b53289b2a..000000000
--- a/bundle/artifacts/whl/autodetect_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package whl
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestExtractModuleName(t *testing.T) {
-	moduleName := extractModuleName("./testdata/setup.py")
-	assert.Equal(t, "my_test_code", moduleName)
-}
-
-func TestExtractModuleNameMinimal(t *testing.T) {
-	moduleName := extractModuleName("./testdata/setup_minimal.py")
-	assert.Equal(t, "my_test_code", moduleName)
-}
-
-func TestExtractModuleNameIncorrect(t *testing.T) {
-	moduleName := extractModuleName("./testdata/setup_incorrect.py")
-	assert.Contains(t, moduleName, "artifact")
-}
diff --git a/bundle/docsgen/templates/reference.md b/bundle/docsgen/templates/reference.md
index 345afc509..38072f70a 100644
--- a/bundle/docsgen/templates/reference.md
+++ b/bundle/docsgen/templates/reference.md
@@ -1,11 +1,13 @@
 ---
-description: Configuration reference for databricks.yml
+description: 'Configuration reference for databricks.yml'
+last_update:
+  date: 2025-02-14
 ---
 
-<!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
+<!--DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli-->
 
 # Configuration reference
 
-This article provides reference for keys supported by <DABS> configuration (YAML). See [_](/dev-tools/bundles/index.md).
+This article provides reference for keys supported by :re[DABS] configuration (YAML). See [\_](/dev-tools/bundles/index.md).
 
-For complete bundle examples, see [_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples).
+For complete bundle examples, see [\_](/dev-tools/bundles/resource-examples.md) and the [bundle-examples GitHub repository](https://github.com/databricks/bundle-examples).
diff --git a/bundle/docsgen/templates/resources.md b/bundle/docsgen/templates/resources.md
index e9a6c8c5b..c6ba3cd88 100644
--- a/bundle/docsgen/templates/resources.md
+++ b/bundle/docsgen/templates/resources.md
@@ -1,71 +1,124 @@
 ---
-description: Learn about resources supported by Databricks Asset Bundles and how to configure them.
+description: 'Learn about resources supported by Databricks Asset Bundles and how to configure them.'
+last_update:
+  date: 2025-02-14
 ---
 
 <!-- DO NOT EDIT. This file is autogenerated with https://github.com/databricks/cli -->
 
-# <DABS> resources
+# :re[DABS] resources
 
-<DABS> allows you to specify information about the <Databricks> resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).
+:re[DABS] allows you to specify information about the :re[Databricks] resources used by the bundle in the `resources` mapping in the bundle configuration. See [resources mapping](/dev-tools/bundles/settings.md#resources) and [resources key reference](/dev-tools/bundles/reference.md#resources).
 
-This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [_](/dev-tools/bundles/resource-examples.md).
+This article outlines supported resource types for bundles and provides details and an example for each supported type. For additional examples, see [\_](/dev-tools/bundles/resource-examples.md).
 
-## <a id="resource-types"></a> Supported resources
+:::tip
+
+To generate YAML for any existing resource, use the `databricks bundle generate` command. See [\_](/dev-tools/cli/bundle-commands.md#generate).
+
+:::
+
+## <a id="resource-types"></a>Supported resources
 
 The following table lists supported resource types for bundles. Some resources can be created by defining them in a bundle and deploying the bundle, and some resources only support referencing an existing resource to include in the bundle.
 
-Resources are defined using the corresponding [Databricks REST API](/api/workspace/introduction) object's create operation request payload, where the object's supported fields, expressed as YAML, are the resource's supported properties. Links to documentation for each resource's corresponding payloads are listed in the table.
+Resources are defined using the corresponding [Databricks REST API](https://docs.databricks.com/api/workspace/introduction) object’s create operation request payload, where the object’s supported fields, expressed as YAML, are the resource’s supported properties. Links to documentation for each resource’s corresponding payloads are listed in the table.
 
-.. tip:: The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files.
+:::tip
 
+The `databricks bundle validate` command returns warnings if unknown resource properties are found in bundle configuration files.
 
-.. list-table::
-    :header-rows: 1
+:::
 
-    * - Resource
-      - Create support
-      - Corresponding REST API object
+::::aws-azure
 
-    * - [cluster](#cluster)
-      - ✓
-      - [Cluster object](/api/workspace/clusters/create)
+:::list-table
 
-    * - [dashboard](#dashboard)
-      -
-      - [Dashboard object](/api/workspace/lakeview/create)
+- - Resource
+  - Create support
+  - Corresponding REST API object
+- - [app](#apps)
+  - ✓
+  - [App object](https://docs.databricks.com/api/workspace/apps/create)
+- - [cluster](#clusters)
+  - ✓
+  - [Cluster object](https://docs.databricks.com/api/workspace/clusters/create)
+- - [dashboard](#dashboards)
+  -
+  - [Dashboard object](https://docs.databricks.com/api/workspace/lakeview/create)
+- - [experiment](#experiments)
+  - ✓
+  - [Experiment object](https://docs.databricks.com/api/workspace/experiments/createexperiment)
+- - [job](#job)
+  - ✓
+  - [Job object](https://docs.databricks.com/api/workspace/jobs/create)
+- - [model (legacy)](#models)
+  - ✓
+  - [Model (legacy) object](https://docs.databricks.com/api/workspace/modelregistry/createmodel)
+- - [model_serving_endpoint](#model_serving_endpoints)
+  - ✓
+  - [Model serving endpoint object](https://docs.databricks.com/api/workspace/servingendpoints/create)
+- - [pipeline](#pipeline)
+  - ✓
+  - [Pipeline object](https://docs.databricks.com/api/workspace/pipelines/create)
+- - [quality_monitor](#quality_monitors)
+  - ✓
+  - [Quality monitor object](https://docs.databricks.com/api/workspace/qualitymonitors/create)
+- - [registered_model](#registered_models) (:re[UC])
+  - ✓
+  - [Registered model object](https://docs.databricks.com/api/workspace/registeredmodels/create)
+- - [schema](#schemas) (:re[UC])
+  - ✓
+  - [Schema object](https://docs.databricks.com/api/workspace/schemas/create)
+- - [volume](#volumes) (:re[UC])
+  - ✓
+  - [Volume object](https://docs.databricks.com/api/workspace/volumes/create)
 
-    * - [experiment](#experiment)
-      - ✓
-      - [Experiment object](/api/workspace/experiments/createexperiment)
+:::
 
-    * - [job](#job)
-      - ✓
-      - [Job object](/api/workspace/jobs/create)
+::::
 
-    * - [model (legacy)](#model-legacy)
-      - ✓
-      - [Model (legacy) object](/api/workspace/modelregistry/createmodel)
+::::gcp
 
-    * - [model_serving_endpoint](#model-serving-endpoint)
-      - ✓
-      - [Model serving endpoint object](/api/workspace/servingendpoints/create)
+:::list-table
 
-    * - [pipeline](#pipeline)
-      - ✓
-      - [Pipeline object](/api/workspace/pipelines/create)
+- - Resource
+  - Create support
+  - Corresponding REST API object
+- - [cluster](#clusters)
+  - ✓
+  - [Cluster object](https://docs.databricks.com/api/workspace/clusters/create)
+- - [dashboard](#dashboards)
+  -
+  - [Dashboard object](https://docs.databricks.com/api/workspace/lakeview/create)
+- - [experiment](#experiments)
+  - ✓
+  - [Experiment object](https://docs.databricks.com/api/workspace/experiments/createexperiment)
+- - [job](#jobs)
+  - ✓
+  - [Job object](https://docs.databricks.com/api/workspace/jobs/create)
+- - [model (legacy)](#models)
+  - ✓
+  - [Model (legacy) object](https://docs.databricks.com/api/workspace/modelregistry/createmodel)
+- - [model_serving_endpoint](#model_serving_endpoints)
+  - ✓
+  - [Model serving endpoint object](https://docs.databricks.com/api/workspace/servingendpoints/create)
+- - [pipeline](#pipelines)
+  - ✓
+  - [Pipeline object]](https://docs.databricks.com/api/workspace/pipelines/create)
+- - [quality_monitor](#quality_monitors)
+  - ✓
+  - [Quality monitor object](https://docs.databricks.com/api/workspace/qualitymonitors/create)
+- - [registered_model](#registered_models) (:re[UC])
+  - ✓
+  - [Registered model object](https://docs.databricks.com/api/workspace/registeredmodels/create)
+- - [schema](#schemas) (:re[UC])
+  - ✓
+  - [Schema object](https://docs.databricks.com/api/workspace/schemas/create)
+- - [volume](#volumes) (:re[UC])
+  - ✓
+  - [Volume object](https://docs.databricks.com/api/workspace/volumes/create)
 
-    * - [quality_monitor](#quality-monitor)
-      - ✓
-      - [Quality monitor object](/api/workspace/qualitymonitors/create)
+:::
 
-    * - [registered_model](#registered-model) (<UC>)
-      - ✓
-      - [Registered model object](/api/workspace/registeredmodels/create)
-
-    * - [schema](#schema) (<UC>)
-      - ✓
-      - [Schema object](/api/workspace/schemas/create)
-
-    * - [volume](#volume) (<UC>)
-      - ✓
-      - [Volume object](/api/workspace/volumes/create)
+::::
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
index c10f43b04..e658f6e53 100644
--- a/bundle/internal/schema/annotations.yml
+++ b/bundle/internal/schema/annotations.yml
@@ -1,25 +1,25 @@
 github.com/databricks/cli/bundle/config.Artifact:
   "build":
     "description": |-
-      An optional set of non-default build commands to run locally before deployment.
+      An optional set of build commands to run locally before deployment.
   "executable":
     "description": |-
       The executable type. Valid values are `bash`, `sh`, and `cmd`.
   "files":
     "description": |-
-      The source files for the artifact.
+      The relative or absolute path to the built artifact files.
   "path":
     "description": |-
-      The location where the built artifact will be saved.
+      The local path of the directory for the artifact.
   "type":
     "description": |-
-      Required. The type of the artifact.
+      Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.
     "markdown_description": |-
-      Required. The type of the artifact. Valid values are `whl`.
+      Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.
 github.com/databricks/cli/bundle/config.ArtifactFile:
   "source":
     "description": |-
-      Required. The path of the files used to build the artifact.
+      Required. The artifact source file.
 github.com/databricks/cli/bundle/config.Bundle:
   "cluster_id":
     "description": |-
@@ -28,7 +28,7 @@ github.com/databricks/cli/bundle/config.Bundle:
       The ID of a cluster to use to run the bundle. See [_](/dev-tools/bundles/settings.md#cluster_id).
   "compute_id":
     "description": |-
-      PLACEHOLDER
+      Deprecated. The ID of the compute to use to run the bundle.
   "databricks_cli_version":
     "description": |-
       The Databricks CLI version to use for the bundle.
@@ -141,62 +141,64 @@ github.com/databricks/cli/bundle/config.Python:
 github.com/databricks/cli/bundle/config.Resources:
   "apps":
     "description": |-
-      PLACEHOLDER
+      The app resource defines a Databricks app.
+    "markdown_description": |-
+      The app resource defines a [Databricks app](/api/workspace/apps/create). For information about Databricks Apps, see [_](/dev-tools/databricks-apps/index.md).
   "clusters":
     "description": |-
       The cluster definitions for the bundle, where each key is the name of a cluster.
     "markdown_description": |-
-      The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters)
+      The cluster definitions for the bundle, where each key is the name of a cluster. See [_](/dev-tools/bundles/resources.md#clusters).
   "dashboards":
     "description": |-
       The dashboard definitions for the bundle, where each key is the name of the dashboard.
     "markdown_description": |-
-      The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards)
+      The dashboard definitions for the bundle, where each key is the name of the dashboard. See [_](/dev-tools/bundles/resources.md#dashboards).
   "experiments":
     "description": |-
       The experiment definitions for the bundle, where each key is the name of the experiment.
     "markdown_description": |-
-      The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments)
+      The experiment definitions for the bundle, where each key is the name of the experiment. See [_](/dev-tools/bundles/resources.md#experiments).
   "jobs":
     "description": |-
       The job definitions for the bundle, where each key is the name of the job.
     "markdown_description": |-
-      The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs)
+      The job definitions for the bundle, where each key is the name of the job. See [_](/dev-tools/bundles/resources.md#jobs).
   "model_serving_endpoints":
     "description": |-
       The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.
     "markdown_description": |-
-      The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints)
+      The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [_](/dev-tools/bundles/resources.md#model_serving_endpoints).
   "models":
     "description": |-
       The model definitions for the bundle, where each key is the name of the model.
     "markdown_description": |-
-      The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models)
+      The model definitions for the bundle, where each key is the name of the model. See [_](/dev-tools/bundles/resources.md#models).
   "pipelines":
     "description": |-
       The pipeline definitions for the bundle, where each key is the name of the pipeline.
     "markdown_description": |-
-      The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines)
+      The pipeline definitions for the bundle, where each key is the name of the pipeline. See [_](/dev-tools/bundles/resources.md#pipelines).
   "quality_monitors":
     "description": |-
       The quality monitor definitions for the bundle, where each key is the name of the quality monitor.
     "markdown_description": |-
-      The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors)
+      The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [_](/dev-tools/bundles/resources.md#quality_monitors).
   "registered_models":
     "description": |-
       The registered model definitions for the bundle, where each key is the name of the <UC> registered model.
     "markdown_description": |-
-      The registered model definitions for the bundle, where each key is the name of the <UC> registered model. See [_](/dev-tools/bundles/resources.md#registered_models)
+      The registered model definitions for the bundle, where each key is the name of the <UC> registered model. See [_](/dev-tools/bundles/resources.md#registered_models).
   "schemas":
     "description": |-
       The schema definitions for the bundle, where each key is the name of the schema.
     "markdown_description": |-
-      The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas)
+      The schema definitions for the bundle, where each key is the name of the schema. See [_](/dev-tools/bundles/resources.md#schemas).
   "volumes":
     "description": |-
       The volume definitions for the bundle, where each key is the name of the volume.
     "markdown_description": |-
-      The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes)
+      The volume definitions for the bundle, where each key is the name of the volume. See [_](/dev-tools/bundles/resources.md#volumes).
 github.com/databricks/cli/bundle/config.Root:
   "artifacts":
     "description": |-
@@ -225,7 +227,7 @@ github.com/databricks/cli/bundle/config.Root:
     "description": |-
       Specifies a list of path globs that contain configuration files to include within the bundle.
     "markdown_description": |-
-      Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include)
+      Specifies a list of path globs that contain configuration files to include within the bundle. See [_](/dev-tools/bundles/settings.md#include).
   "permissions":
     "description": |-
       Defines a permission for a specific entity.
@@ -417,44 +419,44 @@ github.com/databricks/cli/bundle/config/resources.Permission:
 github.com/databricks/cli/bundle/config/variable.Lookup:
   "alert":
     "description": |-
-      PLACEHOLDER
+      The name of the alert for which to retrieve an ID.
   "cluster":
     "description": |-
-      PLACEHOLDER
+      The name of the cluster for which to retrieve an ID.
   "cluster_policy":
     "description": |-
-      PLACEHOLDER
+      The name of the cluster_policy for which to retrieve an ID.
   "dashboard":
     "description": |-
-      PLACEHOLDER
+      The name of the dashboard for which to retrieve an ID.
   "instance_pool":
     "description": |-
-      PLACEHOLDER
+      The name of the instance_pool for which to retrieve an ID.
   "job":
     "description": |-
-      PLACEHOLDER
+      The name of the job for which to retrieve an ID.
   "metastore":
     "description": |-
-      PLACEHOLDER
+      The name of the metastore for which to retrieve an ID.
   "notification_destination":
     "description": |-
-      PLACEHOLDER
+      The name of the notification_destination for which to retrieve an ID.
   "pipeline":
     "description": |-
-      PLACEHOLDER
+      The name of the pipeline for which to retrieve an ID.
   "query":
     "description": |-
-      PLACEHOLDER
+      The name of the query for which to retrieve an ID.
   "service_principal":
     "description": |-
-      PLACEHOLDER
+      The name of the service_principal for which to retrieve an ID.
   "warehouse":
     "description": |-
-      PLACEHOLDER
+      The name of the warehouse for which to retrieve an ID.
 github.com/databricks/cli/bundle/config/variable.TargetVariable:
   "default":
     "description": |-
-      PLACEHOLDER
+      The default value for the variable.
   "description":
     "description": |-
       The description of the variable.
@@ -475,7 +477,7 @@ github.com/databricks/cli/bundle/config/variable.Variable:
       Defines a custom variable for the bundle. See [_](/dev-tools/bundles/settings.md#variables).
   "default":
     "description": |-
-      PLACEHOLDER
+      The default value for the variable.
   "description":
     "description": |-
       The description of the variable
diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml
index 74cd06c66..669ecb9ed 100644
--- a/bundle/internal/schema/annotations_openapi.yml
+++ b/bundle/internal/schema/annotations_openapi.yml
@@ -19,6 +19,9 @@ github.com/databricks/cli/bundle/config/resources.App:
   "description":
     "description": |-
       The description of the app.
+  "id":
+    "description": |-
+      The unique identifier of the app.
   "name":
     "description": |-
       The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.
@@ -67,7 +70,7 @@ github.com/databricks/cli/bundle/config/resources.Cluster:
   "cluster_log_conf":
     "description": |-
       The configuration for delivering spark logs to a long-term storage destination.
-      Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified
+      Three kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified
       for one cluster. If the conf is given, the logs will be delivered to the destination every
       `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
       the destination of executor logs is `$destination/$clusterId/executor`.
@@ -1009,6 +1012,10 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf:
       `{ "s3": { "destination" : "s3://cluster_log_bucket/prefix", "region" : "us-west-2" } }`
       Cluster iam role is used to access s3, please make sure the cluster iam role in
       `instance_profile_arn` has permission to write data to the s3 destination.
+  "volumes":
+    "description": |-
+      destination needs to be provided. e.g.
+      `{ "volumes" : { "destination" : "/Volumes/catalog/schema/volume/cluster_log" } }`
 github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec:
   "apply_policy_default_values":
     "description": |-
@@ -1034,7 +1041,7 @@ github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec:
   "cluster_log_conf":
     "description": |-
       The configuration for delivering spark logs to a long-term storage destination.
-      Two kinds of destinations (dbfs and s3) are supported. Only one destination can be specified
+      Three kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified
       for one cluster. If the conf is given, the logs will be delivered to the destination every
       `5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while
       the destination of executor logs is `$destination/$clusterId/executor`.
@@ -1428,7 +1435,7 @@ github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo:
 github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo:
   "destination":
     "description": |-
-      Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`
+      Unity Catalog volumes file destination, e.g. `/Volumes/catalog/schema/volume/dir/file`
 github.com/databricks/databricks-sdk-go/service/compute.WorkloadType:
   "clients":
     "description": |2-
@@ -2985,7 +2992,7 @@ github.com/databricks/databricks-sdk-go/service/serving.ExternalModel:
       PaLM Config. Only required if the provider is 'palm'.
   "provider":
     "description": |-
-      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.
+      The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', 'palm', and 'custom'.
   "task":
     "description": |-
       The task type of the external model.
diff --git a/bundle/phases/build.go b/bundle/phases/build.go
index 3ddc6b181..cc35983ec 100644
--- a/bundle/phases/build.go
+++ b/bundle/phases/build.go
@@ -3,6 +3,7 @@ package phases
 import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/artifacts"
+	"github.com/databricks/cli/bundle/artifacts/whl"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/scripts"
@@ -14,7 +15,7 @@ func Build() bundle.Mutator {
 		"build",
 		[]bundle.Mutator{
 			scripts.Execute(config.ScriptPreBuild),
-			artifacts.DetectPackages(),
+			whl.DetectPackage(),
 			artifacts.InferMissingProperties(),
 			artifacts.PrepareAll(),
 			artifacts.BuildAll(),
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
index c3c31b58c..4bfbd62fc 100644
--- a/bundle/schema/jsonschema.json
+++ b/bundle/schema/jsonschema.json
@@ -88,6 +88,10 @@
                       "description": {
                         "$ref": "#/$defs/string"
                       },
+                      "id": {
+                        "description": "The unique identifier of the app.",
+                        "$ref": "#/$defs/string"
+                      },
                       "name": {
                         "$ref": "#/$defs/string"
                       },
@@ -160,7 +164,7 @@
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes"
                       },
                       "cluster_log_conf": {
-                        "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
+                        "description": "The configuration for delivering spark logs to a long-term storage destination.\nThree kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                         "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf"
                       },
                       "cluster_name": {
@@ -951,39 +955,51 @@
                     "type": "object",
                     "properties": {
                       "alert": {
+                        "description": "The name of the alert for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "cluster": {
+                        "description": "The name of the cluster for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "cluster_policy": {
+                        "description": "The name of the cluster_policy for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "dashboard": {
+                        "description": "The name of the dashboard for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "instance_pool": {
+                        "description": "The name of the instance_pool for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "job": {
+                        "description": "The name of the job for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "metastore": {
+                        "description": "The name of the metastore for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "notification_destination": {
+                        "description": "The name of the notification_destination for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "pipeline": {
+                        "description": "The name of the pipeline for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "query": {
+                        "description": "The name of the query for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "service_principal": {
+                        "description": "The name of the service_principal for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       },
                       "warehouse": {
+                        "description": "The name of the warehouse for which to retrieve an ID.",
                         "$ref": "#/$defs/string"
                       }
                     },
@@ -1001,6 +1017,7 @@
                     "type": "object",
                     "properties": {
                       "default": {
+                        "description": "The default value for the variable.",
                         "$ref": "#/$defs/interface"
                       },
                       "description": {
@@ -1026,6 +1043,7 @@
                 "description": "Defines a custom variable for the bundle.",
                 "properties": {
                   "default": {
+                    "description": "The default value for the variable.",
                     "$ref": "#/$defs/interface"
                   },
                   "description": {
@@ -1055,7 +1073,7 @@
                   "type": "object",
                   "properties": {
                     "build": {
-                      "description": "An optional set of non-default build commands to run locally before deployment.",
+                      "description": "An optional set of build commands to run locally before deployment.",
                       "$ref": "#/$defs/string"
                     },
                     "executable": {
@@ -1063,17 +1081,17 @@
                       "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType"
                     },
                     "files": {
-                      "description": "The source files for the artifact.",
+                      "description": "The relative or absolute path to the built artifact files.",
                       "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile"
                     },
                     "path": {
-                      "description": "The location where the built artifact will be saved.",
+                      "description": "The local path of the directory for the artifact.",
                       "$ref": "#/$defs/string"
                     },
                     "type": {
-                      "description": "Required. The type of the artifact.",
+                      "description": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.",
                       "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType",
-                      "markdownDescription": "Required. The type of the artifact. Valid values are `whl`."
+                      "markdownDescription": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`."
                     }
                   },
                   "additionalProperties": false,
@@ -1093,7 +1111,7 @@
                   "type": "object",
                   "properties": {
                     "source": {
-                      "description": "Required. The path of the files used to build the artifact.",
+                      "description": "Required. The artifact source file.",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -1122,6 +1140,7 @@
                       "markdownDescription": "The ID of a cluster to use to run the bundle. See [cluster_id](https://docs.databricks.com/dev-tools/bundles/settings.html#cluster_id)."
                     },
                     "compute_id": {
+                      "description": "Deprecated. The ID of the compute to use to run the bundle.",
                       "$ref": "#/$defs/string"
                     },
                     "databricks_cli_version": {
@@ -1367,62 +1386,64 @@
                   "type": "object",
                   "properties": {
                     "apps": {
-                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App"
+                      "description": "The app resource defines a Databricks app.",
+                      "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App",
+                      "markdownDescription": "The app resource defines a [Databricks app](https://docs.databricks.com/api/workspace/apps/create). For information about Databricks Apps, see [link](https://docs.databricks.com/dev-tools/databricks-apps/index.html)."
                     },
                     "clusters": {
                       "description": "The cluster definitions for the bundle, where each key is the name of a cluster.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster",
-                      "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [clusters](https://docs.databricks.com/dev-tools/bundles/resources.html#clusters)"
+                      "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [clusters](https://docs.databricks.com/dev-tools/bundles/resources.html#clusters)."
                     },
                     "dashboards": {
                       "description": "The dashboard definitions for the bundle, where each key is the name of the dashboard.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard",
-                      "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)"
+                      "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)."
                     },
                     "experiments": {
                       "description": "The experiment definitions for the bundle, where each key is the name of the experiment.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment",
-                      "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)"
+                      "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)."
                     },
                     "jobs": {
                       "description": "The job definitions for the bundle, where each key is the name of the job.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job",
-                      "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)"
+                      "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)."
                     },
                     "model_serving_endpoints": {
                       "description": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint",
-                      "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoints](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoints)"
+                      "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoints](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoints)."
                     },
                     "models": {
                       "description": "The model definitions for the bundle, where each key is the name of the model.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel",
-                      "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [models](https://docs.databricks.com/dev-tools/bundles/resources.html#models)"
+                      "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [models](https://docs.databricks.com/dev-tools/bundles/resources.html#models)."
                     },
                     "pipelines": {
                       "description": "The pipeline definitions for the bundle, where each key is the name of the pipeline.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline",
-                      "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipelines](https://docs.databricks.com/dev-tools/bundles/resources.html#pipelines)"
+                      "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipelines](https://docs.databricks.com/dev-tools/bundles/resources.html#pipelines)."
                     },
                     "quality_monitors": {
                       "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor",
-                      "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitors)"
+                      "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitors)."
                     },
                     "registered_models": {
                       "description": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel",
-                      "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model. See [registered_models](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_models)"
+                      "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the \u003cUC\u003e registered model. See [registered_models](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_models)."
                     },
                     "schemas": {
                       "description": "The schema definitions for the bundle, where each key is the name of the schema.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema",
-                      "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)"
+                      "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)."
                     },
                     "volumes": {
                       "description": "The volume definitions for the bundle, where each key is the name of the volume.",
                       "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume",
-                      "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volumes](https://docs.databricks.com/dev-tools/bundles/resources.html#volumes)"
+                      "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volumes](https://docs.databricks.com/dev-tools/bundles/resources.html#volumes)."
                     }
                   },
                   "additionalProperties": false
@@ -2478,6 +2499,10 @@
                     "s3": {
                       "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo"
+                    },
+                    "volumes": {
+                      "description": "destination needs to be provided. e.g.\n`{ \"volumes\" : { \"destination\" : \"/Volumes/catalog/schema/volume/cluster_log\" } }`",
+                      "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo"
                     }
                   },
                   "additionalProperties": false
@@ -2514,7 +2539,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes"
                     },
                     "cluster_log_conf": {
-                      "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
+                      "description": "The configuration for delivering spark logs to a long-term storage destination.\nThree kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf"
                     },
                     "cluster_name": {
@@ -3099,7 +3124,7 @@
                   "type": "object",
                   "properties": {
                     "destination": {
-                      "description": "Unity Catalog Volumes file destination, e.g. `/Volumes/my-init.sh`",
+                      "description": "Unity Catalog volumes file destination, e.g. `/Volumes/catalog/schema/volume/dir/file`",
                       "$ref": "#/$defs/string"
                     }
                   },
@@ -6060,7 +6085,7 @@
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig"
                     },
                     "provider": {
-                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', and 'palm'.",
+                      "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', 'palm', and 'custom'.",
                       "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider"
                     },
                     "task": {
@@ -7296,7 +7321,7 @@
     "include": {
       "description": "Specifies a list of path globs that contain configuration files to include within the bundle.",
       "$ref": "#/$defs/slice/string",
-      "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)"
+      "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)."
     },
     "permissions": {
       "description": "Defines a permission for a specific entity.",
diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go
index 0318df7c9..18fbbb353 100644
--- a/bundle/trampoline/python_dbr_warning.go
+++ b/bundle/trampoline/python_dbr_warning.go
@@ -9,6 +9,7 @@ import (
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/libraries"
 	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/databricks-sdk-go"
 	"golang.org/x/mod/semver"
@@ -60,11 +61,37 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool {
 		}
 
 		if task.ExistingClusterId != "" {
-			version, err := getSparkVersionForCluster(ctx, b.WorkspaceClient(), task.ExistingClusterId)
-			// If there's error getting spark version for cluster, do not mark it as incompatible
-			if err != nil {
-				log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error())
-				return false
+			var version string
+			var err error
+			// If the cluster id is a variable and it's not resolved, it means it references a cluster defined in the same bundle.
+			// So we can get the version from the cluster definition.
+			// It's defined in a form of resources.clusters.<cluster_key>.id
+			if strings.HasPrefix(task.ExistingClusterId, "${") {
+				p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId)
+				if !ok || len(p) < 3 {
+					log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId)
+					return false
+				}
+
+				if p[0].Key() != "resources" || p[1].Key() != "clusters" {
+					log.Warnf(ctx, "incorrect variable reference for cluster id %s", task.ExistingClusterId)
+					return false
+				}
+
+				clusterKey := p[2].Key()
+				cluster, ok := b.Config.Resources.Clusters[clusterKey]
+				if !ok {
+					log.Warnf(ctx, "unable to find cluster with key %s", clusterKey)
+					return false
+				}
+				version = cluster.SparkVersion
+			} else {
+				version, err = getSparkVersionForCluster(ctx, b.WorkspaceClient(), task.ExistingClusterId)
+				// If there's error getting spark version for cluster, do not mark it as incompatible
+				if err != nil {
+					log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error())
+					return false
+				}
 			}
 
 			if lowerThanExpectedVersion(version) {
@@ -82,7 +109,7 @@ func lowerThanExpectedVersion(sparkVersion string) bool {
 		return false
 	}
 
-	if parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99)
+	if len(parts[1]) > 0 && parts[1][0] == 'x' { // treat versions like 13.x as the very latest minor (13.99)
 		parts[1] = "99"
 	}
 
diff --git a/bundle/trampoline/python_dbr_warning_test.go b/bundle/trampoline/python_dbr_warning_test.go
index d293c9477..96fac7329 100644
--- a/bundle/trampoline/python_dbr_warning_test.go
+++ b/bundle/trampoline/python_dbr_warning_test.go
@@ -346,6 +346,7 @@ func TestSparkVersionLowerThanExpected(t *testing.T) {
 		"13.x-rc-scala-2.12":              false,
 		"client.1.10-scala2.12":           false,
 		"latest-stable-gpu-scala2.11":     false,
+		"1.":                              false,
 		"10.4.x-aarch64-photon-scala2.12": true,
 		"10.4.x-scala2.12":                true,
 		"13.0.x-scala2.12":                true,
diff --git a/cmd/account/budget-policy/budget-policy.go b/cmd/account/budget-policy/budget-policy.go
index 28b14ea91..fb9f8e5a6 100755
--- a/cmd/account/budget-policy/budget-policy.go
+++ b/cmd/account/budget-policy/budget-policy.go
@@ -3,8 +3,6 @@
 package budget_policy
 
 import (
-	"fmt"
-
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdio"
 	"github.com/databricks/cli/libs/flags"
@@ -305,6 +303,8 @@ func newUpdate() *cobra.Command {
 	// TODO: short flags
 	cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`)
 
+	// TODO: complex arg: limit_config
+
 	// TODO: array: custom_tags
 	cmd.Flags().StringVar(&updateReq.Policy.PolicyName, "policy-name", updateReq.Policy.PolicyName, `The name of the policy.`)
 
@@ -321,13 +321,6 @@ func newUpdate() *cobra.Command {
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'policy_id' in your JSON input")
-			}
-			return nil
-		}
 		check := root.ExactArgs(1)
 		return check(cmd, args)
 	}
diff --git a/cmd/root/auth.go b/cmd/root/auth.go
index 4fcfbb4d8..e2dac68cc 100644
--- a/cmd/root/auth.go
+++ b/cmd/root/auth.go
@@ -195,6 +195,12 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error {
 		cfg.Profile = profile
 	}
 
+	_, isTargetFlagSet := targetFlagValue(cmd)
+	// If the profile flag is set but the target flag is not, we should skip loading the bundle configuration.
+	if !isTargetFlagSet && hasProfileFlag {
+		cmd.SetContext(SkipLoadBundle(cmd.Context()))
+	}
+
 	ctx := cmd.Context()
 	ctx = context.WithValue(ctx, &configUsed, cfg)
 	cmd.SetContext(ctx)
diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go
index 5842526f3..3037546c3 100644
--- a/cmd/root/bundle.go
+++ b/cmd/root/bundle.go
@@ -14,26 +14,35 @@ import (
 
 // getTarget returns the name of the target to operate in.
 func getTarget(cmd *cobra.Command) (value string) {
+	target, isFlagSet := targetFlagValue(cmd)
+	if isFlagSet {
+		return target
+	}
+
+	// If it's not set, use the environment variable.
+	target, _ = env.Target(cmd.Context())
+	return target
+}
+
+func targetFlagValue(cmd *cobra.Command) (string, bool) {
 	// The command line flag takes precedence.
 	flag := cmd.Flag("target")
 	if flag != nil {
-		value = flag.Value.String()
+		value := flag.Value.String()
 		if value != "" {
-			return
+			return value, true
 		}
 	}
 
 	oldFlag := cmd.Flag("environment")
 	if oldFlag != nil {
-		value = oldFlag.Value.String()
+		value := oldFlag.Value.String()
 		if value != "" {
-			return
+			return value, true
 		}
 	}
 
-	// If it's not set, use the environment variable.
-	target, _ := env.Target(cmd.Context())
-	return target
+	return "", false
 }
 
 func getProfile(cmd *cobra.Command) (value string) {
diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go
index f7c08ece1..6eb85d873 100755
--- a/cmd/workspace/apps/apps.go
+++ b/cmd/workspace/apps/apps.go
@@ -956,13 +956,6 @@ func newUpdate() *cobra.Command {
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input")
-			}
-			return nil
-		}
 		check := root.ExactArgs(1)
 		return check(cmd, args)
 	}
diff --git a/cmd/workspace/genie/genie.go b/cmd/workspace/genie/genie.go
index 25fa9396d..99841637a 100755
--- a/cmd/workspace/genie/genie.go
+++ b/cmd/workspace/genie/genie.go
@@ -40,6 +40,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(newExecuteMessageQuery())
 	cmd.AddCommand(newGetMessage())
 	cmd.AddCommand(newGetMessageQueryResult())
+	cmd.AddCommand(newGetMessageQueryResultByAttachment())
 	cmd.AddCommand(newStartConversation())
 
 	// Apply optional overrides to this command.
@@ -344,6 +345,71 @@ func newGetMessageQueryResult() *cobra.Command {
 	return cmd
 }
 
+// start get-message-query-result-by-attachment command
+
+// Slice with functions to override default command behavior.
+// Functions can be added from the `init()` function in manually curated files in this directory.
+var getMessageQueryResultByAttachmentOverrides []func(
+	*cobra.Command,
+	*dashboards.GenieGetQueryResultByAttachmentRequest,
+)
+
+func newGetMessageQueryResultByAttachment() *cobra.Command {
+	cmd := &cobra.Command{}
+
+	var getMessageQueryResultByAttachmentReq dashboards.GenieGetQueryResultByAttachmentRequest
+
+	// TODO: short flags
+
+	cmd.Use = "get-message-query-result-by-attachment SPACE_ID CONVERSATION_ID MESSAGE_ID ATTACHMENT_ID"
+	cmd.Short = `Get conversation message SQL query result by attachment id.`
+	cmd.Long = `Get conversation message SQL query result by attachment id.
+  
+  Get the result of SQL query by attachment id This is only available if a
+  message has a query attachment and the message status is EXECUTING_QUERY.
+
+  Arguments:
+    SPACE_ID: Genie space ID
+    CONVERSATION_ID: Conversation ID
+    MESSAGE_ID: Message ID
+    ATTACHMENT_ID: Attachment ID`
+
+	cmd.Annotations = make(map[string]string)
+
+	cmd.Args = func(cmd *cobra.Command, args []string) error {
+		check := root.ExactArgs(4)
+		return check(cmd, args)
+	}
+
+	cmd.PreRunE = root.MustWorkspaceClient
+	cmd.RunE = func(cmd *cobra.Command, args []string) (err error) {
+		ctx := cmd.Context()
+		w := root.WorkspaceClient(ctx)
+
+		getMessageQueryResultByAttachmentReq.SpaceId = args[0]
+		getMessageQueryResultByAttachmentReq.ConversationId = args[1]
+		getMessageQueryResultByAttachmentReq.MessageId = args[2]
+		getMessageQueryResultByAttachmentReq.AttachmentId = args[3]
+
+		response, err := w.Genie.GetMessageQueryResultByAttachment(ctx, getMessageQueryResultByAttachmentReq)
+		if err != nil {
+			return err
+		}
+		return cmdio.Render(ctx, response)
+	}
+
+	// Disable completions since they are not applicable.
+	// Can be overridden by manual implementation in `override.go`.
+	cmd.ValidArgsFunction = cobra.NoFileCompletions
+
+	// Apply optional overrides to this command.
+	for _, fn := range getMessageQueryResultByAttachmentOverrides {
+		fn(cmd, &getMessageQueryResultByAttachmentReq)
+	}
+
+	return cmd
+}
+
 // start start-conversation command
 
 // Slice with functions to override default command behavior.
diff --git a/cmd/workspace/lakeview/lakeview.go b/cmd/workspace/lakeview/lakeview.go
index 6686f16da..eb2f5d8fa 100755
--- a/cmd/workspace/lakeview/lakeview.go
+++ b/cmd/workspace/lakeview/lakeview.go
@@ -163,13 +163,6 @@ func newCreateSchedule() *cobra.Command {
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'cron_schedule' in your JSON input")
-			}
-			return nil
-		}
 		check := root.ExactArgs(1)
 		return check(cmd, args)
 	}
@@ -242,13 +235,6 @@ func newCreateSubscription() *cobra.Command {
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'subscriber' in your JSON input")
-			}
-			return nil
-		}
 		check := root.ExactArgs(2)
 		return check(cmd, args)
 	}
@@ -1195,13 +1181,6 @@ func newUpdateSchedule() *cobra.Command {
 	cmd.Annotations = make(map[string]string)
 
 	cmd.Args = func(cmd *cobra.Command, args []string) error {
-		if cmd.Flags().Changed("json") {
-			err := root.ExactArgs(0)(cmd, args)
-			if err != nil {
-				return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'cron_schedule' in your JSON input")
-			}
-			return nil
-		}
 		check := root.ExactArgs(2)
 		return check(cmd, args)
 	}
diff --git a/go.mod b/go.mod
index b4157c61b..6e3c51e79 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0 // MIT
 	github.com/Masterminds/semver/v3 v3.3.1 // MIT
 	github.com/briandowns/spinner v1.23.1 // Apache 2.0
-	github.com/databricks/databricks-sdk-go v0.57.0 // Apache 2.0
+	github.com/databricks/databricks-sdk-go v0.58.1 // Apache 2.0
 	github.com/fatih/color v1.18.0 // MIT
 	github.com/google/uuid v1.6.0 // BSD-3-Clause
 	github.com/gorilla/mux v1.8.1 // BSD 3-Clause
@@ -23,7 +23,7 @@ require (
 	github.com/nwidger/jsoncolor v0.3.2 // MIT
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
-	github.com/spf13/cobra v1.8.1 // Apache 2.0
+	github.com/spf13/cobra v1.9.1 // Apache 2.0
 	github.com/spf13/pflag v1.0.6 // BSD-3-Clause
 	github.com/stretchr/testify v1.10.0 // MIT
 	github.com/wI2L/jsondiff v0.6.1 // MIT
@@ -31,14 +31,13 @@ require (
 	golang.org/x/mod v0.23.0
 	golang.org/x/oauth2 v0.26.0
 	golang.org/x/sync v0.11.0
+	golang.org/x/sys v0.30.0
 	golang.org/x/term v0.29.0
 	golang.org/x/text v0.22.0
 	gopkg.in/ini.v1 v1.67.0 // Apache 2.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require golang.org/x/sys v0.30.0
-
 require (
 	cloud.google.com/go/auth v0.4.2 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
diff --git a/go.sum b/go.sum
index fbf942148..2caabeb95 100644
--- a/go.sum
+++ b/go.sum
@@ -31,11 +31,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo=
 github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/databricks/databricks-sdk-go v0.57.0 h1:Vs3a+Zmg403er4+xpD7ZTQWm7e51d2q3yYEyIIgvtYw=
-github.com/databricks/databricks-sdk-go v0.57.0/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
+github.com/databricks/databricks-sdk-go v0.58.1 h1:dUs9ZmFi7hYiL3NwLSAbxqQu66E3BzwM8EU/wcCTJ10=
+github.com/databricks/databricks-sdk-go v0.58.1/go.mod h1:JpLizplEs+up9/Z4Xf2x++o3sM9eTTWFGzIXAptKJzI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -147,9 +147,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
 github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
diff --git a/integration/bundle/init_default_python_test.go b/integration/bundle/init_default_python_test.go
index 931660032..ca66491ab 100644
--- a/integration/bundle/init_default_python_test.go
+++ b/integration/bundle/init_default_python_test.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/databricks/cli/integration/internal/acc"
@@ -53,6 +54,7 @@ func testDefaultPython(t *testing.T, pythonVersion string) {
 	uniqueProjectId := testutil.RandomName("")
 	ctx, replacements := testdiff.WithReplacementsMap(ctx)
 	replacements.Set(uniqueProjectId, "$UNIQUE_PRJ")
+	replacements.Set(strings.ToLower(uniqueProjectId), "$UNIQUE_PRJ")
 
 	user, err := wt.W.CurrentUser.Me(ctx)
 	require.NoError(t, err)
diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt
index 076e7618f..fe1cc4fac 100644
--- a/integration/bundle/testdata/default_python/bundle_deploy.txt
+++ b/integration/bundle/testdata/default_python/bundle_deploy.txt
@@ -1,4 +1,4 @@
-Building project_name_$UNIQUE_PRJ...
+Building python_artifact...
 Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl...
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files...
 Deploying resources...
diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt
index 450f01c46..968009759 100644
--- a/integration/bundle/testdata/default_python/bundle_summary.txt
+++ b/integration/bundle/testdata/default_python/bundle_summary.txt
@@ -82,6 +82,7 @@
                 "max_workers": 4,
                 "min_workers": 1
               },
+              "data_security_mode": "SINGLE_USER",
               "node_type_id": "i3.xlarge",
               "spark_version": "15.4.x-scala2.12"
             }
diff --git a/libs/cmdgroup/command_test.go b/libs/cmdgroup/command_test.go
index 2c248f09f..20904aad0 100644
--- a/libs/cmdgroup/command_test.go
+++ b/libs/cmdgroup/command_test.go
@@ -41,7 +41,7 @@ func TestCommandFlagGrouping(t *testing.T) {
 	cmd.Flags().BoolP("bool", "b", false, "Bool flag")
 
 	buf := bytes.NewBuffer(nil)
-	cmd.SetOutput(buf)
+	cmd.SetOut(buf)
 	err := cmd.Usage()
 	require.NoError(t, err)
 
diff --git a/libs/daemon/daemon.go b/libs/daemon/daemon.go
index 91914477b..7ab9a6f81 100644
--- a/libs/daemon/daemon.go
+++ b/libs/daemon/daemon.go
@@ -8,8 +8,6 @@ import (
 	"strconv"
 )
 
-const DatabricksCliParentPid = "DATABRICKS_CLI_PARENT_PID"
-
 type Daemon struct {
 	// If provided, the child process's pid will be written in the file at this
 	// path.
@@ -33,22 +31,17 @@ type Daemon struct {
 }
 
 func (d *Daemon) Start() error {
-	cli, err := os.Executable()
-	if err != nil {
-		return err
-	}
-
+	var err error
 	executable := d.Executable
 	if executable == "" {
-		executable = cli
+		// If Executable is not provided, use the current CLI executable.
+		executable, err = os.Executable()
+		if err != nil {
+			return err
+		}
 	}
 
 	d.cmd = exec.Command(executable, d.Args...)
-
-	// Set environment variable so that the child process knows its parent's PID.
-	// In unix systems orphaned processes are automatically re-parented to init (pid 1)
-	// so we cannot rely on os.Getppid() to get the original parent's pid.
-	d.Env = append(d.Env, fmt.Sprintf("%s=%d", DatabricksCliParentPid, os.Getpid()))
 	d.cmd.Env = d.Env
 
 	d.cmd.SysProcAttr = sysProcAttr()
@@ -64,6 +57,7 @@ func (d *Daemon) Start() error {
 			return fmt.Errorf("failed to open log file: %w", err)
 		}
 
+		// The file descriptor for the log file is closed in the [Daemon.Release] method.
 		d.cmd.Stdout = d.logFile
 		d.cmd.Stderr = d.logFile
 	}
@@ -101,7 +95,7 @@ func (d *Daemon) Release() error {
 		}
 	}
 
-	// Note that the child process will stream it's output directly to the log file.
+	// Note that the child process will stream its output directly to the log file.
 	// So it's safe to close this file handle even if the child process is still running.
 	if d.logFile != nil {
 		err := d.logFile.Close()
@@ -114,7 +108,7 @@ func (d *Daemon) Release() error {
 		return nil
 	}
 
-	// The docs for [os.Process.Release] recommend calling Release if Wait is not called.
-	// It's probably not necessary but we call it just to be safe.
+	// The docs for [os.Process.Release] specify that we need to call Release if
+	// Wait is not called.
 	return d.cmd.Process.Release()
 }
diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go
index b2ec388bd..f9588edd1 100644
--- a/libs/template/renderer_test.go
+++ b/libs/template/renderer_test.go
@@ -116,14 +116,17 @@ func TestBuiltinPythonTemplateValid(t *testing.T) {
 		for _, includeDlt := range options {
 			for _, includePython := range options {
 				for _, isServicePrincipal := range []bool{true, false} {
-					config := map[string]any{
-						"project_name":     "my_project",
-						"include_notebook": includeNotebook,
-						"include_dlt":      includeDlt,
-						"include_python":   includePython,
+					for _, serverless := range options {
+						config := map[string]any{
+							"project_name":     "my_project",
+							"include_notebook": includeNotebook,
+							"include_dlt":      includeDlt,
+							"include_python":   includePython,
+							"serverless":       serverless,
+						}
+						tempDir := t.TempDir()
+						assertBuiltinTemplateValid(t, "default-python", config, "dev", isServicePrincipal, build, tempDir)
 					}
-					tempDir := t.TempDir()
-					assertBuiltinTemplateValid(t, "default-python", config, "dev", isServicePrincipal, build, tempDir)
 				}
 			}
 		}
@@ -135,6 +138,7 @@ func TestBuiltinPythonTemplateValid(t *testing.T) {
 		"include_notebook": "yes",
 		"include_dlt":      "yes",
 		"include_python":   "yes",
+		"serverless":       "yes",
 	}
 	isServicePrincipal = false
 	build = true
diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/.gitignore b/libs/template/templates/dbt-sql/template/{{.project_name}}/.gitignore
new file mode 100644
index 000000000..231162918
--- /dev/null
+++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/.gitignore
@@ -0,0 +1,15 @@
+# DABs
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
+
+# dbt
+target/
+dbt_packages/
+dbt_modules/
+logs/
diff --git a/libs/template/templates/default-python/databricks_template_schema.json b/libs/template/templates/default-python/databricks_template_schema.json
index d53bad91a..6d42d4115 100644
--- a/libs/template/templates/default-python/databricks_template_schema.json
+++ b/libs/template/templates/default-python/databricks_template_schema.json
@@ -29,6 +29,14 @@
             "enum": ["yes", "no"],
             "description": "Include a stub (sample) Python package in '{{.project_name}}{{path_separator}}src'",
             "order": 4
+        },
+        "serverless": {
+            "type": "string",
+            "default": "no",
+            "enum": ["no", "yes"],
+            "description": "Use serverless compute",
+            "skip_prompt_if": {},
+            "order": 5
         }
     },
     "success_message": "Workspace to use (auto-detected, edit in '{{.project_name}}/databricks.yml'): {{workspace_host}}\n\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html."
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl
index 5211e3894..22434aa64 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl
@@ -4,6 +4,7 @@
 {{if and (eq .include_dlt "yes") (and (eq .include_notebook "no") (eq .include_python "no")) -}}
 # This job runs {{.project_name}}_pipeline on a schedule.
 {{end -}}
+{{$with_serverless := (eq .serverless "yes") -}}
 
 resources:
   jobs:
@@ -29,7 +30,8 @@ resources:
       tasks:
         {{- if eq .include_notebook "yes" }}
         - task_key: notebook_task
-          job_cluster_key: job_cluster
+        {{- if not $with_serverless}}
+          job_cluster_key: job_cluster{{end}}
           notebook_task:
             notebook_path: ../src/notebook.ipynb
         {{end -}}
@@ -52,23 +54,41 @@ resources:
           depends_on:
             - task_key: notebook_task
           {{end}}
-          job_cluster_key: job_cluster
+          {{- if $with_serverless }}
+          environment_key: default
+          {{- else }}
+          job_cluster_key: job_cluster{{end}}
           python_wheel_task:
             package_name: {{.project_name}}
             entry_point: main
+          {{- if not $with_serverless }}
           libraries:
             # By default we just include the .whl file generated for the {{.project_name}} package.
             # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
             # for more information on how to add other libraries.
             - whl: ../dist/*.whl
+{{- end -}}
+{{else}}
+{{- end}}
+{{if $with_serverless}}
+      # A list of task execution environment specifications that can be referenced by tasks of this job.
+      environments:
+        - environment_key: default
 
-      {{else}}
-      {{end -}}
+          # Full documentation of this spec can be found at:
+          # https://docs.databricks.com/api/workspace/jobs/create#environments-spec
+          spec:
+            client: "1"
+            dependencies:
+              - ../dist/*.whl
+{{ else }}
       job_clusters:
         - job_cluster_key: job_cluster
           new_cluster:
             spark_version: {{template "latest_lts_dbr_version"}}
             node_type_id: {{smallest_node_type}}
+            data_security_mode: SINGLE_USER
             autoscale:
                 min_workers: 1
                 max_workers: 4
+{{end -}}
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl
index 50f11fe2c..024c1ab15 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl
@@ -1,15 +1,22 @@
+{{$with_serverless := (eq .serverless "yes") -}}
 # The main pipeline for {{.project_name}}
 resources:
   pipelines:
     {{.project_name}}_pipeline:
       name: {{.project_name}}_pipeline
       {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}}
+      {{- if $with_serverless }}
+      ## Catalog is required for serverless compute
+      catalog: main{{else}}
       ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:
-      # catalog: catalog_name
+      # catalog: catalog_name{{end}}
       {{- else}}
       catalog: {{default_catalog}}
       {{- end}}
       target: {{.project_name}}_${bundle.target}
+      {{- if $with_serverless }}
+      serverless: true
+      {{- end}}
       libraries:
         - notebook:
             path: ../src/dlt_pipeline.ipynb
diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/.gitignore b/libs/template/templates/default-sql/template/{{.project_name}}/.gitignore
new file mode 100644
index 000000000..0dab7f499
--- /dev/null
+++ b/libs/template/templates/default-sql/template/{{.project_name}}/.gitignore
@@ -0,0 +1,8 @@
+.databricks/
+build/
+dist/
+__pycache__/
+*.egg-info
+.venv/
+scratch/**
+!scratch/README.md
diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
index 7c7a0d33f..d9d248799 100644
--- a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
+++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl
@@ -97,6 +97,7 @@ This job runs {{.project_name}}_pipeline on a schedule.
                 "new_cluster": {
                     "spark_version": "{{template "latest_lts_dbr_version"}}",
                     "node_type_id": "{{smallest_node_type}}",
+                    "data_security_mode": "SINGLE_USER",
                     "autoscale": {
                         "min_workers": 1,
                         "max_workers": 4,