Merge remote-tracking branch 'origin' into auth-validate

.gitignore

@@ -25,11 +25,7 @@ coverage-acceptance.txt
 __pycache__
 *.pyc
 
-.terraform
-.terraform.lock.hcl
-
 .vscode/launch.json
 .vscode/tasks.json
 
-.databricks
 .ruff_cache

Makefile

@@ -1,4 +1,4 @@
-default: vendor fmt lint
+default: vendor fmt lint tidy
 
 PACKAGES=./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... .
 
@@ -9,6 +9,10 @@ GOTESTSUM_CMD ?= gotestsum --format ${GOTESTSUM_FORMAT} --no-summary=skipped
 lint:
 	golangci-lint run --fix
 
+tidy:
+	@# not part of golangci-lint, apparently
+	go mod tidy
+
 lintcheck:
 	golangci-lint run ./...
 
@@ -59,4 +63,4 @@ integration: vendor
 integration-short: vendor
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs
+.PHONY: lint tidy lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs

NOTICE

@@ -109,3 +109,8 @@ License - https://github.com/hexops/gotextdiff/blob/main/LICENSE
 https://github.com/BurntSushi/toml
 Copyright (c) 2013 TOML authors
 https://github.com/BurntSushi/toml/blob/master/COPYING
+
+dario.cat/mergo
+Copyright (c) 2013 Dario Castañé. All rights reserved.
+Copyright (c) 2012 The Go Authors. All rights reserved.
+https://github.com/darccio/mergo/blob/master/LICENSE

acceptance/acceptance_test.go

@@ -19,6 +19,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/google/uuid"
+
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/testdiff"
@@ -123,7 +125,6 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		AddHandlers(defaultServer)
 		// Redirect API access to local server:
 		t.Setenv("DATABRICKS_HOST", defaultServer.URL)
-		t.Setenv("DATABRICKS_TOKEN", "dapi1234")
 
 		homeDir := t.TempDir()
 		// Do not read user's ~/.databrickscfg
@@ -146,7 +147,15 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	// do it last so that full paths match first:
 	repls.SetPath(buildDir, "[BUILD_DIR]")
 
-	workspaceClient, err := databricks.NewWorkspaceClient()
+	var config databricks.Config
+	if cloudEnv == "" {
+		// use fake token for local tests
+		config = databricks.Config{Token: "dbapi1234"}
+	} else {
+		// non-local tests rely on environment variables
+		config = databricks.Config{}
+	}
+	workspaceClient, err := databricks.NewWorkspaceClient(&config)
 	require.NoError(t, err)
 
 	user, err := workspaceClient.CurrentUser.Me(ctx)
@@ -264,7 +273,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 		for _, stub := range config.Server {
 			require.NotEmpty(t, stub.Pattern)
-			server.Handle(stub.Pattern, func(req *http.Request) (any, int) {
+			server.Handle(stub.Pattern, func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
 				statusCode := http.StatusOK
 				if stub.Response.StatusCode != 0 {
 					statusCode = stub.Response.StatusCode
@@ -285,6 +294,15 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	// Each local test should use a new token that will result into a new fake workspace,
+	// so that test don't interfere with each other.
+	if cloudEnv == "" {
+		tokenSuffix := strings.ReplaceAll(uuid.NewString(), "-", "")
+		token := "dbapi" + tokenSuffix
+		cmd.Env = append(cmd.Env, "DATABRICKS_TOKEN="+token)
+		repls.Set(token, "[DATABRICKS_TOKEN]")
+	}
+
 	// Write combined output to a file
 	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
 	require.NoError(t, err)
@@ -303,8 +321,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 			reqJson, err := json.Marshal(req)
 			require.NoError(t, err)
 
-			line := fmt.Sprintf("%s\n", reqJson)
+			reqJsonWithRepls := repls.Replace(string(reqJson))
-			_, err = f.WriteString(line)
+			_, err = f.WriteString(reqJsonWithRepls + "\n")
 			require.NoError(t, err)
 		}
 
@@ -325,6 +343,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 
 	// Make sure there are not unaccounted for new files
 	files := ListDir(t, tmpDir)
+	unexpected := []string{}
 	for _, relPath := range files {
 		if _, ok := inputs[relPath]; ok {
 			continue
@@ -332,13 +351,17 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if _, ok := outputs[relPath]; ok {
 			continue
 		}
-		t.Errorf("Unexpected output: %s", relPath)
+		unexpected = append(unexpected, relPath)
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
 			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 		}
 	}
+
+	if len(unexpected) > 0 {
+		t.Error("Test produced unexpected files:\n" + strings.Join(unexpected, "\n"))
+	}
 }
 
 func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {

acceptance/bin/sort_lines.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH.
+"""
+
+import sys
+
+lines = sys.stdin.readlines()
+lines.sort()
+sys.stdout.write("".join(lines))

acceptance/bundle/artifacts/same_name_libraries/databricks.yml

@@ -0,0 +1,50 @@
+bundle:
+  name: same_name_libraries
+
+variables:
+  cluster:
+    default:
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      data_security_mode: SINGLE_USER
+      num_workers: 0
+      spark_conf:
+          spark.master: "local[*, 4]"
+          spark.databricks.cluster.profile: singleNode
+      custom_tags:
+        ResourceClass: SingleNode
+
+artifacts:
+  whl1:
+    type: whl
+    path: ./whl1
+  whl2:
+    type: whl
+    path: ./whl2
+
+resources:
+  jobs:
+    test:
+      name: "test"
+      tasks:
+        - task_key: task1
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl1/dist/*.whl
+        - task_key: task2
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl2/dist/*.whl
+        - task_key: task3
+          new_cluster: ${var.cluster}
+          python_wheel_task:
+            entry_point: main
+            package_name: my_default_python
+          libraries:
+            - whl: ./whl1/dist/*.whl

acceptance/bundle/artifacts/same_name_libraries/output.txt

@@ -0,0 +1,14 @@
+
+>>> errcode [CLI] bundle deploy
+Building whl1...
+Building whl2...
+Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
+  at resources.jobs.test.tasks[0].libraries[0].whl
+     resources.jobs.test.tasks[1].libraries[0].whl
+  in databricks.yml:36:15
+     databricks.yml:43:15
+
+Local library names must be unique
+
+
+Exit code: 1

acceptance/bundle/artifacts/same_name_libraries/script

@@ -0,0 +1,2 @@
+trace errcode $CLI bundle deploy
+rm -rf whl1 whl2

acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py

@@ -0,0 +1,36 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import my_default_python
+
+setup(
+    name="my_default_python",
+    version=my_default_python.__version__,
+    url="https://databricks.com",
+    author="[USERNAME]",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)

acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py

@@ -0,0 +1,36 @@
+"""
+setup.py configuration script describing how to build and package this project.
+
+This file is primarily used by the setuptools library and typically should not
+be executed directly. See README.md for how to deploy, test, and run
+the my_default_python project.
+"""
+
+from setuptools import setup, find_packages
+
+import sys
+
+sys.path.append("./src")
+
+import my_default_python
+
+setup(
+    name="my_default_python",
+    version=my_default_python.__version__,
+    url="https://databricks.com",
+    author="[USERNAME]",
+    description="wheel file based on my_default_python/src",
+    packages=find_packages(where="./src"),
+    package_dir={"": "src"},
+    entry_points={
+        "packages": [
+            "main=my_default_python.main:main",
+        ],
+    },
+    install_requires=[
+        # Dependencies in case the output wheel file is used as a library dependency.
+        # For defining dependencies, when this package is used in Databricks, see:
+        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
+        "setuptools"
+    ],
+)

acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/debug/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: debug

acceptance/bundle/debug/out.stderr.parallel.txt

@@ -0,0 +1,15 @@
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly)
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:SingleNodeCluster
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:artifact_paths
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_cluster_key_defined
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=fast_validate(readonly) mutator (read-only)=parallel mutator (read-only)=validate:job_task_cluster_spec
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
+10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
+10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
+10:07:59 Debug: non-retriable error:  pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true

acceptance/bundle/debug/out.stderr.txt

@@ -0,0 +1,92 @@
+10:07:59 Info: start pid=12345 version=[DEV_VERSION] args="[CLI], bundle, validate, --debug"
+10:07:59 Debug: Found bundle root at [TMPDIR] (file [TMPDIR]/databricks.yml) pid=12345
+10:07:59 Debug: Apply pid=12345 mutator=load
+10:07:59 Info: Phase: load pid=12345 mutator=load
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EntryPoint
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=scripts.preinit
+10:07:59 Debug: No script defined for preinit, skipping pid=12345 mutator=load mutator=seq mutator=scripts.preinit
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ProcessRootIncludes mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=VerifyCliVersion
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=EnvironmentsToTargets
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=ComputeIdToClusterId
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=InitializeVariables
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=DefineDefaultTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=PythonMutator(load)
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=validate:unique_resource_keys
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget
+10:07:59 Debug: Apply pid=12345 mutator=load mutator=seq mutator=SelectDefaultTarget mutator=SelectTarget(default)
+10:07:59 Debug: Apply pid=12345 mutator=<func>
+10:07:59 Debug: Apply pid=12345 mutator=initialize
+10:07:59 Info: Phase: initialize pid=12345 mutator=initialize
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=validate:AllResourcesHaveValues
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteSyncPaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncDefaultPath
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SyncInferRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser
+10:07:59 Debug: GET /api/2.0/preview/scim/v2/Me
+< HTTP/1.1 200 OK
+< {
+<   "id": "[USERID]",
+<   "userName": "[USERNAME]"
+< } pid=12345 mutator=initialize mutator=seq mutator=PopulateCurrentUser sdk=true
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=LoadGitDetails
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplySourceLinkedDeploymentPreset
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefineDefaultWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandWorkspaceRoot
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultWorkspacePaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PrependWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=RewriteWorkspacePrefix
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetVariables
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(init)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(load_resources)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonMutator(apply_mutators)
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveResourceReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ResolveVariableReferences
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobClusters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobParameters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeJobTasks
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergePipelineClusters
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=MergeApps
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CaptureSchemaDependency
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=CheckPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=SetRunAs
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=OverrideCompute
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureDashboardDefaults
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureVolumeDefaults
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ProcessTargetMode
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyPresets
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=DefaultQueueing
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ExpandPipelineGlobPaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ConfigureWSFS
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=TranslatePaths
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=PythonWrapperWarning
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=apps.Validate
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ValidateSharedRootPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=ApplyBundlePermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=FilterCurrentUserFromPermissions
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotateJobs
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=metadata.AnnotatePipelines
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Using Terraform from DATABRICKS_TF_EXEC_PATH at [TERRAFORM] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Using Terraform CLI config from DATABRICKS_TF_CLI_CONFIG_FILE at [DATABRICKS_TF_CLI_CONFIG_FILE] pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Environment variables for Terraform: ...redacted... pid=12345 mutator=initialize mutator=seq mutator=terraform.Initialize
+10:07:59 Debug: Apply pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit
+10:07:59 Debug: No script defined for postinit, skipping pid=12345 mutator=initialize mutator=seq mutator=scripts.postinit
+10:07:59 Debug: Apply pid=12345 mutator=validate
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
+< HTTP/1.1 404 Not Found
+10:07:59 Debug: POST /api/2.0/workspace/mkdirs
+> {
+>   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
+> }
+< HTTP/1.1 200 OK
+10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
+< HTTP/1.1 200 OK
+< {
+<   "object_type": "DIRECTORY",
+<   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
+10:07:59 Info: completed execution pid=12345 exit_code=0

acceptance/bundle/debug/output.txt

@@ -0,0 +1,7 @@
+Name: debug
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/debug/default
+
+Validation OK!

acceptance/bundle/debug/script

@@ -0,0 +1,4 @@
+$CLI bundle validate --debug 2> full.stderr.txt
+grep -vw parallel full.stderr.txt > out.stderr.txt
+grep -w parallel full.stderr.txt | sed 's/[0-9]/0/g' | sort_lines.py > out.stderr.parallel.txt
+rm full.stderr.txt

acceptance/bundle/debug/test.toml

@@ -0,0 +1,18 @@
+LocalOnly = true
+
+[[Repls]]
+# The keys are unsorted and also vary per OS
+Old = 'Environment variables for Terraform: ([A-Z_ ,]+) '
+New = 'Environment variables for Terraform: ...redacted... '
+
+[[Repls]]
+Old = 'pid=[0-9]+'
+New = 'pid=12345'
+
+[[Repls]]
+Old = '\d\d:\d\d:\d\d'
+New = '10:07:59'
+
+[[Repls]]
+Old = '\\'
+New = '/'

acceptance/bundle/generate/git_job/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: git_job

acceptance/bundle/generate/git_job/out.job.yml

@@ -0,0 +1,17 @@
+resources:
+  jobs:
+    out:
+      name: gitjob
+      tasks:
+        - task_key: test_task
+          notebook_task:
+            notebook_path: some/test/notebook.py
+        - task_key: test_task_2
+          notebook_task:
+            notebook_path: /Workspace/Users/foo@bar.com/some/test/notebook.py
+            source: WORKSPACE
+      git_source:
+        git_branch: main
+        git_commit: abcdef
+        git_provider: github
+        git_url: https://git.databricks.com

acceptance/bundle/generate/git_job/output.txt

@@ -0,0 +1,2 @@
+Job is using Git source, skipping downloading files
+Job configuration successfully saved to out.job.yml

acceptance/bundle/generate/git_job/script

@@ -0,0 +1 @@
+$CLI bundle generate job --existing-job-id 1234 --config-dir . --key out

acceptance/bundle/generate/git_job/test.toml

@@ -0,0 +1,33 @@
+LocalOnly = true  # This test needs to run against stubbed Databricks API
+
+[[Server]]
+Pattern = "GET /api/2.1/jobs/get"
+Response.Body = '''
+{
+    "job_id": 11223344,
+    "settings": {
+        "name": "gitjob",
+        "git_source": {
+            "git_url":      "https://git.databricks.com",
+            "git_provider": "github",
+            "git_branch":   "main",
+            "git_commit":   "abcdef"
+        },
+        "tasks": [
+            {
+                "task_key": "test_task",
+                "notebook_task": {
+                    "notebook_path": "some/test/notebook.py"
+                }
+            },
+            {
+                "task_key": "test_task_2",
+                "notebook_task": {
+                    "source": "WORKSPACE",
+                    "notebook_path": "/Workspace/Users/foo@bar.com/some/test/notebook.py"
+                }
+            }
+        ]
+    }
+}
+'''

acceptance/bundle/git-permerror/output.txt

@@ -3,6 +3,7 @@
 >>> chmod 000 .git
 
 >>> [CLI] bundle validate
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 Name: git-permerror
@@ -16,6 +17,7 @@ Found 1 error
 Exit code: 1
 
 >>> [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -25,6 +27,7 @@ Exit code: 1
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -39,11 +42,15 @@ Exit code: 1
 >>> chmod 000 .git/HEAD
 
 >>> [CLI] bundle validate -o json
+Warn: failed to load current branch: open HEAD: permission denied
+Warn: failed to load latest commit: open HEAD: permission denied
 {
   "bundle_root_path": "."
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to load current branch: open HEAD: permission denied
+Warn: failed to load latest commit: open HEAD: permission denied
 {
   "bundle_root_path": "."
 }
@@ -54,6 +61,7 @@ Exit code: 1
 >>> chmod 000 .git/config
 
 >>> [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 
@@ -63,6 +71,7 @@ Exit code: 1
 }
 
 >>> withdir subdir/a/b [CLI] bundle validate -o json
+Warn: failed to read .git: unable to load repository specific gitconfig: open config: permission denied
 Error: unable to load repository specific gitconfig: open config: permission denied
 
 

acceptance/bundle/git-permerror/test.toml

@@ -1,4 +1,4 @@
-Badness = "Warning logs not shown; inferred flag is set to true incorrect; bundle_root_path is not correct"
+Badness = "inferred flag is set to true incorrect; bundle_root_path is not correct; Warn and Error talk about the same; Warn goes to stderr, Error goes to stdout (for backward compat); Warning about permissions repeated twice"
 
 [GOOS]
 # This test relies on chmod which does not work on Windows

acceptance/bundle/scripts/output.txt

@@ -42,11 +42,9 @@ from myscript.py 0 postbuild: hello stderr!
 Executing 'predeploy' script
 from myscript.py 0 predeploy: hello stdout!
 from myscript.py 0 predeploy: hello stderr!
-Error: unable to deploy to /Workspace/Users/[USERNAME]/.bundle/scripts/default/state as [USERNAME].
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/scripts/default/files...
-Please make sure the current user or one of their groups is listed under the permissions of this bundle.
+Deploying resources...
-For assistance, contact the owners of this project.
+Deployment complete!
-They may need to redeploy the bundle to apply the new permissions.
+Executing 'postdeploy' script
-Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.
+from myscript.py 0 postdeploy: hello stdout!
-
+from myscript.py 0 postdeploy: hello stderr!
-
-Exit code: 1

acceptance/bundle/scripts/test.toml

@@ -1 +0,0 @@
-LocalOnly = true  # Deployment currently fails when run locally; once that is fixed, remove this setting

acceptance/bundle/templates-machinery/helpers-error/databricks_template_schema.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/templates-machinery/helpers-error/output.txt

@@ -0,0 +1,3 @@
+Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at <user_name>: error calling user_name: 
+
+Exit code: 1

acceptance/bundle/templates-machinery/helpers-error/script

@@ -0,0 +1 @@
+$CLI bundle init .

acceptance/bundle/templates-machinery/helpers-error/template/helpers.txt.tmpl

@@ -0,0 +1 @@
+user_name: {{ user_name }}

acceptance/bundle/templates-machinery/helpers-error/test.toml

@@ -0,0 +1,7 @@
+Badness = '''(minor) error message is not great: executing "" at <user_name>: error calling user_name:'''
+LocalOnly = true
+
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/Me"
+Response.Body = '{}'
+Response.StatusCode = 500

acceptance/bundle/templates-machinery/helpers/databricks_template_schema.json

@@ -0,0 +1 @@
+{}

acceptance/bundle/templates-machinery/helpers/output.txt

@@ -0,0 +1,2 @@
+✨ Successfully initialized template
+user_name: [USERNAME]

acceptance/bundle/templates-machinery/helpers/script

@@ -0,0 +1,3 @@
+$CLI bundle init .
+cat helpers.txt
+rm helpers.txt

acceptance/bundle/templates-machinery/helpers/template/helpers.txt.tmpl

@@ -0,0 +1 @@
+user_name: {{ user_name }}

acceptance/bundle/templates-machinery/helpers/test.toml

@@ -0,0 +1 @@
+LocalOnly = true

acceptance/bundle/templates-machinery/test.toml

@@ -0,0 +1,2 @@
+# Testing template machinery, by default there is no need to check against cloud.
+LocalOnly = true

acceptance/bundle/templates/dbt-sql/script

@@ -3,3 +3,6 @@ trace $CLI bundle init dbt-sql --config-file ./input.json --output-dir output
 cd output/my_dbt_sql
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore

acceptance/bundle/templates/default-python/script

@@ -3,3 +3,6 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir ou
 cd output/my_default_python
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore

acceptance/bundle/templates/default-sql/script

@@ -3,3 +3,6 @@ trace $CLI bundle init default-sql --config-file ./input.json --output-dir outpu
 cd output/my_default_sql
 trace $CLI bundle validate -t dev
 trace $CLI bundle validate -t prod
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore

acceptance/bundle/templates/experimental-jobs-as-code/script

@@ -8,3 +8,6 @@ uv sync -q
 trace $CLI bundle validate -t dev --output json | jq ".resources"
 
 rm -fr .venv resources/__pycache__ uv.lock my_jobs_as_code.egg-info
+
+# Do not affect this repository's git behaviour #2318
+mv .gitignore out.gitignore

acceptance/bundle/variables/env_overrides/test.toml

@@ -1,2 +0,0 @@
-# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist
-LocalOnly = true

acceptance/bundle/variables/resolve-builtin/test.toml

@@ -1,2 +0,0 @@
-# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/'
-LocalOnly = true

acceptance/bundle/variables/resolve-vars-in-root-path/test.toml

@@ -1,2 +0,0 @@
-# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/'
-LocalOnly = true

acceptance/bundle/variables/test.toml

@@ -0,0 +1,3 @@
+# The tests here intend to test variable interpolation via "bundle validate".
+# Even though "bundle validate" does a few API calls, that's not the focus there.
+LocalOnly = true

acceptance/cmd_server_test.go

@@ -15,7 +15,10 @@ import (
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
 
-	server.Handle("/", func(r *http.Request) (any, int) {
+	// {$} is a wildcard that only matches the end of the URL. We explicitly use
+	// /{$} to disambiguate it from the generic handler for '/' which is used to
+	// identify unhandled API endpoints in the test server.
+	server.Handle("/{$}", func(w *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 

acceptance/config_test.go

@@ -3,9 +3,11 @@ package acceptance_test
 import (
 	"os"
 	"path/filepath"
-	"sync"
+	"slices"
+	"strings"
 	"testing"
 
+	"dario.cat/mergo"
 	"github.com/BurntSushi/toml"
 	"github.com/databricks/cli/libs/testdiff"
 	"github.com/stretchr/testify/require"
@@ -13,11 +15,6 @@ import (
 
 const configFilename = "test.toml"
 
-var (
-	configCache map[string]TestConfig
-	configMutex sync.Mutex
-)
-
 type TestConfig struct {
 	// Place to describe what's wrong with this test. Does not affect how the test is run.
 	Badness string
@@ -65,58 +62,55 @@ type ServerStub struct {
 	}
 }
 
-// FindConfig finds the closest config file.
+// FindConfigs finds all the config relevant for this test,
-func FindConfig(t *testing.T, dir string) (string, bool) {
+// ordered from the most outermost (at acceptance/) to current test directory (identified by dir).
-	shared := false
+// Argument dir must be a relative path from the root of acceptance tests (<project_root>/acceptance/).
+func FindConfigs(t *testing.T, dir string) []string {
+	configs := []string{}
 	for {
 		path := filepath.Join(dir, configFilename)
 		_, err := os.Stat(path)
 
 		if err == nil {
-			return path, shared
+			configs = append(configs, path)
 		}
 
-		shared = true
-
 		if dir == "" || dir == "." {
 			break
 		}
 
-		if os.IsNotExist(err) {
+		dir = filepath.Dir(dir)
-			dir = filepath.Dir(dir)
+
+		if err == nil || os.IsNotExist(err) {
 			continue
 		}
 
 		t.Fatalf("Error while reading %s: %s", path, err)
 	}
 
-	t.Fatal("Config not found: " + configFilename)
+	slices.Reverse(configs)
-	return "", shared
+	return configs
 }
 
 // LoadConfig loads the config file. Non-leaf configs are cached.
 func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
-	path, leafConfig := FindConfig(t, dir)
+	configs := FindConfigs(t, dir)
 
-	if leafConfig {
+	if len(configs) == 0 {
-		return DoLoadConfig(t, path), path
+		return TestConfig{}, "(no config)"
 	}
 
-	configMutex.Lock()
+	result := DoLoadConfig(t, configs[0])
-	defer configMutex.Unlock()
 
-	if configCache == nil {
+	for _, cfgName := range configs[1:] {
-		configCache = make(map[string]TestConfig)
+		cfg := DoLoadConfig(t, cfgName)
+		err := mergo.Merge(&result, cfg, mergo.WithOverride, mergo.WithAppendSlice)
+		if err != nil {
+			t.Fatalf("Error during config merge: %s: %s", cfgName, err)
+		}
 	}
 
-	result, ok := configCache[path]
+	return result, strings.Join(configs, ", ")
-	if ok {
-		return result, path
-	}
-
-	result = DoLoadConfig(t, path)
-	configCache[path] = result
-	return result, path
 }
 
 func DoLoadConfig(t *testing.T, path string) TestConfig {

acceptance/server_test.go

@@ -1,17 +1,23 @@
 package acceptance_test
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
 	"net/http"
 
-	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
-	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/iam"
+
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go/service/workspace"
 )
 
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET /api/2.0/policies/clusters/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -26,7 +32,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/instance-pools/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -37,7 +43,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/clusters/list", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -52,31 +58,74 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return iam.User{
 			Id:       "1000012345",
 			UserName: "tester@databricks.com",
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/workspace/get-status", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return workspace.ObjectInfo{
+		path := r.URL.Query().Get("path")
-			ObjectId:   1001,
+
-			ObjectType: "DIRECTORY",
+		return fakeWorkspace.WorkspaceGetStatus(path)
-			Path:       "",
-			ResourceId: "1001",
-		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(r *http.Request) (any, int) {
+	server.Handle("POST /api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		request := workspace.Mkdirs{}
+		decoder := json.NewDecoder(r.Body)
+
+		err := decoder.Decode(&request)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.WorkspaceMkdirs(request)
+	})
+
+	server.Handle("GET /api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.URL.Query().Get("path")
+
+		return fakeWorkspace.WorkspaceExport(path)
+	})
+
+	server.Handle("POST /api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.URL.Query().Get("path")
+		recursiveStr := r.URL.Query().Get("recursive")
+		var recursive bool
+
+		if recursiveStr == "true" {
+			recursive = true
+		} else {
+			recursive = false
+		}
+
+		return fakeWorkspace.WorkspaceDelete(path, recursive)
+	})
+
+	server.Handle("POST /api/2.0/workspace-files/import-file/{path}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		path := r.PathValue("path")
+
+		body := new(bytes.Buffer)
+		_, err := body.ReadFrom(r.Body)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes())
+	})
+
+	server.Handle("GET /api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
 		}, http.StatusOK
 	})
 
-	server.Handle("GET /api/2.0/permissions/directories/1001", func(r *http.Request) (any, int) {
+	server.Handle("GET /api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		objectId := r.PathValue("objectId")
+
 		return workspace.WorkspaceObjectPermissions{
-			ObjectId:   "1001",
+			ObjectId:   objectId,
 			ObjectType: "DIRECTORY",
 			AccessControlList: []workspace.WorkspaceObjectAccessControlResponse{
 				{
@@ -91,8 +140,26 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 
-	server.Handle("POST /api/2.0/workspace/mkdirs", func(r *http.Request) (any, int) {
+	server.Handle("POST /api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return "{}", http.StatusOK
+		request := jobs.CreateJob{}
+		decoder := json.NewDecoder(r.Body)
+
+		err := decoder.Decode(&request)
+		if err != nil {
+			return internalError(err)
+		}
+
+		return fakeWorkspace.JobsCreate(request)
+	})
+
+	server.Handle("GET /api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		jobId := r.URL.Query().Get("job_id")
+
+		return fakeWorkspace.JobsGet(jobId)
+	})
+
+	server.Handle("GET /api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+		return fakeWorkspace.JobsList()
 	})
 
 	server.Handle("GET /oidc/.well-known/oauth-authorization-server", func(r *http.Request) (any, int) {
@@ -111,3 +178,7 @@ func AddHandlers(server *testserver.Server) {
 		}, http.StatusOK
 	})
 }
+
+func internalError(err error) (any, int) {
+	return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError
+}

acceptance/test.toml

@@ -1,2 +0,0 @@
-# If test directory nor any of its parents do not have test.toml then this file serves as fallback configuration.
-# The configurations are not merged across parents; the closest one is used fully.

acceptance/workspace/jobs/create/out.requests.txt

@@ -1 +1 @@
-{"headers":{"Authorization":"Bearer dapi1234","User-Agent":"cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}
+{"headers":{"Authorization":["Bearer [DATABRICKS_TOKEN]"],"User-Agent":["cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/jobs_create cmd-exec-id/[UUID] auth/pat"]},"method":"POST","path":"/api/2.1/jobs/create","body":{"name":"abc"}}

bundle/artifacts/upload.go

@@ -29,7 +29,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 	// We intentionally ignore the error because it is not critical to the deployment
 	err := client.Delete(ctx, ".", filer.DeleteRecursively)
 	if err != nil {
-		log.Errorf(ctx, "failed to delete %s: %v", uploadPath, err)
+		log.Debugf(ctx, "failed to delete %s: %v", uploadPath, err)
 	}
 
 	err = client.Mkdir(ctx, ".")

bundle/config/generate/job.go

@@ -13,7 +13,6 @@ var (
 
 func ConvertJobToValue(job *jobs.Job) (dyn.Value, error) {
 	value := make(map[string]dyn.Value)
-
 	if job.Settings.Tasks != nil {
 		tasks := make([]dyn.Value, 0)
 		for _, task := range job.Settings.Tasks {

bundle/config/mutator/apply_presets.go

@@ -84,7 +84,7 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 
 	// Pipelines presets: Prefix, PipelinesDevelopment
 	for key, p := range r.Pipelines {
-		if p.PipelineSpec == nil {
+		if p.CreatePipeline == nil {
 			diags = diags.Extend(diag.Errorf("pipeline %s is not defined", key))
 			continue
 		}

bundle/config/mutator/capture_schema_dependency.go

@@ -56,7 +56,7 @@ func resolveVolume(v *resources.Volume, b *bundle.Bundle) {
 }
 
 func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) {
-	if p == nil || p.PipelineSpec == nil {
+	if p == nil || p.CreatePipeline == nil {
 		return
 	}
 	if p.Schema == "" {
@@ -71,7 +71,7 @@ func resolvePipelineSchema(p *resources.Pipeline, b *bundle.Bundle) {
 }
 
 func resolvePipelineTarget(p *resources.Pipeline, b *bundle.Bundle) {
-	if p == nil || p.PipelineSpec == nil {
+	if p == nil || p.CreatePipeline == nil {
 		return
 	}
 	if p.Target == "" {

bundle/config/mutator/capture_schema_dependency_test.go

@@ -118,43 +118,43 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog2",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline3": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "barfoo",
 						},
 					},
 					"pipeline4": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalogX",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline5": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Schema:  "schemaX",
 						},
 					},
 					"pipeline6": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Schema:  "foobar",
 						},
 					},
 					"pipeline7": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Schema:  "",
 							Name:    "whatever",
@@ -179,7 +179,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) {
 	assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema)
 
 	assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"])
-	assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec)
+	assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].CreatePipeline)
 
 	for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} {
 		assert.Empty(t, b.Config.Resources.Pipelines[k].Target)
@@ -214,43 +214,43 @@ func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "foobar",
 						},
 					},
 					"pipeline2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog2",
 							Target:  "foobar",
 						},
 					},
 					"pipeline3": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "barfoo",
 						},
 					},
 					"pipeline4": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalogX",
 							Target:  "foobar",
 						},
 					},
 					"pipeline5": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "catalog1",
 							Target:  "schemaX",
 						},
 					},
 					"pipeline6": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Target:  "foobar",
 						},
 					},
 					"pipeline7": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Catalog: "",
 							Target:  "",
 							Name:    "whatever",

bundle/config/mutator/expand_pipeline_glob_paths_test.go

@@ -47,7 +47,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{

bundle/config/mutator/initialize_urls_test.go

@@ -31,8 +31,8 @@ func TestInitializeURLs(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline1": {
-						ID:           "3",
+						ID:             "3",
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1"},
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1"},
 					},
 				},
 				Experiments: map[string]*resources.MlflowExperiment{

bundle/config/mutator/merge_pipeline_clusters_test.go

@@ -19,7 +19,7 @@ func TestMergePipelineClusters(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Clusters: []pipelines.PipelineCluster{
 								{
 									NodeTypeId: "i3.xlarge",
@@ -68,7 +68,7 @@ func TestMergePipelineClustersCaseInsensitive(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Clusters: []pipelines.PipelineCluster{
 								{
 									Label:      "default",

bundle/config/mutator/process_target_mode_test.go

@@ -88,7 +88,7 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline1": {PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1", Continuous: true}},
+					"pipeline1": {CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1", Continuous: true}},
 				},
 				Experiments: map[string]*resources.MlflowExperiment{
 					"experiment1": {Experiment: &ml.Experiment{Name: "/Users/lennart.kats@databricks.com/experiment1"}},
@@ -181,7 +181,7 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
 	// Pipeline 1
 	assert.Equal(t, "[dev lennart] pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
 	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].Continuous)
-	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.True(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 
 	// Experiment 1
 	assert.Equal(t, "/Users/lennart.kats@databricks.com/[dev lennart] experiment1", b.Config.Resources.Experiments["experiment1"].Name)
@@ -316,7 +316,7 @@ func TestProcessTargetModeDefault(t *testing.T) {
 	require.NoError(t, diags.Error())
 	assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name)
 	assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
@@ -362,7 +362,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
 
 	assert.Equal(t, "job1", b.Config.Resources.Jobs["job1"].Name)
 	assert.Equal(t, "pipeline1", b.Config.Resources.Pipelines["pipeline1"].Name)
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 	assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
 	assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
 	assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
@@ -568,5 +568,5 @@ func TestPipelinesDevelopmentDisabled(t *testing.T) {
 	diags := bundle.Apply(context.Background(), b, m)
 	require.NoError(t, diags.Error())
 
-	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
+	assert.False(t, b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Development)
 }

bundle/config/mutator/resolve_variable_references_test.go

@@ -20,7 +20,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 			true,
 			func(t *testing.T, b *bundle.Bundle) {
 				// Variables that use workspace file path should have SyncRootValue during resolution phase
-				require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+				require.Equal(t, "sync/root/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"])
 
 				// The file path itself should remain the same
 				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
@@ -29,7 +29,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 		{
 			false,
 			func(t *testing.T, b *bundle.Bundle) {
-				require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Configuration["source"])
+				require.Equal(t, "file/path", b.Config.Resources.Pipelines["pipeline1"].CreatePipeline.Configuration["source"])
 				require.Equal(t, "file/path", b.Config.Workspace.FilePath)
 			},
 		},
@@ -48,7 +48,7 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 				Resources: config.Resources{
 					Pipelines: map[string]*resources.Pipeline{
 						"pipeline1": {
-							PipelineSpec: &pipelines.PipelineSpec{
+							CreatePipeline: &pipelines.CreatePipeline{
 								Configuration: map[string]string{
 									"source": "${workspace.file_path}",
 								},

bundle/config/mutator/translate_paths_test.go

@@ -179,7 +179,7 @@ func TestTranslatePaths(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -333,7 +333,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -488,7 +488,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -532,7 +532,7 @@ func TestPipelineNotebookDoesNotExistErrorWithoutExtension(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"pipeline": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Libraries: []pipelines.PipelineLibrary{
 										{
 											Notebook: &pipelines.NotebookLibrary{
@@ -572,7 +572,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -677,7 +677,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{
@@ -712,7 +712,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									File: &pipelines.FileLibrary{
@@ -916,7 +916,7 @@ func TestTranslatePathsWithSourceLinkedDeployment(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Libraries: []pipelines.PipelineLibrary{
 								{
 									Notebook: &pipelines.NotebookLibrary{

bundle/config/resources/pipeline.go

@@ -16,7 +16,7 @@ type Pipeline struct {
 	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
 	URL            string         `json:"url,omitempty" bundle:"internal"`
 
-	*pipelines.PipelineSpec
+	*pipelines.CreatePipeline
 }
 
 func (s *Pipeline) UnmarshalJSON(b []byte) error {
@@ -59,5 +59,5 @@ func (s *Pipeline) GetURL() string {
 }
 
 func (s *Pipeline) IsNil() bool {
-	return s.PipelineSpec == nil
+	return s.CreatePipeline == nil
 }

bundle/config/validate/single_node_cluster_test.go

@@ -238,7 +238,7 @@ func TestValidateSingleNodeClusterFailForPipelineClusters(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"foo": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Clusters: []pipelines.PipelineCluster{
 										{
 											SparkConf:  tc.sparkConf,
@@ -493,7 +493,7 @@ func TestValidateSingleNodeClusterPassPipelineClusters(t *testing.T) {
 					Resources: config.Resources{
 						Pipelines: map[string]*resources.Pipeline{
 							"foo": {
-								PipelineSpec: &pipelines.PipelineSpec{
+								CreatePipeline: &pipelines.CreatePipeline{
 									Clusters: []pipelines.PipelineCluster{
 										{
 											SparkConf:  tc.sparkConf,

bundle/deploy/metadata/annotate_pipelines.go

@@ -20,11 +20,11 @@ func (m *annotatePipelines) Name() string {
 
 func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	for _, pipeline := range b.Config.Resources.Pipelines {
-		if pipeline.PipelineSpec == nil {
+		if pipeline.CreatePipeline == nil {
 			continue
 		}
 
-		pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{
+		pipeline.CreatePipeline.Deployment = &pipelines.PipelineDeployment{
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: metadataFilePath(b),
 		}

bundle/deploy/metadata/annotate_pipelines_test.go

@@ -21,12 +21,12 @@ func TestAnnotatePipelinesMutator(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"my-pipeline-1": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Name: "My Pipeline One",
 						},
 					},
 					"my-pipeline-2": {
-						PipelineSpec: &pipelines.PipelineSpec{
+						CreatePipeline: &pipelines.CreatePipeline{
 							Name: "My Pipeline Two",
 						},
 					},
@@ -43,14 +43,14 @@ func TestAnnotatePipelinesMutator(t *testing.T) {
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: "/a/b/c/metadata.json",
 		},
-		b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment)
+		b.Config.Resources.Pipelines["my-pipeline-1"].CreatePipeline.Deployment)
 
 	assert.Equal(t,
 		&pipelines.PipelineDeployment{
 			Kind:             pipelines.DeploymentKindBundle,
 			MetadataFilePath: "/a/b/c/metadata.json",
 		},
-		b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment)
+		b.Config.Resources.Pipelines["my-pipeline-2"].CreatePipeline.Deployment)
 }
 
 func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) {

bundle/deploy/terraform/convert_test.go

@@ -203,7 +203,7 @@ func TestBundleToTerraformForEachTaskLibraries(t *testing.T) {
 
 func TestBundleToTerraformPipeline(t *testing.T) {
 	src := resources.Pipeline{
-		PipelineSpec: &pipelines.PipelineSpec{
+		CreatePipeline: &pipelines.CreatePipeline{
 			Name: "my pipeline",
 			Libraries: []pipelines.PipelineLibrary{
 				{
@@ -759,7 +759,7 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
 			},
 			Pipelines: map[string]*resources.Pipeline{
 				"test_pipeline": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline",
 					},
 				},
@@ -898,12 +898,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
 			},
 			Pipelines: map[string]*resources.Pipeline{
 				"test_pipeline": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline",
 					},
 				},
 				"test_pipeline_new": {
-					PipelineSpec: &pipelines.PipelineSpec{
+					CreatePipeline: &pipelines.CreatePipeline{
 						Name: "test_pipeline_new",
 					},
 				},

bundle/deploy/terraform/tfdyn/convert_pipeline.go

@@ -21,6 +21,11 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err
 		return dyn.InvalidValue, err
 	}
 
+	vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"})
+	if err != nil {
+		return dyn.InvalidValue, err
+	}
+
 	// Normalize the output value to the target schema.
 	vout, diags := convert.Normalize(schema.ResourcePipeline{}, vout)
 	for _, diag := range diags {

bundle/deploy/terraform/tfdyn/convert_pipeline_test.go

@@ -15,8 +15,17 @@ import (
 
 func TestConvertPipeline(t *testing.T) {
 	src := resources.Pipeline{
-		PipelineSpec: &pipelines.PipelineSpec{
+		CreatePipeline: &pipelines.CreatePipeline{
 			Name: "my pipeline",
+			// This fields is not part of TF schema yet, but once we upgrade to TF version that supports it, this test will fail because run_as
+			// will be exposed which is expected and test will need to be updated.
+			RunAs: &pipelines.RunAs{
+				UserName: "foo@bar.com",
+			},
+			// We expect AllowDuplicateNames and DryRun to be ignored and not passed to the TF output.
+			// This is not supported by TF now, so we don't want to expose it.
+			AllowDuplicateNames: true,
+			DryRun:              true,
 			Libraries: []pipelines.PipelineLibrary{
 				{
 					Notebook: &pipelines.NotebookLibrary{

bundle/docsgen/main.go

@@ -130,6 +130,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.MarkdownDescription = a.MarkdownDescription
 	}
 	if a.MarkdownExamples != "" {
-		s.Examples = []any{a.MarkdownExamples}
+		s.Examples = []string{a.MarkdownExamples}
 	}
 }

bundle/docsgen/nodes.go

@@ -220,9 +220,9 @@ func isCycleField(field string) bool {
 }
 
 func getExample(v *jsonschema.Schema) string {
-	examples := v.Examples
+	examples := getExamples(v.Examples)
 	if len(examples) == 0 {
 		return ""
 	}
-	return examples[0].(string)
+	return examples[0]
 }

bundle/docsgen/refs.go

@@ -58,7 +58,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 	node := s
 	description := s.Description
 	markdownDescription := s.MarkdownDescription
-	examples := s.Examples
+	examples := getExamples(s.Examples)
 
 	for node.Reference != nil {
 		ref := getRefType(node)
@@ -75,7 +75,7 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 			markdownDescription = newNode.MarkdownDescription
 		}
 		if len(examples) == 0 {
-			examples = newNode.Examples
+			examples = getExamples(newNode.Examples)
 		}
 
 		node = newNode
@@ -89,6 +89,14 @@ func resolveRefs(s *jsonschema.Schema, schemas map[string]*jsonschema.Schema) *j
 	return &newNode
 }
 
+func getExamples(examples any) []string {
+	typedExamples, ok := examples.([]string)
+	if !ok {
+		return []string{}
+	}
+	return typedExamples
+}
+
 func getRefType(node *jsonschema.Schema) string {
 	if node.Reference == nil {
 		return ""

bundle/internal/schema/annotations.yml

@@ -414,6 +414,16 @@ github.com/databricks/cli/bundle/config/resources.Permission:
   "user_name":
     "description": |-
       The name of the user that has the permission set in level.
+github.com/databricks/cli/bundle/config/resources.Pipeline:
+  "allow_duplicate_names":
+    "description": |-
+      PLACEHOLDER
+  "dry_run":
+    "description": |-
+      PLACEHOLDER
+  "run_as":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/variable.Lookup:
   "alert":
     "description": |-

bundle/internal/schema/annotations_openapi.yml

@@ -371,6 +371,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
     "description": |-
       Tags to be attached to the serving endpoint and automatically propagated to billing logs.
 github.com/databricks/cli/bundle/config/resources.Pipeline:
+  "allow_duplicate_names":
+    "description": |-
+      If false, deployment will fail if name conflicts with that of another pipeline.
   "budget_policy_id":
     "description": |-
       Budget policy of this pipeline.
@@ -395,6 +398,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "development":
     "description": |-
       Whether the pipeline is in Development mode. Defaults to false.
+  "dry_run": {}
   "edition":
     "description": |-
       Pipeline product edition.
@@ -425,6 +429,7 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "restart_window":
     "description": |-
       Restart window of this pipeline.
+  "run_as": {}
   "schema":
     "description": |-
       The default schema (database) where tables are read from or published to. The presence of this field implies that the pipeline is in direct publishing mode.
@@ -2624,6 +2629,18 @@ github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow:
     "description": |-
       Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.
       If not specified, UTC will be used.
+github.com/databricks/databricks-sdk-go/service/pipelines.RunAs:
+  "_":
+    "description": |-
+      Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
+
+      Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.
+  "service_principal_name":
+    "description": |-
+      Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.
+  "user_name":
+    "description": |-
+      The email of an active workspace user. Users can only set this field to their own email.
 github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec:
   "destination_catalog":
     "description": |-

bundle/internal/schema/annotations_openapi_overrides.yml

@@ -60,7 +60,6 @@ github.com/databricks/cli/bundle/config/resources.Cluster:
   "_":
     "markdown_description": |-
       The cluster resource defines an [all-purpose cluster](/api/workspace/clusters/create).
-
     "markdown_examples": |-
       The following example creates a cluster named `my_cluster` and sets that as the cluster to use to run the notebook in `my_job`:
 
@@ -123,7 +122,6 @@ github.com/databricks/cli/bundle/config/resources.Dashboard:
       If you use the UI to modify the dashboard, modifications made through the UI are not applied to the dashboard JSON file in the local bundle unless you explicitly update it using `bundle generate`. You can use the `--watch` option to continuously poll and retrieve changes to the dashboard. See [_](/dev-tools/cli/bundle-commands.md#generate).
 
       In addition, if you attempt to deploy a bundle that contains a dashboard JSON file that is different than the one in the remote workspace, an error will occur. To force the deploy and overwrite the dashboard in the remote workspace with the local one, use the `--force` option. See [_](/dev-tools/cli/bundle-commands.md#deploy).
-
   "embed_credentials":
     "description": |-
       PLACEHOLDER
@@ -241,9 +239,15 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
               - notebook:
                   path: ./pipeline.py
       ```
+  "dry_run":
+    "description": |-
+      PLACEHOLDER
   "permissions":
     "description": |-
       PLACEHOLDER
+  "run_as":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.QualityMonitor:
   "_":
     "markdown_description": |-
@@ -356,7 +360,6 @@ github.com/databricks/cli/bundle/config/resources.Volume:
       - A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use <DABS> to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.
 
       - Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [_](/dev-tools/bundles/deployment-modes.md#custom-presets).
-
     "markdown_examples": |-
       The following example creates a <UC> volume with the key `my_volume`:
 
@@ -376,6 +379,42 @@ github.com/databricks/cli/bundle/config/resources.Volume:
   "volume_type":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
+  "create_time":
+    "description": |-
+      PLACEHOLDER
+  "creator":
+    "description": |-
+      PLACEHOLDER
+  "deployment_artifacts":
+    "description": |-
+      PLACEHOLDER
+  "deployment_id":
+    "description": |-
+      PLACEHOLDER
+  "mode":
+    "description": |-
+      PLACEHOLDER
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+  "status":
+    "description": |-
+      PLACEHOLDER
+  "update_time":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
+  "source_code_path":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "job":
     "description": |-
@@ -389,6 +428,49 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "sql_warehouse":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
+  "key":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+  "scope":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
+  "id":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state":
+    "description": |-
+      PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
+  "message":
+    "description": |-
+      PLACEHOLDER
+  "state": {}
 github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes:
   "availability":
     "description": |-
@@ -473,85 +555,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeployment:
-  "create_time":
-    "description": |-
-      PLACEHOLDER
-  "creator":
-    "description": |-
-      PLACEHOLDER
-  "deployment_artifacts":
-    "description": |-
-      PLACEHOLDER
-  "deployment_id":
-    "description": |-
-      PLACEHOLDER
-  "mode":
-    "description": |-
-      PLACEHOLDER
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-  "status":
-    "description": |-
-      PLACEHOLDER
-  "update_time":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts:
-  "source_code_path":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret:
-  "key":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-  "scope":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint:
-  "name":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse:
-  "id":
-    "description": |-
-      PLACEHOLDER
-  "permission":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
-    "description": |-
-      PLACEHOLDER
-github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus:
-  "message":
-    "description": |-
-      PLACEHOLDER
-  "state":
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_version":
     "description": |-

bundle/internal/schema/main.go

@@ -109,6 +109,20 @@ func removeJobsFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
 	return s
 }
 
+func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Schema {
+	switch typ {
+	case reflect.TypeOf(resources.Pipeline{}):
+		// Even though DABs supports this field, TF provider does not. Thus, we
+		// should not expose it to the user.
+		delete(s.Properties, "dry_run")
+		delete(s.Properties, "allow_duplicate_names")
+	default:
+		// Do nothing
+	}
+
+	return s
+}
+
 // While volume_type is required in the volume create API, DABs automatically sets
 // it's value to "MANAGED" if it's not provided. Thus, we make it optional
 // in the bundle schema.
@@ -168,6 +182,7 @@ func generateSchema(workdir, outputFile string) {
 	// Generate the JSON schema from the bundle Go struct.
 	s, err := jsonschema.FromType(reflect.TypeOf(config.Root{}), []func(reflect.Type, jsonschema.Schema) jsonschema.Schema{
 		removeJobsFields,
+		removePipelineFields,
 		makeVolumeTypeOptional,
 		a.addAnnotations,
 		addInterpolationPatterns,

bundle/internal/schema/parser.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -9,8 +10,9 @@ import (
 	"strings"
 
 	"github.com/databricks/cli/bundle/internal/annotation"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/dyn/yamlloader"
 	"github.com/databricks/cli/libs/jsonschema"
-	"gopkg.in/yaml.v3"
 )
 
 type Components struct {
@@ -122,7 +124,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid
 	if err != nil {
 		return err
 	}
-	err = yaml.Unmarshal(b, &overrides)
+	overridesDyn, err := yamlloader.LoadYAML(overridesPath, bytes.NewBuffer(b))
+	if err != nil {
+		return err
+	}
+	err = convert.ToTyped(&overrides, overridesDyn)
 	if err != nil {
 		return err
 	}

bundle/libraries/expand_glob_references.go

@@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic
 
 		for _, match := range matches {
 			output = append(output, dyn.NewValue(map[string]dyn.Value{
-				libType: dyn.V(match),
+				libType: dyn.NewValue(match, lib.Locations()),
 			}, lib.Locations()))
 		}
 	}

bundle/libraries/same_name_libraries.go

@@ -0,0 +1,97 @@
+package libraries
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+)
+
+type checkForSameNameLibraries struct{}
+
+var patterns = []dyn.Pattern{
+	taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
+	forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
+	envDepsPattern.Append(dyn.AnyIndex()),
+}
+
+type libData struct {
+	fullPath  string
+	locations []dyn.Location
+	paths     []dyn.Path
+}
+
+func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	libs := make(map[string]*libData)
+
+	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
+		var err error
+		for _, pattern := range patterns {
+			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
+				libPath := lv.MustString()
+				// If not local library, skip the check
+				if !IsLibraryLocal(libPath) {
+					return lv, nil
+				}
+
+				libFullPath := lv.MustString()
+				lib := filepath.Base(libFullPath)
+				// If the same basename was seen already but full path is different
+				// then it's a duplicate. Add the location to the location list.
+				lp, ok := libs[lib]
+				if !ok {
+					libs[lib] = &libData{
+						fullPath:  libFullPath,
+						locations: []dyn.Location{lv.Location()},
+						paths:     []dyn.Path{p},
+					}
+				} else if lp.fullPath != libFullPath {
+					lp.locations = append(lp.locations, lv.Location())
+					lp.paths = append(lp.paths, p)
+				}
+
+				return lv, nil
+			})
+			if err != nil {
+				return dyn.InvalidValue, err
+			}
+		}
+
+		if err != nil {
+			return dyn.InvalidValue, err
+		}
+
+		return v, nil
+	})
+
+	// Iterate over all the libraries and check if there are any duplicates.
+	// Duplicates will have more than one location.
+	// If there are duplicates, add a diagnostic.
+	for lib, lv := range libs {
+		if len(lv.locations) > 1 {
+			diags = append(diags, diag.Diagnostic{
+				Severity:  diag.Error,
+				Summary:   "Duplicate local library name " + lib,
+				Detail:    "Local library names must be unique",
+				Locations: lv.locations,
+				Paths:     lv.paths,
+			})
+		}
+	}
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+
+	return diags
+}
+
+func (c checkForSameNameLibraries) Name() string {
+	return "CheckForSameNameLibraries"
+}
+
+func CheckForSameNameLibraries() bundle.Mutator {
+	return checkForSameNameLibraries{}
+}

bundle/libraries/same_name_libraries_test.go

@@ -0,0 +1,121 @@
+package libraries
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/bundletest"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSameNameLibraries(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"test": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test.whl",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "other/path/test.whl",
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{
+		{File: "databricks.yml", Line: 10, Column: 1},
+	})
+	bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{
+		{File: "databricks.yml", Line: 20, Column: 1},
+	})
+
+	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
+	require.Len(t, diags, 1)
+	require.Equal(t, diag.Error, diags[0].Severity)
+	require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary)
+	require.Equal(t, []dyn.Location{
+		{File: "databricks.yml", Line: 10, Column: 1},
+		{File: "databricks.yml", Line: 20, Column: 1},
+	}, diags[0].Locations)
+
+	paths := make([]string, 0)
+	for _, p := range diags[0].Paths {
+		paths = append(paths, p.String())
+	}
+	require.Equal(t, []string{
+		"resources.jobs.test.tasks[0].libraries[0].whl",
+		"resources.jobs.test.tasks[1].libraries[0].whl",
+	}, paths)
+}
+
+func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"test": {
+						JobSettings: &jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test-0.1.1.whl",
+										},
+
+										{
+											Whl: "cowsay",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "other/path/test-0.1.0.whl",
+										},
+
+										{
+											Whl: "cowsay",
+										},
+									},
+								},
+								{
+									Libraries: []compute.Library{
+										{
+											Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
+	require.Empty(t, diags)
+}

bundle/mutator.go

@@ -42,7 +42,7 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics {
 	// such that they are not logged multiple times.
 	// If this is done, we can omit this block.
 	if err := diags.Error(); err != nil {
-		log.Errorf(ctx, "Error: %s", err)
+		log.Debugf(ctx, "Error: %s", err)
 	}
 
 	return diags

bundle/mutator_read_only.go

@@ -22,7 +22,7 @@ func ApplyReadOnly(ctx context.Context, rb ReadOnlyBundle, m ReadOnlyMutator) di
 	log.Debugf(ctx, "ApplyReadOnly")
 	diags := m.Apply(ctx, rb)
 	if err := diags.Error(); err != nil {
-		log.Errorf(ctx, "Error: %s", err)
+		log.Debugf(ctx, "Error: %s", err)
 	}
 
 	return diags

bundle/permissions/workspace_root_test.go

@@ -38,8 +38,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
 					"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}},
-					"pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}},
 				},
 				Models: map[string]*resources.MlflowModel{
 					"model_1": {Model: &ml.Model{}},
@@ -98,8 +98,8 @@ func TestApplyWorkspaceRootPermissionsForAllPaths(t *testing.T) {
 					"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
 				},
 				Pipelines: map[string]*resources.Pipeline{
-					"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_1": {CreatePipeline: &pipelines.CreatePipeline{}},
-					"pipeline_2": {PipelineSpec: &pipelines.PipelineSpec{}},
+					"pipeline_2": {CreatePipeline: &pipelines.CreatePipeline{}},
 				},
 				Models: map[string]*resources.MlflowModel{
 					"model_1": {Model: &ml.Model{}},

bundle/phases/deploy.go

@@ -155,6 +155,11 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 				mutator.ValidateGitDetails(),
 				artifacts.CleanUp(),
 				libraries.ExpandGlobReferences(),
+				// libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we
+				// know what are the actual library paths.
+				// libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this
+				// mutator is part of the deploy step rather than validate.
+				libraries.CheckForSameNameLibraries(),
 				libraries.Upload(),
 				trampoline.TransformWheelTask(),
 				files.Upload(outputHandler),

bundle/render/render_text_output_test.go

@@ -530,12 +530,12 @@ func TestRenderSummary(t *testing.T) {
 					"pipeline2": {
 						ID: "4",
 						// no URL
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline2-name"},
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline2-name"},
 					},
 					"pipeline1": {
-						ID:           "3",
+						ID:             "3",
-						URL:          "https://url3",
+						URL:            "https://url3",
-						PipelineSpec: &pipelines.PipelineSpec{Name: "pipeline1-name"},
+						CreatePipeline: &pipelines.CreatePipeline{Name: "pipeline1-name"},
 					},
 				},
 				Schemas: map[string]*resources.Schema{

bundle/resources/completion_test.go

@@ -25,7 +25,7 @@ func TestCompletions_SkipDuplicates(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
@@ -50,7 +50,7 @@ func TestCompletions_Filter(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"bar": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},

bundle/resources/lookup_test.go

@@ -56,7 +56,7 @@ func TestLookup_MultipleFound(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"foo": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},
@@ -107,7 +107,7 @@ func TestLookup_NominalWithFilters(t *testing.T) {
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"bar": {
-						PipelineSpec: &pipelines.PipelineSpec{},
+						CreatePipeline: &pipelines.CreatePipeline{},
 					},
 				},
 			},