From bfde3585b9df9eb5f38a4bed63d2f85426ad1119 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 12 Feb 2025 12:45:01 +0100
Subject: [PATCH 01/11] acc: Fix priority of stubs in test.toml (#2339)

## Changes
Reverse the order of stubs to match expectation (leaf configuration
takes precedence over parent configuration).
Follow up to #2334 .

## Tests
acceptance/selftest/server is extended with duplicate handler
---
 acceptance/acceptance_test.go        | 4 ++++
 acceptance/selftest/server/test.toml | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 117172f60..e61166c31 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -259,6 +259,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 			server.RecordRequests = config.RecordRequests
 			server.IncludeRequestHeaders = config.IncludeRequestHeaders
 
+			// We want later stubs takes precedence, because then leaf configs take precedence over parent directory configs
+			// In gorilla/mux earlier handlers take precedence, so we need to reverse the order
+			slices.Reverse(config.Server)
+
 			for _, stub := range config.Server {
 				require.NotEmpty(t, stub.Pattern)
 				items := strings.Split(stub.Pattern, " ")
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
index 2531fb910..fca41bf02 100644
--- a/acceptance/selftest/server/test.toml
+++ b/acceptance/selftest/server/test.toml
@@ -1,6 +1,10 @@
 LocalOnly = true
 RecordRequests = true
 
+[[Server]]
+Pattern = "GET /custom/endpoint"
+Response.Body = '''should not see this response, latter response takes precedence'''
+
 [[Server]]
 Pattern = "GET /custom/endpoint"
 Response.Body = '''custom

From 4034766c939620c062a0dbad2207eaad6821c54b Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Wed, 12 Feb 2025 14:00:57 +0100
Subject: [PATCH 02/11] acc: Simplify writing handlers; support headers in
 responses (#2338)

## Changes

Handlers now receive testserver.Request and return any which could be
- string or []byte (returns it as is but sets content-type to json or
plain text depending on content)
- struct (encodes it as json and sets content-type to json)
- testserver.Response (full control over status and headers)

Note if testserver.Response is returned from the handler, it's Body
attribute can still be an object. In that case, it'll be serialized and
appropriate content-type header will be added.

The config is now using the same testserver.Response struct, the same
logic applies both configured responses and responses returned from
handlers.

As a result, one can set headers both in Golang handlers and in
test.toml.

This also fixes a bug with RecordRequest not seeing the body if it was
already consumed by the handler.

## Tests
- Existing rests.
- acceptance/selftest/server is extended to set response header.
---
 acceptance/acceptance_test.go                 |   9 +-
 .../bundle/debug/out.stderr.parallel.txt      |   6 +-
 acceptance/bundle/debug/out.stderr.txt        |   3 +-
 acceptance/cmd_server_test.go                 |   8 +-
 acceptance/config_test.go                     |   6 +-
 acceptance/selftest/server/out.requests.txt   |   4 +
 acceptance/selftest/server/output.txt         |   8 +-
 acceptance/selftest/server/script             |   2 +
 acceptance/selftest/server/test.toml          |   2 +
 acceptance/server_test.go                     | 132 ++++-----
 libs/testserver/fake_workspace.go             |  98 +++----
 libs/testserver/server.go                     | 251 ++++++++++++++----
 12 files changed, 336 insertions(+), 193 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index e61166c31..85c345032 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -267,12 +266,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 				require.NotEmpty(t, stub.Pattern)
 				items := strings.Split(stub.Pattern, " ")
 				require.Len(t, items, 2)
-				server.Handle(items[0], items[1], func(fakeWorkspace *testserver.FakeWorkspace, req *http.Request) (any, int) {
-					statusCode := http.StatusOK
-					if stub.Response.StatusCode != 0 {
-						statusCode = stub.Response.StatusCode
-					}
-					return stub.Response.Body, statusCode
+				server.Handle(items[0], items[1], func(req testserver.Request) any {
+					return stub.Response
 				})
 			}
 
diff --git a/acceptance/bundle/debug/out.stderr.parallel.txt b/acceptance/bundle/debug/out.stderr.parallel.txt
index 7dd770068..13c81c511 100644
--- a/acceptance/bundle/debug/out.stderr.parallel.txt
+++ b/acceptance/bundle/debug/out.stderr.parallel.txt
@@ -9,7 +9,7 @@
 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:folder_permissions
 10:07:59 Debug: ApplyReadOnly pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:validate_sync_patterns
 10:07:59 Debug: Path /Workspace/Users/[USERNAME]/.bundle/debug/default/files has type directory (ID: 0) pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync
-10:07:59 Debug: non-retriable error:  pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
-< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
-< {} pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+10:07:59 Debug: non-retriable error: Workspace path not found pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< HTTP/0.0 000 OK pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
+< } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
 < } pid=12345 mutator=validate mutator (read-only)=parallel mutator (read-only)=validate:files_to_sync sdk=true
diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt
index 9cac8bb2b..e5867e008 100644
--- a/acceptance/bundle/debug/out.stderr.txt
+++ b/acceptance/bundle/debug/out.stderr.txt
@@ -79,11 +79,12 @@
 10:07:59 Debug: Apply pid=12345 mutator=validate
 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
 < HTTP/1.1 404 Not Found
+< {
+<   "message": "Workspace path not found"
 10:07:59 Debug: POST /api/2.0/workspace/mkdirs
 > {
 >   "path": "/Workspace/Users/[USERNAME]/.bundle/debug/default/files"
 > }
-< HTTP/1.1 200 OK
 10:07:59 Debug: GET /api/2.0/workspace/get-status?path=/Workspace/Users/[USERNAME]/.bundle/debug/default/files
 < HTTP/1.1 200 OK
 < {
diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go
index d3db06003..dc48a85d7 100644
--- a/acceptance/cmd_server_test.go
+++ b/acceptance/cmd_server_test.go
@@ -1,8 +1,8 @@
 package acceptance_test
 
 import (
+	"context"
 	"encoding/json"
-	"net/http"
 	"os"
 	"strings"
 	"testing"
@@ -14,7 +14,7 @@ import (
 
 func StartCmdServer(t *testing.T) *testserver.Server {
 	server := testserver.New(t)
-	server.Handle("GET", "/", func(_ *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/", func(r testserver.Request) any {
 		q := r.URL.Query()
 		args := strings.Split(q.Get("args"), " ")
 
@@ -27,7 +27,7 @@ func StartCmdServer(t *testing.T) *testserver.Server {
 
 		defer Chdir(t, q.Get("cwd"))()
 
-		c := testcli.NewRunner(t, r.Context(), args...)
+		c := testcli.NewRunner(t, context.Background(), args...)
 		c.Verbose = false
 		stdout, stderr, err := c.Run()
 		result := map[string]any{
@@ -39,7 +39,7 @@ func StartCmdServer(t *testing.T) *testserver.Server {
 			exitcode = 1
 		}
 		result["exitcode"] = exitcode
-		return result, http.StatusOK
+		return result
 	})
 	return server
 }
diff --git a/acceptance/config_test.go b/acceptance/config_test.go
index 920e713a1..ec0d1baee 100644
--- a/acceptance/config_test.go
+++ b/acceptance/config_test.go
@@ -10,6 +10,7 @@ import (
 	"dario.cat/mergo"
 	"github.com/BurntSushi/toml"
 	"github.com/databricks/cli/libs/testdiff"
+	"github.com/databricks/cli/libs/testserver"
 	"github.com/stretchr/testify/require"
 )
 
@@ -56,10 +57,7 @@ type ServerStub struct {
 	Pattern string
 
 	// The response body to return.
-	Response struct {
-		Body       string
-		StatusCode int
-	}
+	Response testserver.Response
 }
 
 // FindConfigs finds all the config relevant for this test,
diff --git a/acceptance/selftest/server/out.requests.txt b/acceptance/selftest/server/out.requests.txt
index 2cb8708ac..34f4c4899 100644
--- a/acceptance/selftest/server/out.requests.txt
+++ b/acceptance/selftest/server/out.requests.txt
@@ -6,3 +6,7 @@
   "method": "GET",
   "path": "/custom/endpoint"
 }
+{
+  "method": "GET",
+  "path": "/api/2.0/workspace/get-status"
+}
diff --git a/acceptance/selftest/server/output.txt b/acceptance/selftest/server/output.txt
index f9e51caa9..7147f9c9b 100644
--- a/acceptance/selftest/server/output.txt
+++ b/acceptance/selftest/server/output.txt
@@ -6,10 +6,16 @@
 }
 >>> curl -sD - [DATABRICKS_URL]/custom/endpoint?query=param
 HTTP/1.1 201 Created
-Content-Type: application/json
+X-Custom-Header: hello
 Date: (redacted)
 Content-Length: (redacted)
+Content-Type: text/plain; charset=utf-8
 
 custom
 ---
 response
+
+>>> errcode [CLI] workspace get-status /a/b/c
+Error: Workspace path not found
+
+Exit code: 1
diff --git a/acceptance/selftest/server/script b/acceptance/selftest/server/script
index 53e2c4b8a..810ea64b6 100644
--- a/acceptance/selftest/server/script
+++ b/acceptance/selftest/server/script
@@ -1,2 +1,4 @@
 trace curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me
 trace curl -sD - $DATABRICKS_HOST/custom/endpoint?query=param
+
+trace errcode $CLI workspace get-status /a/b/c
diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml
index fca41bf02..43ad1e85b 100644
--- a/acceptance/selftest/server/test.toml
+++ b/acceptance/selftest/server/test.toml
@@ -12,6 +12,8 @@ Response.Body = '''custom
 response
 '''
 Response.StatusCode = 201
+[Server.Response.Headers]
+"X-Custom-Header" = ["hello"]
 
 [[Repls]]
 Old = 'Date: .*'
diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index fd8006b8f..f73872e0b 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -1,14 +1,12 @@
 package acceptance_test
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
 
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/iam"
-	"github.com/gorilla/mux"
 
 	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
@@ -23,7 +21,7 @@ var testUser = iam.User{
 }
 
 func AddHandlers(server *testserver.Server) {
-	server.Handle("GET", "/api/2.0/policies/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/policies/clusters/list", func(req testserver.Request) any {
 		return compute.ListPoliciesResponse{
 			Policies: []compute.Policy{
 				{
@@ -35,10 +33,10 @@ func AddHandlers(server *testserver.Server) {
 					Name:     "some-test-cluster-policy",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/instance-pools/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.0/instance-pools/list", func(req testserver.Request) any {
 		return compute.ListInstancePools{
 			InstancePools: []compute.InstancePoolAndStats{
 				{
@@ -46,10 +44,10 @@ func AddHandlers(server *testserver.Server) {
 					InstancePoolId:   "1234",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.1/clusters/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/clusters/list", func(req testserver.Request) any {
 		return compute.ListClustersResponse{
 			Clusters: []compute.ClusterDetails{
 				{
@@ -61,74 +59,57 @@ func AddHandlers(server *testserver.Server) {
 					ClusterId:   "9876",
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return testUser, http.StatusOK
+	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any {
+		return testUser
 	})
 
-	server.Handle("GET", "/api/2.0/workspace/get-status", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-
-		return fakeWorkspace.WorkspaceGetStatus(path)
+	server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		return req.Workspace.WorkspaceGetStatus(path)
 	})
 
-	server.Handle("POST", "/api/2.0/workspace/mkdirs", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		request := workspace.Mkdirs{}
-		decoder := json.NewDecoder(r.Body)
-
-		err := decoder.Decode(&request)
-		if err != nil {
-			return internalError(err)
+	server.Handle("POST", "/api/2.0/workspace/mkdirs", func(req testserver.Request) any {
+		var request workspace.Mkdirs
+		if err := json.Unmarshal(req.Body, &request); err != nil {
+			return testserver.Response{
+				Body:       fmt.Sprintf("internal error: %s", err),
+				StatusCode: http.StatusInternalServerError,
+			}
 		}
 
-		return fakeWorkspace.WorkspaceMkdirs(request)
+		req.Workspace.WorkspaceMkdirs(request)
+		return ""
 	})
 
-	server.Handle("GET", "/api/2.0/workspace/export", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-
-		return fakeWorkspace.WorkspaceExport(path)
+	server.Handle("GET", "/api/2.0/workspace/export", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		return req.Workspace.WorkspaceExport(path)
 	})
 
-	server.Handle("POST", "/api/2.0/workspace/delete", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		path := r.URL.Query().Get("path")
-		recursiveStr := r.URL.Query().Get("recursive")
-		var recursive bool
-
-		if recursiveStr == "true" {
-			recursive = true
-		} else {
-			recursive = false
-		}
-
-		return fakeWorkspace.WorkspaceDelete(path, recursive)
+	server.Handle("POST", "/api/2.0/workspace/delete", func(req testserver.Request) any {
+		path := req.URL.Query().Get("path")
+		recursive := req.URL.Query().Get("recursive") == "true"
+		req.Workspace.WorkspaceDelete(path, recursive)
+		return ""
 	})
 
-	server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		vars := mux.Vars(r)
-		path := vars["path"]
-
-		body := new(bytes.Buffer)
-		_, err := body.ReadFrom(r.Body)
-		if err != nil {
-			return internalError(err)
-		}
-
-		return fakeWorkspace.WorkspaceFilesImportFile(path, body.Bytes())
+	server.Handle("POST", "/api/2.0/workspace-files/import-file/{path:.*}", func(req testserver.Request) any {
+		path := req.Vars["path"]
+		req.Workspace.WorkspaceFilesImportFile(path, req.Body)
+		return ""
 	})
 
-	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/api/2.1/unity-catalog/current-metastore-assignment", func(req testserver.Request) any {
 		return catalog.MetastoreAssignment{
 			DefaultCatalogName: "main",
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		vars := mux.Vars(r)
-		objectId := vars["objectId"]
-
+	server.Handle("GET", "/api/2.0/permissions/directories/{objectId}", func(req testserver.Request) any {
+		objectId := req.Vars["objectId"]
 		return workspace.WorkspaceObjectPermissions{
 			ObjectId:   objectId,
 			ObjectType: "DIRECTORY",
@@ -142,48 +123,43 @@ func AddHandlers(server *testserver.Server) {
 					},
 				},
 			},
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("POST", "/api/2.1/jobs/create", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		request := jobs.CreateJob{}
-		decoder := json.NewDecoder(r.Body)
-
-		err := decoder.Decode(&request)
-		if err != nil {
-			return internalError(err)
+	server.Handle("POST", "/api/2.1/jobs/create", func(req testserver.Request) any {
+		var request jobs.CreateJob
+		if err := json.Unmarshal(req.Body, &request); err != nil {
+			return testserver.Response{
+				Body:       fmt.Sprintf("internal error: %s", err),
+				StatusCode: 500,
+			}
 		}
 
-		return fakeWorkspace.JobsCreate(request)
+		return req.Workspace.JobsCreate(request)
 	})
 
-	server.Handle("GET", "/api/2.1/jobs/get", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		jobId := r.URL.Query().Get("job_id")
-
-		return fakeWorkspace.JobsGet(jobId)
+	server.Handle("GET", "/api/2.1/jobs/get", func(req testserver.Request) any {
+		jobId := req.URL.Query().Get("job_id")
+		return req.Workspace.JobsGet(jobId)
 	})
 
-	server.Handle("GET", "/api/2.1/jobs/list", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
-		return fakeWorkspace.JobsList()
+	server.Handle("GET", "/api/2.1/jobs/list", func(req testserver.Request) any {
+		return req.Workspace.JobsList()
 	})
 
-	server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("GET", "/oidc/.well-known/oauth-authorization-server", func(_ testserver.Request) any {
 		return map[string]string{
 			"authorization_endpoint": server.URL + "oidc/v1/authorize",
 			"token_endpoint":         server.URL + "/oidc/v1/token",
-		}, http.StatusOK
+		}
 	})
 
-	server.Handle("POST", "/oidc/v1/token", func(fakeWorkspace *testserver.FakeWorkspace, r *http.Request) (any, int) {
+	server.Handle("POST", "/oidc/v1/token", func(_ testserver.Request) any {
 		return map[string]string{
 			"access_token": "oauth-token",
 			"expires_in":   "3600",
 			"scope":        "all-apis",
 			"token_type":   "Bearer",
-		}, http.StatusOK
+		}
 	})
 }
-
-func internalError(err error) (any, int) {
-	return fmt.Errorf("internal error: %w", err), http.StatusInternalServerError
-}
diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go
index c3e4f9a71..4e943f828 100644
--- a/libs/testserver/fake_workspace.go
+++ b/libs/testserver/fake_workspace.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -33,40 +32,39 @@ func NewFakeWorkspace() *FakeWorkspace {
 	}
 }
 
-func (s *FakeWorkspace) WorkspaceGetStatus(path string) (workspace.ObjectInfo, int) {
+func (s *FakeWorkspace) WorkspaceGetStatus(path string) Response {
 	if s.directories[path] {
-		return workspace.ObjectInfo{
-			ObjectType: "DIRECTORY",
-			Path:       path,
-		}, http.StatusOK
+		return Response{
+			Body: &workspace.ObjectInfo{
+				ObjectType: "DIRECTORY",
+				Path:       path,
+			},
+		}
 	} else if _, ok := s.files[path]; ok {
-		return workspace.ObjectInfo{
-			ObjectType: "FILE",
-			Path:       path,
-			Language:   "SCALA",
-		}, http.StatusOK
+		return Response{
+			Body: &workspace.ObjectInfo{
+				ObjectType: "FILE",
+				Path:       path,
+				Language:   "SCALA",
+			},
+		}
 	} else {
-		return workspace.ObjectInfo{}, http.StatusNotFound
+		return Response{
+			StatusCode: 404,
+			Body:       map[string]string{"message": "Workspace path not found"},
+		}
 	}
 }
 
-func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) (string, int) {
+func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) {
 	s.directories[request.Path] = true
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) WorkspaceExport(path string) ([]byte, int) {
-	file := s.files[path]
-
-	if file == nil {
-		return nil, http.StatusNotFound
-	}
-
-	return file, http.StatusOK
+func (s *FakeWorkspace) WorkspaceExport(path string) []byte {
+	return s.files[path]
 }
 
-func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, int) {
+func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) {
 	if !recursive {
 		s.files[path] = nil
 	} else {
@@ -76,28 +74,26 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) (string, in
 			}
 		}
 	}
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) (any, int) {
+func (s *FakeWorkspace) WorkspaceFilesImportFile(path string, body []byte) {
 	if !strings.HasPrefix(path, "/") {
 		path = "/" + path
 	}
-
 	s.files[path] = body
-
-	return "{}", http.StatusOK
 }
 
-func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) {
+func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) Response {
 	jobId := s.nextJobId
 	s.nextJobId++
 
 	jobSettings := jobs.JobSettings{}
 	err := jsonConvert(request, &jobSettings)
 	if err != nil {
-		return internalError(err)
+		return Response{
+			StatusCode: 400,
+			Body:       fmt.Sprintf("Cannot convert request to jobSettings: %s", err),
+		}
 	}
 
 	s.jobs[jobId] = jobs.Job{
@@ -105,32 +101,44 @@ func (s *FakeWorkspace) JobsCreate(request jobs.CreateJob) (any, int) {
 		Settings: &jobSettings,
 	}
 
-	return jobs.CreateResponse{JobId: jobId}, http.StatusOK
+	return Response{
+		Body: jobs.CreateResponse{JobId: jobId},
+	}
 }
 
-func (s *FakeWorkspace) JobsGet(jobId string) (any, int) {
+func (s *FakeWorkspace) JobsGet(jobId string) Response {
 	id := jobId
 
 	jobIdInt, err := strconv.ParseInt(id, 10, 64)
 	if err != nil {
-		return internalError(fmt.Errorf("failed to parse job id: %s", err))
+		return Response{
+			StatusCode: 400,
+			Body:       fmt.Sprintf("Failed to parse job id: %s: %v", err, id),
+		}
 	}
 
 	job, ok := s.jobs[jobIdInt]
 	if !ok {
-		return jobs.Job{}, http.StatusNotFound
+		return Response{
+			StatusCode: 404,
+		}
 	}
 
-	return job, http.StatusOK
+	return Response{
+		Body: job,
+	}
 }
 
-func (s *FakeWorkspace) JobsList() (any, int) {
+func (s *FakeWorkspace) JobsList() Response {
 	list := make([]jobs.BaseJob, 0, len(s.jobs))
 	for _, job := range s.jobs {
 		baseJob := jobs.BaseJob{}
 		err := jsonConvert(job, &baseJob)
 		if err != nil {
-			return internalError(fmt.Errorf("failed to convert job to base job: %w", err))
+			return Response{
+				StatusCode: 400,
+				Body:       fmt.Sprintf("failed to convert job to base job: %s", err),
+			}
 		}
 
 		list = append(list, baseJob)
@@ -141,9 +149,11 @@ func (s *FakeWorkspace) JobsList() (any, int) {
 		return list[i].JobId < list[j].JobId
 	})
 
-	return jobs.ListJobsResponse{
-		Jobs: list,
-	}, http.StatusOK
+	return Response{
+		Body: jobs.ListJobsResponse{
+			Jobs: list,
+		},
+	}
 }
 
 // jsonConvert saves input to a value pointed by output
@@ -163,7 +173,3 @@ func jsonConvert(input, output any) error {
 
 	return nil
 }
-
-func internalError(err error) (string, int) {
-	return fmt.Sprintf("internal error: %s", err), http.StatusInternalServerError
-}
diff --git a/libs/testserver/server.go b/libs/testserver/server.go
index cf4d5aca2..fa15973d7 100644
--- a/libs/testserver/server.go
+++ b/libs/testserver/server.go
@@ -5,14 +5,14 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"reflect"
 	"slices"
 	"strings"
 	"sync"
 
 	"github.com/gorilla/mux"
 
-	"github.com/stretchr/testify/assert"
-
 	"github.com/databricks/cli/internal/testutil"
 	"github.com/databricks/databricks-sdk-go/apierr"
 )
@@ -29,10 +29,10 @@ type Server struct {
 	RecordRequests        bool
 	IncludeRequestHeaders []string
 
-	Requests []Request
+	Requests []LoggedRequest
 }
 
-type Request struct {
+type LoggedRequest struct {
 	Headers http.Header `json:"headers,omitempty"`
 	Method  string      `json:"method"`
 	Path    string      `json:"path"`
@@ -40,6 +40,153 @@ type Request struct {
 	RawBody string      `json:"raw_body,omitempty"`
 }
 
+type Request struct {
+	Method    string
+	URL       *url.URL
+	Headers   http.Header
+	Body      []byte
+	Vars      map[string]string
+	Workspace *FakeWorkspace
+}
+
+type Response struct {
+	StatusCode int
+	Headers    http.Header
+	Body       any
+}
+
+type encodedResponse struct {
+	StatusCode int
+	Headers    http.Header
+	Body       []byte
+}
+
+func NewRequest(t testutil.TestingT, r *http.Request, fakeWorkspace *FakeWorkspace) Request {
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		t.Fatalf("Failed to read request body: %s", err)
+	}
+
+	return Request{
+		Method:    r.Method,
+		URL:       r.URL,
+		Headers:   r.Header,
+		Body:      body,
+		Vars:      mux.Vars(r),
+		Workspace: fakeWorkspace,
+	}
+}
+
+func normalizeResponse(t testutil.TestingT, resp any) encodedResponse {
+	result := normalizeResponseBody(t, resp)
+	if result.StatusCode == 0 {
+		result.StatusCode = 200
+	}
+	return result
+}
+
+func normalizeResponseBody(t testutil.TestingT, resp any) encodedResponse {
+	if isNil(resp) {
+		t.Errorf("Handler must not return nil")
+		return encodedResponse{StatusCode: 500}
+	}
+
+	respBytes, ok := resp.([]byte)
+	if ok {
+		return encodedResponse{
+			Body:    respBytes,
+			Headers: getHeaders(respBytes),
+		}
+	}
+
+	respString, ok := resp.(string)
+	if ok {
+		return encodedResponse{
+			Body:    []byte(respString),
+			Headers: getHeaders([]byte(respString)),
+		}
+	}
+
+	respStruct, ok := resp.(Response)
+	if ok {
+		if isNil(respStruct.Body) {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       []byte{},
+			}
+		}
+
+		bytesVal, isBytes := respStruct.Body.([]byte)
+		if isBytes {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       bytesVal,
+			}
+		}
+
+		stringVal, isString := respStruct.Body.(string)
+		if isString {
+			return encodedResponse{
+				StatusCode: respStruct.StatusCode,
+				Headers:    respStruct.Headers,
+				Body:       []byte(stringVal),
+			}
+		}
+
+		respBytes, err := json.MarshalIndent(respStruct.Body, "", "    ")
+		if err != nil {
+			t.Errorf("JSON encoding error: %s", err)
+			return encodedResponse{
+				StatusCode: 500,
+				Body:       []byte("internal error"),
+			}
+		}
+
+		headers := respStruct.Headers
+		if headers == nil {
+			headers = getJsonHeaders()
+		}
+
+		return encodedResponse{
+			StatusCode: respStruct.StatusCode,
+			Headers:    headers,
+			Body:       respBytes,
+		}
+	}
+
+	respBytes, err := json.MarshalIndent(resp, "", "    ")
+	if err != nil {
+		t.Errorf("JSON encoding error: %s", err)
+		return encodedResponse{
+			StatusCode: 500,
+			Body:       []byte("internal error"),
+		}
+	}
+
+	return encodedResponse{
+		Body:    respBytes,
+		Headers: getJsonHeaders(),
+	}
+}
+
+func getJsonHeaders() http.Header {
+	return map[string][]string{
+		"Content-Type": {"application/json"},
+	}
+}
+
+func getHeaders(value []byte) http.Header {
+	if json.Valid(value) {
+		return getJsonHeaders()
+	} else {
+		return map[string][]string{
+			"Content-Type": {"text/plain"},
+		}
+	}
+}
+
 func New(t testutil.TestingT) *Server {
 	router := mux.NewRouter()
 	server := httptest.NewServer(router)
@@ -96,7 +243,7 @@ Response.StatusCode = <response status-code here>
 	return s
 }
 
-type HandlerFunc func(fakeWorkspace *FakeWorkspace, req *http.Request) (resp any, statusCode int)
+type HandlerFunc func(req Request) any
 
 func (s *Server) Handle(method, path string, handler HandlerFunc) {
 	s.Router.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
@@ -117,56 +264,22 @@ func (s *Server) Handle(method, path string, handler HandlerFunc) {
 			fakeWorkspace = s.fakeWorkspaces[token]
 		}
 
-		resp, statusCode := handler(fakeWorkspace, r)
-
+		request := NewRequest(s.t, r, fakeWorkspace)
 		if s.RecordRequests {
-			body, err := io.ReadAll(r.Body)
-			assert.NoError(s.t, err)
-
-			headers := make(http.Header)
-			for k, v := range r.Header {
-				if !slices.Contains(s.IncludeRequestHeaders, k) {
-					continue
-				}
-				for _, vv := range v {
-					headers.Add(k, vv)
-				}
-			}
-
-			req := Request{
-				Headers: headers,
-				Method:  r.Method,
-				Path:    r.URL.Path,
-			}
-
-			if json.Valid(body) {
-				req.Body = json.RawMessage(body)
-			} else {
-				req.RawBody = string(body)
-			}
-
-			s.Requests = append(s.Requests, req)
+			s.Requests = append(s.Requests, getLoggedRequest(request, s.IncludeRequestHeaders))
 		}
 
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(statusCode)
+		respAny := handler(request)
+		resp := normalizeResponse(s.t, respAny)
 
-		var respBytes []byte
-		var err error
-		if respString, ok := resp.(string); ok {
-			respBytes = []byte(respString)
-		} else if respBytes0, ok := resp.([]byte); ok {
-			respBytes = respBytes0
-		} else {
-			respBytes, err = json.MarshalIndent(resp, "", "    ")
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
+		for k, v := range resp.Headers {
+			w.Header()[k] = v
 		}
 
-		if _, err := w.Write(respBytes); err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+		w.WriteHeader(resp.StatusCode)
+
+		if _, err := w.Write(resp.Body); err != nil {
+			s.t.Errorf("Failed to write response: %s", err)
 			return
 		}
 	}).Methods(method)
@@ -182,3 +295,43 @@ func getToken(r *http.Request) string {
 
 	return header[len(prefix):]
 }
+
+func getLoggedRequest(req Request, includedHeaders []string) LoggedRequest {
+	result := LoggedRequest{
+		Method:  req.Method,
+		Path:    req.URL.Path,
+		Headers: filterHeaders(req.Headers, includedHeaders),
+	}
+
+	if json.Valid(req.Body) {
+		result.Body = json.RawMessage(req.Body)
+	} else {
+		result.RawBody = string(req.Body)
+	}
+
+	return result
+}
+
+func filterHeaders(h http.Header, includedHeaders []string) http.Header {
+	headers := make(http.Header)
+	for k, v := range h {
+		if !slices.Contains(includedHeaders, k) {
+			continue
+		}
+		headers[k] = v
+	}
+	return headers
+}
+
+func isNil(i any) bool {
+	if i == nil {
+		return true
+	}
+	v := reflect.ValueOf(i)
+	switch v.Kind() {
+	case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Interface, reflect.Slice:
+		return v.IsNil()
+	default:
+		return false
+	}
+}

From 1dadc794f57b98a833949557f2a61cb44e33146f Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 14:29:23 +0000
Subject: [PATCH 03/11] [Release] Release v0.241.0 (#2340)

Bundles:
* Added support to generate Git based jobs
([#2304](https://github.com/databricks/cli/pull/2304)).
* Added support for run_as in pipelines
([#2287](https://github.com/databricks/cli/pull/2287)).
* Raise an error when there are multiple local libraries with the same
basename used ([#2297](https://github.com/databricks/cli/pull/2297)).
* Fix env variable for AzureCli local config
([#2248](https://github.com/databricks/cli/pull/2248)).
* Accept JSON files in includes section
([#2265](https://github.com/databricks/cli/pull/2265)).
* Always print warnings and errors; clean up format
([#2213](https://github.com/databricks/cli/pull/2213))

API Changes:
 * Added `databricks account budget-policy` command group.
 * Added `databricks lakeview-embedded` command group.
 * Added `databricks query-execution` command group.
 * Added `databricks account enable-ip-access-lists` command group.
 * Added `databricks redash-config` command group.

OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03)
Dependency updates:
* Upgrade to TF provider 1.65.1
([#2328](https://github.com/databricks/cli/pull/2328)).
* Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0
([#2237](https://github.com/databricks/cli/pull/2237)).
* Bump github.com/spf13/pflag from 1.0.5 to 1.0.6
([#2281](https://github.com/databricks/cli/pull/2281)).
* Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0
([#2321](https://github.com/databricks/cli/pull/2321)).
* Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0
([#2322](https://github.com/databricks/cli/pull/2322)).
* Bump golang.org/x/term from 0.28.0 to 0.29.0
([#2325](https://github.com/databricks/cli/pull/2325)).
* Bump golang.org/x/text from 0.21.0 to 0.22.0
([#2323](https://github.com/databricks/cli/pull/2323)).
* Bump golang.org/x/mod from 0.22.0 to 0.23.0
([#2324](https://github.com/databricks/cli/pull/2324)).
---
 CHANGELOG.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 449c30288..c0202b6a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,33 @@
 # Version changelog
 
+## [Release] Release v0.241.0
+
+Bundles:
+ * Added support to generate Git based jobs ([#2304](https://github.com/databricks/cli/pull/2304)).
+ * Added support for run_as in pipelines ([#2287](https://github.com/databricks/cli/pull/2287)).
+ * Raise an error when there are multiple local libraries with the same basename used ([#2297](https://github.com/databricks/cli/pull/2297)).
+ * Fix env variable for AzureCli local config ([#2248](https://github.com/databricks/cli/pull/2248)).
+ * Accept JSON files in includes section ([#2265](https://github.com/databricks/cli/pull/2265)).
+ * Always print warnings and errors; clean up format ([#2213](https://github.com/databricks/cli/pull/2213))
+
+API Changes:
+ * Added `databricks account budget-policy` command group.
+ * Added `databricks lakeview-embedded` command group.
+ * Added `databricks query-execution` command group.
+ * Added `databricks account enable-ip-access-lists` command group.
+ * Added `databricks redash-config` command group.
+
+OpenAPI commit c72c58f97b950fcb924a90ef164bcb10cfcd5ece (2025-02-03)
+Dependency updates:
+ * Upgrade to TF provider 1.65.1 ([#2328](https://github.com/databricks/cli/pull/2328)).
+ * Bump github.com/hashicorp/terraform-exec from 0.21.0 to 0.22.0 ([#2237](https://github.com/databricks/cli/pull/2237)).
+ * Bump github.com/spf13/pflag from 1.0.5 to 1.0.6 ([#2281](https://github.com/databricks/cli/pull/2281)).
+ * Bump github.com/databricks/databricks-sdk-go from 0.56.1 to 0.57.0 ([#2321](https://github.com/databricks/cli/pull/2321)).
+ * Bump golang.org/x/oauth2 from 0.25.0 to 0.26.0 ([#2322](https://github.com/databricks/cli/pull/2322)).
+ * Bump golang.org/x/term from 0.28.0 to 0.29.0 ([#2325](https://github.com/databricks/cli/pull/2325)).
+ * Bump golang.org/x/text from 0.21.0 to 0.22.0 ([#2323](https://github.com/databricks/cli/pull/2323)).
+ * Bump golang.org/x/mod from 0.22.0 to 0.23.0 ([#2324](https://github.com/databricks/cli/pull/2324)).
+
 ## [Release] Release v0.240.0
 
 Bundles:

From ac439f8c1a60f7b5dd046caa88ad3ed93d9e0c51 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 16:14:30 +0000
Subject: [PATCH 04/11] Fix for regression deploying resources with PyPi and
 Maven library types (#2341)

## Changes

The CheckForSameNameLibraries mutator incorrectly assumed all resource
libraries define libraries as paths of the `string` type, but some
libraries, such as PyPi and Maven, define them as objects.

This PR addresses this issue. It was introduced in #2297.

## Tests

Added regression test.
---
 .../artifacts/same_name_libraries/databricks.yml      |  2 ++
 .../bundle/artifacts/same_name_libraries/output.txt   |  2 +-
 bundle/libraries/same_name_libraries.go               | 11 ++++++++---
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
index a065bae76..837cd01e8 100644
--- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
+++ b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
@@ -34,6 +34,8 @@ resources:
             package_name: my_default_python
           libraries:
             - whl: ./whl1/dist/*.whl
+            - pypi:
+                package: test_package
         - task_key: task2
           new_cluster: ${var.cluster}
           python_wheel_task:
diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt
index 38cdd43c4..1253d9680 100644
--- a/acceptance/bundle/artifacts/same_name_libraries/output.txt
+++ b/acceptance/bundle/artifacts/same_name_libraries/output.txt
@@ -6,7 +6,7 @@ Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
   at resources.jobs.test.tasks[0].libraries[0].whl
      resources.jobs.test.tasks[1].libraries[0].whl
   in databricks.yml:36:15
-     databricks.yml:43:15
+     databricks.yml:45:15
 
 Local library names must be unique
 
diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go
index 88b96ab54..8de34cfec 100644
--- a/bundle/libraries/same_name_libraries.go
+++ b/bundle/libraries/same_name_libraries.go
@@ -31,13 +31,18 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle)
 		var err error
 		for _, pattern := range patterns {
 			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
-				libPath := lv.MustString()
+				libFullPath, ok := lv.AsString()
+				// If the value is not a string, skip the check because it's not whl or jar type which defines the library
+				// as a string versus PyPi or Maven which defines the library as a map.
+				if !ok {
+					return v, nil
+				}
+
 				// If not local library, skip the check
-				if !IsLibraryLocal(libPath) {
+				if !IsLibraryLocal(libFullPath) {
 					return lv, nil
 				}
 
-				libFullPath := lv.MustString()
 				lib := filepath.Base(libFullPath)
 				// If the same basename was seen already but full path is different
 				// then it's a duplicate. Add the location to the location list.

From 9c90984688060d9c1c889055ab0336536d2a6732 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 12 Feb 2025 16:17:38 +0000
Subject: [PATCH 05/11] [Release] Release v0.241.1 (#2342)

Bundles:
* Fix for regression deploying resources with PyPi and Maven library
types ([#2341](https://github.com/databricks/cli/pull/2341)).
---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0202b6a2..d3df510b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Version changelog
 
+## [Release] Release v0.241.1
+
+Bundles:
+ * Fix for regression deploying resources with PyPi and Maven library types ([#2341](https://github.com/databricks/cli/pull/2341)).
+
 ## [Release] Release v0.241.0
 
 Bundles:

From 96302c7415331e925fc9d6056b3e34923e84f64f Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 12 Feb 2025 20:05:49 +0100
Subject: [PATCH 06/11] Revert changes related to basename check for local
 libraries (#2345)

## Changes

These changes break the use of non-local libraries (such as PyPI
libraries).

This reverts the set so we can cut a patch release and take a closer
look later.

Original PRs are #2297 and #2341.

Issue reported in #2343.

## Tests

Manually confirmed that a bundle with PyPI package in libraries now
deploys fine.
---
 .../same_name_libraries/databricks.yml        |  52 --------
 .../artifacts/same_name_libraries/output.txt  |  14 --
 .../artifacts/same_name_libraries/script      |   2 -
 .../artifacts/same_name_libraries/test.toml   |   0
 .../same_name_libraries/whl1/setup.py         |  36 ------
 .../whl1/src/my_default_python/__init__.py    |   1 -
 .../whl1/src/my_default_python/main.py        |   1 -
 .../same_name_libraries/whl2/setup.py         |  36 ------
 .../whl2/src/my_default_python/__init__.py    |   1 -
 .../whl2/src/my_default_python/main.py        |   1 -
 bundle/libraries/expand_glob_references.go    |   2 +-
 bundle/libraries/same_name_libraries.go       | 102 ---------------
 bundle/libraries/same_name_libraries_test.go  | 121 ------------------
 bundle/phases/deploy.go                       |   5 -
 14 files changed, 1 insertion(+), 373 deletions(-)
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/databricks.yml
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/output.txt
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/script
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/test.toml
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
 delete mode 100644 acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
 delete mode 100644 bundle/libraries/same_name_libraries.go
 delete mode 100644 bundle/libraries/same_name_libraries_test.go

diff --git a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml b/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
deleted file mode 100644
index 837cd01e8..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/databricks.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-bundle:
-  name: same_name_libraries
-
-variables:
-  cluster:
-    default:
-      spark_version: 15.4.x-scala2.12
-      node_type_id: i3.xlarge
-      data_security_mode: SINGLE_USER
-      num_workers: 0
-      spark_conf:
-          spark.master: "local[*, 4]"
-          spark.databricks.cluster.profile: singleNode
-      custom_tags:
-        ResourceClass: SingleNode
-
-artifacts:
-  whl1:
-    type: whl
-    path: ./whl1
-  whl2:
-    type: whl
-    path: ./whl2
-
-resources:
-  jobs:
-    test:
-      name: "test"
-      tasks:
-        - task_key: task1
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl1/dist/*.whl
-            - pypi:
-                package: test_package
-        - task_key: task2
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl2/dist/*.whl
-        - task_key: task3
-          new_cluster: ${var.cluster}
-          python_wheel_task:
-            entry_point: main
-            package_name: my_default_python
-          libraries:
-            - whl: ./whl1/dist/*.whl
diff --git a/acceptance/bundle/artifacts/same_name_libraries/output.txt b/acceptance/bundle/artifacts/same_name_libraries/output.txt
deleted file mode 100644
index 1253d9680..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/output.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-
->>> errcode [CLI] bundle deploy
-Building whl1...
-Building whl2...
-Error: Duplicate local library name my_default_python-0.0.1-py3-none-any.whl
-  at resources.jobs.test.tasks[0].libraries[0].whl
-     resources.jobs.test.tasks[1].libraries[0].whl
-  in databricks.yml:36:15
-     databricks.yml:45:15
-
-Local library names must be unique
-
-
-Exit code: 1
diff --git a/acceptance/bundle/artifacts/same_name_libraries/script b/acceptance/bundle/artifacts/same_name_libraries/script
deleted file mode 100644
index 6c899df07..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/script
+++ /dev/null
@@ -1,2 +0,0 @@
-trace errcode $CLI bundle deploy
-rm -rf whl1 whl2
diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml
deleted file mode 100644
index e69de29bb..000000000
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
deleted file mode 100644
index 1afaf3a4f..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-setup.py configuration script describing how to build and package this project.
-
-This file is primarily used by the setuptools library and typically should not
-be executed directly. See README.md for how to deploy, test, and run
-the my_default_python project.
-"""
-
-from setuptools import setup, find_packages
-
-import sys
-
-sys.path.append("./src")
-
-import my_default_python
-
-setup(
-    name="my_default_python",
-    version=my_default_python.__version__,
-    url="https://databricks.com",
-    author="[USERNAME]",
-    description="wheel file based on my_default_python/src",
-    packages=find_packages(where="./src"),
-    package_dir={"": "src"},
-    entry_points={
-        "packages": [
-            "main=my_default_python.main:main",
-        ],
-    },
-    install_requires=[
-        # Dependencies in case the output wheel file is used as a library dependency.
-        # For defining dependencies, when this package is used in Databricks, see:
-        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
-        "setuptools"
-    ],
-)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
deleted file mode 100644
index f102a9cad..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
deleted file mode 100644
index 11b15b1a4..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl1/src/my_default_python/main.py
+++ /dev/null
@@ -1 +0,0 @@
-print("hello")
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
deleted file mode 100644
index 1afaf3a4f..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""
-setup.py configuration script describing how to build and package this project.
-
-This file is primarily used by the setuptools library and typically should not
-be executed directly. See README.md for how to deploy, test, and run
-the my_default_python project.
-"""
-
-from setuptools import setup, find_packages
-
-import sys
-
-sys.path.append("./src")
-
-import my_default_python
-
-setup(
-    name="my_default_python",
-    version=my_default_python.__version__,
-    url="https://databricks.com",
-    author="[USERNAME]",
-    description="wheel file based on my_default_python/src",
-    packages=find_packages(where="./src"),
-    package_dir={"": "src"},
-    entry_points={
-        "packages": [
-            "main=my_default_python.main:main",
-        ],
-    },
-    install_requires=[
-        # Dependencies in case the output wheel file is used as a library dependency.
-        # For defining dependencies, when this package is used in Databricks, see:
-        # https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
-        "setuptools"
-    ],
-)
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
deleted file mode 100644
index f102a9cad..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-__version__ = "0.0.1"
diff --git a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py b/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
deleted file mode 100644
index 11b15b1a4..000000000
--- a/acceptance/bundle/artifacts/same_name_libraries/whl2/src/my_default_python/main.py
+++ /dev/null
@@ -1 +0,0 @@
-print("hello")
diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go
index 7a808f627..bb1905045 100644
--- a/bundle/libraries/expand_glob_references.go
+++ b/bundle/libraries/expand_glob_references.go
@@ -92,7 +92,7 @@ func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostic
 
 		for _, match := range matches {
 			output = append(output, dyn.NewValue(map[string]dyn.Value{
-				libType: dyn.NewValue(match, lib.Locations()),
+				libType: dyn.V(match),
 			}, lib.Locations()))
 		}
 	}
diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go
deleted file mode 100644
index 8de34cfec..000000000
--- a/bundle/libraries/same_name_libraries.go
+++ /dev/null
@@ -1,102 +0,0 @@
-package libraries
-
-import (
-	"context"
-	"path/filepath"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-)
-
-type checkForSameNameLibraries struct{}
-
-var patterns = []dyn.Pattern{
-	taskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
-	forEachTaskLibrariesPattern.Append(dyn.AnyIndex(), dyn.AnyKey()),
-	envDepsPattern.Append(dyn.AnyIndex()),
-}
-
-type libData struct {
-	fullPath  string
-	locations []dyn.Location
-	paths     []dyn.Path
-}
-
-func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	var diags diag.Diagnostics
-	libs := make(map[string]*libData)
-
-	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
-		var err error
-		for _, pattern := range patterns {
-			v, err = dyn.MapByPattern(v, pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
-				libFullPath, ok := lv.AsString()
-				// If the value is not a string, skip the check because it's not whl or jar type which defines the library
-				// as a string versus PyPi or Maven which defines the library as a map.
-				if !ok {
-					return v, nil
-				}
-
-				// If not local library, skip the check
-				if !IsLibraryLocal(libFullPath) {
-					return lv, nil
-				}
-
-				lib := filepath.Base(libFullPath)
-				// If the same basename was seen already but full path is different
-				// then it's a duplicate. Add the location to the location list.
-				lp, ok := libs[lib]
-				if !ok {
-					libs[lib] = &libData{
-						fullPath:  libFullPath,
-						locations: []dyn.Location{lv.Location()},
-						paths:     []dyn.Path{p},
-					}
-				} else if lp.fullPath != libFullPath {
-					lp.locations = append(lp.locations, lv.Location())
-					lp.paths = append(lp.paths, p)
-				}
-
-				return lv, nil
-			})
-			if err != nil {
-				return dyn.InvalidValue, err
-			}
-		}
-
-		if err != nil {
-			return dyn.InvalidValue, err
-		}
-
-		return v, nil
-	})
-
-	// Iterate over all the libraries and check if there are any duplicates.
-	// Duplicates will have more than one location.
-	// If there are duplicates, add a diagnostic.
-	for lib, lv := range libs {
-		if len(lv.locations) > 1 {
-			diags = append(diags, diag.Diagnostic{
-				Severity:  diag.Error,
-				Summary:   "Duplicate local library name " + lib,
-				Detail:    "Local library names must be unique",
-				Locations: lv.locations,
-				Paths:     lv.paths,
-			})
-		}
-	}
-	if err != nil {
-		diags = diags.Extend(diag.FromErr(err))
-	}
-
-	return diags
-}
-
-func (c checkForSameNameLibraries) Name() string {
-	return "CheckForSameNameLibraries"
-}
-
-func CheckForSameNameLibraries() bundle.Mutator {
-	return checkForSameNameLibraries{}
-}
diff --git a/bundle/libraries/same_name_libraries_test.go b/bundle/libraries/same_name_libraries_test.go
deleted file mode 100644
index 42c38773b..000000000
--- a/bundle/libraries/same_name_libraries_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package libraries
-
-import (
-	"context"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config"
-	"github.com/databricks/cli/bundle/config/resources"
-	"github.com/databricks/cli/bundle/internal/bundletest"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/databricks/databricks-sdk-go/service/compute"
-	"github.com/databricks/databricks-sdk-go/service/jobs"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSameNameLibraries(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"test": {
-						JobSettings: &jobs.JobSettings{
-							Tasks: []jobs.Task{
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test.whl",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "other/path/test.whl",
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	bundletest.SetLocation(b, "resources.jobs.test.tasks[0]", []dyn.Location{
-		{File: "databricks.yml", Line: 10, Column: 1},
-	})
-	bundletest.SetLocation(b, "resources.jobs.test.tasks[1]", []dyn.Location{
-		{File: "databricks.yml", Line: 20, Column: 1},
-	})
-
-	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
-	require.Len(t, diags, 1)
-	require.Equal(t, diag.Error, diags[0].Severity)
-	require.Equal(t, "Duplicate local library name test.whl", diags[0].Summary)
-	require.Equal(t, []dyn.Location{
-		{File: "databricks.yml", Line: 10, Column: 1},
-		{File: "databricks.yml", Line: 20, Column: 1},
-	}, diags[0].Locations)
-
-	paths := make([]string, 0)
-	for _, p := range diags[0].Paths {
-		paths = append(paths, p.String())
-	}
-	require.Equal(t, []string{
-		"resources.jobs.test.tasks[0].libraries[0].whl",
-		"resources.jobs.test.tasks[1].libraries[0].whl",
-	}, paths)
-}
-
-func TestSameNameLibrariesWithUniqueLibraries(t *testing.T) {
-	b := &bundle.Bundle{
-		Config: config.Root{
-			Resources: config.Resources{
-				Jobs: map[string]*resources.Job{
-					"test": {
-						JobSettings: &jobs.JobSettings{
-							Tasks: []jobs.Task{
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test-0.1.1.whl",
-										},
-
-										{
-											Whl: "cowsay",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "other/path/test-0.1.0.whl",
-										},
-
-										{
-											Whl: "cowsay",
-										},
-									},
-								},
-								{
-									Libraries: []compute.Library{
-										{
-											Whl: "full/path/test-0.1.1.whl", // Use the same library as the first task
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	diags := bundle.Apply(context.Background(), b, CheckForSameNameLibraries())
-	require.Empty(t, diags)
-}
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
index 2e9211a7e..c6ec04962 100644
--- a/bundle/phases/deploy.go
+++ b/bundle/phases/deploy.go
@@ -155,11 +155,6 @@ func Deploy(outputHandler sync.OutputHandler) bundle.Mutator {
 				mutator.ValidateGitDetails(),
 				artifacts.CleanUp(),
 				libraries.ExpandGlobReferences(),
-				// libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we
-				// know what are the actual library paths.
-				// libraries.ExpandGlobReferences() has to be run after the libraries are built and thus this
-				// mutator is part of the deploy step rather than validate.
-				libraries.CheckForSameNameLibraries(),
 				libraries.Upload(),
 				trampoline.TransformWheelTask(),
 				files.Upload(outputHandler),

From a20894b1f2e5569f2dd73df13f973806d8700f93 Mon Sep 17 00:00:00 2001
From: Pieter Noordhuis <pieter.noordhuis@databricks.com>
Date: Wed, 12 Feb 2025 20:21:48 +0100
Subject: [PATCH 07/11] [Release] Release v0.241.2 (#2346)

This is a bugfix release to address an issue where jobs with tasks with
a
libraries section with PyPI packages could not be deployed.

Bundles:
* Revert changes related to basename check for local libraries
([#2345](https://github.com/databricks/cli/pull/2345)).
---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3df510b7..23c696ab7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # Version changelog
 
+## [Release] Release v0.241.2
+
+This is a bugfix release to address an issue where jobs with tasks with a
+libraries section with PyPI packages could not be deployed.
+
+Bundles:
+ * Revert changes related to basename check for local libraries ([#2345](https://github.com/databricks/cli/pull/2345)).
+
 ## [Release] Release v0.241.1
 
 Bundles:

From fac9bcf1afe6365dfe49c81ae9ea92135d660de9 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 13 Feb 2025 08:26:22 +0100
Subject: [PATCH 08/11] acc: Set X-Databricks-Org-Id on scim/v2/Me endpoint
 (#2349)

This is needed for b.WorkspaceClient().CurrentWorkspaceID(ctx) which is
used by initialize_urls.go mutator ("bundle summary") #2316

It also also needed for to call serverless detection endpoint #2348

Builds on top of #2338
---
 acceptance/server_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/acceptance/server_test.go b/acceptance/server_test.go
index f73872e0b..4fc3108d2 100644
--- a/acceptance/server_test.go
+++ b/acceptance/server_test.go
@@ -63,7 +63,10 @@ func AddHandlers(server *testserver.Server) {
 	})
 
 	server.Handle("GET", "/api/2.0/preview/scim/v2/Me", func(req testserver.Request) any {
-		return testUser
+		return testserver.Response{
+			Headers: map[string][]string{"X-Databricks-Org-Id": {"900800700600"}},
+			Body:    testUser,
+		}
 	})
 
 	server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any {

From 2d0963661136d5901f6acf8fa7c296ff90b9ca8d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 13 Feb 2025 08:31:04 +0100
Subject: [PATCH 09/11] acc: do not show diff for missing output file (#2350)

It's not interesting since it just dumps what is in the repo. This is
especially annoying with bundle/templates tests with a lot of files.
---
 acceptance/acceptance_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 85c345032..d99ad2991 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -402,8 +402,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// The test did not produce an expected output file.
 	if okRef && !okNew {
-		t.Errorf("Missing output file: %s\npathRef: %s\npathNew: %s", relPath, pathRef, pathNew)
-		testdiff.AssertEqualTexts(t, pathRef, pathNew, valueRef, valueNew)
+		t.Errorf("Missing output file: %s", relPath)
 		if testdiff.OverwriteMode {
 			t.Logf("Removing output file: %s", relPath)
 			require.NoError(t, os.Remove(pathRef))

From c0a56a93fb5ad472d9ba88c6e4fd1cf14973ec70 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Fri, 14 Feb 2025 12:02:12 +0100
Subject: [PATCH 10/11] acc: add a helper to diff with replacements (#2352)

## Changes

diff.py is like "diff -r -U2" but it applies replacements first to the
argument.

This allows comparing different output files and directories but ignore
differences that are going to be replaced by placeholders.

This is useful for tests that record large amount of files, specifically
"bundle init" with standard templates. In those tests, changing one
parameter results in a small diff so recording the full directory is not
helpful, because it's hard to see what changed there. I'm using it in
implementation of serverless mode for templates that need it: #2348 The
serverless templates are slightly different from classic, capturing the
diff helps to see exactly where.

Related small changes:
- Add [TESTROOT] replacement for absolute path to acceptance directory
in git repo.
- Add $TESTDIR env var for absolute path to a given test in git repo.

## Tests
- New test acceptance/selftest/diff to test the helper.
- Via #2348 which makes use of this feature.
---
 acceptance/acceptance_test.go                 | 20 +++++++
 acceptance/bin/diff.py                        | 56 +++++++++++++++++++
 acceptance/selftest/diff/out_dir_a/output.txt |  7 +++
 acceptance/selftest/diff/out_dir_b/output.txt |  7 +++
 acceptance/selftest/diff/output.txt           | 13 +++++
 acceptance/selftest/diff/script               | 17 ++++++
 acceptance/selftest/test.toml                 |  1 +
 7 files changed, 121 insertions(+)
 create mode 100755 acceptance/bin/diff.py
 create mode 100644 acceptance/selftest/diff/out_dir_a/output.txt
 create mode 100644 acceptance/selftest/diff/out_dir_b/output.txt
 create mode 100644 acceptance/selftest/diff/output.txt
 create mode 100644 acceptance/selftest/diff/script
 create mode 100644 acceptance/selftest/test.toml

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index d99ad2991..c7b1151ab 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -57,6 +57,8 @@ const (
 	CleanupScript    = "script.cleanup"
 	PrepareScript    = "script.prepare"
 	MaxFileSize      = 100_000
+	// Filename to save replacements to (used by diff.py)
+	ReplsFile = "repls.json"
 )
 
 var Scripts = map[string]bool{
@@ -65,6 +67,10 @@ var Scripts = map[string]bool{
 	PrepareScript:    true,
 }
 
+var Ignored = map[string]bool{
+	ReplsFile: true,
+}
+
 func TestAccept(t *testing.T) {
 	testAccept(t, InprocessMode, SingleTest)
 }
@@ -152,6 +158,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 	testdiff.PrepareReplacementSdkVersion(t, &repls)
 	testdiff.PrepareReplacementsGoVersion(t, &repls)
 
+	repls.SetPath(cwd, "[TESTROOT]")
+
 	repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"})
 
 	testDirs := getTests(t)
@@ -310,6 +318,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 	// User replacements come last:
 	repls.Repls = append(repls.Repls, config.Repls...)
 
+	// Save replacements to temp test directory so that it can be read by diff.py
+	replsJson, err := json.MarshalIndent(repls.Repls, "", "  ")
+	require.NoError(t, err)
+	testutil.WriteFile(t, filepath.Join(tmpDir, ReplsFile), string(replsJson))
+
 	if coverDir != "" {
 		// Creating individual coverage directory for each test, because writing to the same one
 		// results in sporadic failures like this one (only if tests are running in parallel):
@@ -320,6 +333,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	absDir, err := filepath.Abs(dir)
+	require.NoError(t, err)
+	cmd.Env = append(cmd.Env, "TESTDIR="+absDir)
+
 	// Write combined output to a file
 	out, err := os.Create(filepath.Join(tmpDir, "output.txt"))
 	require.NoError(t, err)
@@ -368,6 +385,9 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		if _, ok := outputs[relPath]; ok {
 			continue
 		}
+		if _, ok := Ignored[relPath]; ok {
+			continue
+		}
 		unexpected = append(unexpected, relPath)
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
diff --git a/acceptance/bin/diff.py b/acceptance/bin/diff.py
new file mode 100755
index 000000000..0a91d57ce
--- /dev/null
+++ b/acceptance/bin/diff.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""This script implements "diff -r -U2 dir1 dir2" but applies replacements first"""
+
+import sys
+import difflib
+import json
+import re
+from pathlib import Path
+
+
+def replaceAll(patterns, s):
+    for comp, new in patterns:
+        s = comp.sub(new, s)
+    return s
+
+
+def main():
+    d1, d2 = sys.argv[1:]
+    d1, d2 = Path(d1), Path(d2)
+
+    with open("repls.json") as f:
+        repls = json.load(f)
+
+    patterns = []
+    for r in repls:
+        try:
+            c = re.compile(r["Old"])
+            patterns.append((c, r["New"]))
+        except re.error as e:
+            print(f"Regex error for pattern {r}: {e}", file=sys.stderr)
+
+    files1 = [str(p.relative_to(d1)) for p in d1.rglob("*") if p.is_file()]
+    files2 = [str(p.relative_to(d2)) for p in d2.rglob("*") if p.is_file()]
+
+    set1 = set(files1)
+    set2 = set(files2)
+
+    for f in sorted(set1 | set2):
+        p1 = d1 / f
+        p2 = d2 / f
+        if f not in set2:
+            print(f"Only in {d1}: {f}")
+        elif f not in set1:
+            print(f"Only in {d2}: {f}")
+        else:
+            a = [replaceAll(patterns, x) for x in p1.read_text().splitlines(True)]
+            b = [replaceAll(patterns, x) for x in p2.read_text().splitlines(True)]
+            if a != b:
+                p1_str = p1.as_posix()
+                p2_str = p2.as_posix()
+                for line in difflib.unified_diff(a, b, p1_str, p2_str, "", "", 2):
+                    print(line, end="")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/acceptance/selftest/diff/out_dir_a/output.txt b/acceptance/selftest/diff/out_dir_a/output.txt
new file mode 100644
index 000000000..303c1867b
--- /dev/null
+++ b/acceptance/selftest/diff/out_dir_a/output.txt
@@ -0,0 +1,7 @@
+Hello!
+{
+    "id": "[USERID]",
+    "userName": "[USERNAME]"
+}
+
+Footer
\ No newline at end of file
diff --git a/acceptance/selftest/diff/out_dir_b/output.txt b/acceptance/selftest/diff/out_dir_b/output.txt
new file mode 100644
index 000000000..f4f01af13
--- /dev/null
+++ b/acceptance/selftest/diff/out_dir_b/output.txt
@@ -0,0 +1,7 @@
+Hello!
+{
+    "id": "[UUID]",
+    "userName": "[USERNAME]"
+}
+
+Footer
\ No newline at end of file
diff --git a/acceptance/selftest/diff/output.txt b/acceptance/selftest/diff/output.txt
new file mode 100644
index 000000000..aef99f1e3
--- /dev/null
+++ b/acceptance/selftest/diff/output.txt
@@ -0,0 +1,13 @@
+
+>>> diff.py out_dir_a out_dir_b
+Only in out_dir_a: only_in_a
+Only in out_dir_b: only_in_b
+--- out_dir_a/output.txt
++++ out_dir_b/output.txt
+@@ -1,5 +1,5 @@
+ Hello!
+ {
+-    "id": "[USERID]",
++    "id": "[UUID]",
+     "userName": "[USERNAME]"
+ }
diff --git a/acceptance/selftest/diff/script b/acceptance/selftest/diff/script
new file mode 100644
index 000000000..a7b8706e6
--- /dev/null
+++ b/acceptance/selftest/diff/script
@@ -0,0 +1,17 @@
+mkdir out_dir_a
+mkdir out_dir_b
+
+touch out_dir_a/only_in_a
+touch out_dir_b/only_in_b
+
+echo Hello! >> out_dir_a/output.txt
+echo Hello! >> out_dir_b/output.txt
+
+curl -s $DATABRICKS_HOST/api/2.0/preview/scim/v2/Me >> out_dir_a/output.txt
+printf "\n\nFooter" >> out_dir_a/output.txt
+printf '{\n    "id": "7d639bad-ac6d-4e6f-abd7-9522a86b0239",\n    "userName": "[USERNAME]"\n}\n\nFooter' >> out_dir_b/output.txt
+
+# Unlike regular diff, diff.py will apply replacements first before doing the comparison
+errcode trace diff.py out_dir_a out_dir_b
+
+rm out_dir_a/only_in_a out_dir_b/only_in_b
diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml
new file mode 100644
index 000000000..b76e712fb
--- /dev/null
+++ b/acceptance/selftest/test.toml
@@ -0,0 +1 @@
+LocalOnly = true

From bc30d440972efe1b280d54d3afe5d7593115c978 Mon Sep 17 00:00:00 2001
From: "Lennart Kats (databricks)" <lennart.kats@databricks.com>
Date: Mon, 17 Feb 2025 13:38:03 +0100
Subject: [PATCH 11/11] Provide instructions for testing in the default-python
 template (#2355)

## Changes
Adds instructions for testing to the default-python template.

## Tests
- Unit & acceptance tests.
---
 .../default-python/output/my_default_python/README.md  | 10 ++++++----
 .../template/{{.project_name}}/README.md.tmpl          | 10 ++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
index 97d7d7949..10f570bf4 100644
--- a/acceptance/bundle/templates/default-python/output/my_default_python/README.md
+++ b/acceptance/bundle/templates/default-python/output/my_default_python/README.md
@@ -37,10 +37,12 @@ The 'my_default_python' project was generated by using the default-python templa
    ```
    $ databricks bundle run
    ```
-
-6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
-   https://docs.databricks.com/dev-tools/vscode-ext.html. Or read the "getting started" documentation for
-   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
 
 7. For documentation on the Databricks asset bundles format used
    for this project, and for CI/CD configuration, see
diff --git a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
index 53847a9c9..b8811fa3e 100644
--- a/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
+++ b/libs/template/templates/default-python/template/{{.project_name}}/README.md.tmpl
@@ -38,10 +38,16 @@ The '{{.project_name}}' project was generated by using the default-python templa
    $ databricks bundle run
    ```
 
+{{- if (eq .include_python "no") }}
 6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
    https://docs.databricks.com/dev-tools/vscode-ext.html.
-{{- if (eq .include_python "yes") }} Or read the "getting started" documentation for
-   **Databricks Connect** for instructions on running the included Python code from a different IDE.
+{{- else }}
+6. Optionally, install the Databricks extension for Visual Studio code for local development from
+   https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your
+   virtual environment and setup Databricks Connect for running unit tests locally.
+   When not using these tools, consult your development environment's documentation
+   and/or the documentation for Databricks Connect for manually setting up your environment
+   (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html).
 {{- end}}
 
 7. For documentation on the Databricks asset bundles format used