Make resource and artifact paths in bundle config relative to config folder (#708)

# Warning: breaking change

## Changes
Instead of having paths in bundle config files be relative to bundle
root even if the config file is nested, this PR makes such paths
relative to the folder where the config is located.

When bundle is initialised, these paths will be transformed to relative
paths based on bundle root. For example,
we have file structure like this
```
- mybundle
| - bundle.yml
| - subfolder
| -- resource.yml
| -- my.whl
```

Previously, we had to reference `my.whl` in resource.yml like this,
which was confusing because resource.yml is in the same subfolder
```
sync:
  include:
    - ./subfolder/*.whl
...
tasks:
  - task_key: name
    libraries:
      - whl: ./subfolder/my.whl
...
```

After the change we can reference it like this (which is in line with
the current behaviour for notebooks)

```
sync:
  include:
    - ./*.whl
...
tasks:
  - task_key: name
    libraries:
      - whl: ./my.whl
...
```

## Tests
Existing `translate_path_tests` successfully passed after refactoring.

Added a couple of uses cases for `Libraries` paths.

Added a bundle config tests with include config and sync section

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

bundle/config/artifact.go

@@ -8,9 +8,18 @@ import (
 	"path"
 	"strings"
 
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/databricks-sdk-go/service/compute"
 )
 
+type Artifacts map[string]*Artifact
+
+func (artifacts Artifacts) SetConfigFilePath(path string) {
+	for _, artifact := range artifacts {
+		artifact.ConfigFilePath = path
+	}
+}
+
 type ArtifactType string
 
 const ArtifactPythonWheel ArtifactType = `whl`
@@ -34,6 +43,8 @@ type Artifact struct {
 	// (Python wheel, Java jar and etc) itself
 	Files        []ArtifactFile `json:"files"`
 	BuildCommand string         `json:"build"`
+
+	paths.Paths
 }
 
 func (a *Artifact) Build(ctx context.Context) ([]byte, error) {

bundle/config/mutator/trampoline_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@@ -64,7 +65,7 @@ func TestGenerateTrampoline(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"test": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: tmpDir,
 						},
 						JobSettings: &jobs.JobSettings{

bundle/config/mutator/translate_paths.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -11,8 +12,6 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/notebook"
-	"github.com/databricks/databricks-sdk-go/service/jobs"
-	"github.com/databricks/databricks-sdk-go/service/pipelines"
 )
 
 type ErrIsNotebook struct {
@@ -44,7 +43,9 @@ func (m *translatePaths) Name() string {
 	return "TranslatePaths"
 }
 
-// rewritePath converts a given relative path to a stable remote workspace path.
+type rewriteFunc func(literal, localFullPath, localRelPath, remotePath string) (string, error)
+
+// rewritePath converts a given relative path from the loaded config to a new path based on the passed rewriting function
 //
 // It takes these arguments:
 //   - The argument `dir` is the directory relative to which the given relative path is.
@@ -57,13 +58,23 @@ func (m *translatePaths) rewritePath(
 	dir string,
 	b *bundle.Bundle,
 	p *string,
-	fn func(literal, localPath, remotePath string) (string, error),
+	fn rewriteFunc,
 ) error {
 	// We assume absolute paths point to a location in the workspace
 	if path.IsAbs(filepath.ToSlash(*p)) {
 		return nil
 	}
 
+	url, err := url.Parse(*p)
+	if err != nil {
+		return err
+	}
+
+	// If the file path has scheme, it's a full path and we don't need to transform it
+	if url.Scheme != "" {
+		return nil
+	}
+
 	// Local path is relative to the directory the resource was defined in.
 	localPath := filepath.Join(dir, filepath.FromSlash(*p))
 	if interp, ok := m.seen[localPath]; ok {
@@ -72,19 +83,19 @@ func (m *translatePaths) rewritePath(
 	}
 
 	// Remote path must be relative to the bundle root.
-	remotePath, err := filepath.Rel(b.Config.Path, localPath)
+	localRelPath, err := filepath.Rel(b.Config.Path, localPath)
 	if err != nil {
 		return err
 	}
-	if strings.HasPrefix(remotePath, "..") {
+	if strings.HasPrefix(localRelPath, "..") {
 		return fmt.Errorf("path %s is not contained in bundle root path", localPath)
 	}
 
 	// Prefix remote path with its remote root path.
-	remotePath = path.Join(b.Config.Workspace.FilesPath, filepath.ToSlash(remotePath))
+	remotePath := path.Join(b.Config.Workspace.FilesPath, filepath.ToSlash(localRelPath))
 
 	// Convert local path into workspace path via specified function.
-	interp, err := fn(*p, localPath, filepath.ToSlash(remotePath))
+	interp, err := fn(*p, localPath, localRelPath, filepath.ToSlash(remotePath))
 	if err != nil {
 		return err
 	}
@@ -94,81 +105,69 @@ func (m *translatePaths) rewritePath(
 	return nil
 }
 
-func (m *translatePaths) translateNotebookPath(literal, localPath, remotePath string) (string, error) {
-	nb, _, err := notebook.Detect(localPath)
+func translateNotebookPath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+	nb, _, err := notebook.Detect(localFullPath)
 	if os.IsNotExist(err) {
 		return "", fmt.Errorf("notebook %s not found", literal)
 	}
 	if err != nil {
-		return "", fmt.Errorf("unable to determine if %s is a notebook: %w", localPath, err)
+		return "", fmt.Errorf("unable to determine if %s is a notebook: %w", localFullPath, err)
 	}
 	if !nb {
-		return "", ErrIsNotNotebook{localPath}
+		return "", ErrIsNotNotebook{localFullPath}
 	}
 
 	// Upon import, notebooks are stripped of their extension.
-	return strings.TrimSuffix(remotePath, filepath.Ext(localPath)), nil
+	return strings.TrimSuffix(remotePath, filepath.Ext(localFullPath)), nil
 }
 
-func (m *translatePaths) translateFilePath(literal, localPath, remotePath string) (string, error) {
-	nb, _, err := notebook.Detect(localPath)
+func translateFilePath(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+	nb, _, err := notebook.Detect(localFullPath)
 	if os.IsNotExist(err) {
 		return "", fmt.Errorf("file %s not found", literal)
 	}
 	if err != nil {
-		return "", fmt.Errorf("unable to determine if %s is not a notebook: %w", localPath, err)
+		return "", fmt.Errorf("unable to determine if %s is not a notebook: %w", localFullPath, err)
 	}
 	if nb {
-		return "", ErrIsNotebook{localPath}
+		return "", ErrIsNotebook{localFullPath}
 	}
 	return remotePath, nil
 }
 
-func (m *translatePaths) translateJobTask(dir string, b *bundle.Bundle, task *jobs.Task) error {
-	var err error
-
-	if task.NotebookTask != nil {
-		err = m.rewritePath(dir, b, &task.NotebookTask.NotebookPath, m.translateNotebookPath)
-		if target := (&ErrIsNotNotebook{}); errors.As(err, target) {
-			return fmt.Errorf(`expected a notebook for "tasks.notebook_task.notebook_path" but got a file: %w`, target)
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	if task.SparkPythonTask != nil {
-		err = m.rewritePath(dir, b, &task.SparkPythonTask.PythonFile, m.translateFilePath)
-		if target := (&ErrIsNotebook{}); errors.As(err, target) {
-			return fmt.Errorf(`expected a file for "tasks.spark_python_task.python_file" but got a notebook: %w`, target)
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+func translateNoOp(literal, localFullPath, localRelPath, remotePath string) (string, error) {
+	return localRelPath, nil
 }
 
-func (m *translatePaths) translatePipelineLibrary(dir string, b *bundle.Bundle, library *pipelines.PipelineLibrary) error {
-	var err error
+type transformer struct {
+	// A directory path relative to which `path` will be transformed
+	dir string
+	// A path to transform
+	path *string
+	// Name of the config property where the path string is coming from
+	configPath string
+	// A function that performs the actual rewriting logic.
+	fn rewriteFunc
+}
 
-	if library.Notebook != nil {
-		err = m.rewritePath(dir, b, &library.Notebook.Path, m.translateNotebookPath)
-		if target := (&ErrIsNotNotebook{}); errors.As(err, target) {
-			return fmt.Errorf(`expected a notebook for "libraries.notebook.path" but got a file: %w`, target)
+type transformFunc func(resource any, dir string) *transformer
+
+// Apply all matches transformers for the given resource
+func (m *translatePaths) applyTransformers(funcs []transformFunc, b *bundle.Bundle, resource any, dir string) error {
+	for _, transformFn := range funcs {
+		transformer := transformFn(resource, dir)
+		if transformer == nil {
+			continue
 		}
+
+		err := m.rewritePath(transformer.dir, b, transformer.path, transformer.fn)
 		if err != nil {
-			return err
-		}
-	}
-
-	if library.File != nil {
-		err = m.rewritePath(dir, b, &library.File.Path, m.translateFilePath)
 			if target := (&ErrIsNotebook{}); errors.As(err, target) {
-			return fmt.Errorf(`expected a file for "libraries.file.path" but got a notebook: %w`, target)
+				return fmt.Errorf(`expected a file for "%s" but got a notebook: %w`, transformer.configPath, target)
+			}
+			if target := (&ErrIsNotNotebook{}); errors.As(err, target) {
+				return fmt.Errorf(`expected a notebook for "%s" but got a file: %w`, transformer.configPath, target)
 			}
-		if err != nil {
 			return err
 		}
 	}
@@ -179,38 +178,16 @@ func (m *translatePaths) translatePipelineLibrary(dir string, b *bundle.Bundle,
 func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) error {
 	m.seen = make(map[string]string)
 
-	for key, job := range b.Config.Resources.Jobs {
-		dir, err := job.ConfigFileDirectory()
-		if err != nil {
-			return fmt.Errorf("unable to determine directory for job %s: %w", key, err)
-		}
-
-		// Do not translate job task paths if using git source
-		if job.GitSource != nil {
-			continue
-		}
-
-		for i := 0; i < len(job.Tasks); i++ {
-			err := m.translateJobTask(dir, b, &job.Tasks[i])
+	for _, fn := range []func(*translatePaths, *bundle.Bundle) error{
+		applyJobTransformers,
+		applyPipelineTransformers,
+		applyArtifactTransformers,
+	} {
+		err := fn(m, b)
 		if err != nil {
 			return err
 		}
 	}
-	}
-
-	for key, pipeline := range b.Config.Resources.Pipelines {
-		dir, err := pipeline.ConfigFileDirectory()
-		if err != nil {
-			return fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err)
-		}
-
-		for i := 0; i < len(pipeline.Libraries); i++ {
-			err := m.translatePipelineLibrary(dir, b, &pipeline.Libraries[i])
-			if err != nil {
-				return err
-			}
-		}
-	}
 
 	return nil
 }

bundle/config/mutator/translate_paths_artifacts.go

@@ -0,0 +1,42 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config"
+)
+
+func transformArtifactPath(resource any, dir string) *transformer {
+	artifact, ok := resource.(*config.Artifact)
+	if !ok {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&artifact.Path,
+		"artifacts.path",
+		translateNoOp,
+	}
+}
+
+func applyArtifactTransformers(m *translatePaths, b *bundle.Bundle) error {
+	artifactTransformers := []transformFunc{
+		transformArtifactPath,
+	}
+
+	for key, artifact := range b.Config.Artifacts {
+		dir, err := artifact.ConfigFileDirectory()
+		if err != nil {
+			return fmt.Errorf("unable to determine directory for artifact %s: %w", key, err)
+		}
+
+		err = m.applyTransformers(artifactTransformers, b, artifact, dir)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

bundle/config/mutator/translate_paths_jobs.go

@@ -0,0 +1,103 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/databricks-sdk-go/service/compute"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+)
+
+func transformNotebookTask(resource any, dir string) *transformer {
+	task, ok := resource.(*jobs.Task)
+	if !ok || task.NotebookTask == nil {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&task.NotebookTask.NotebookPath,
+		"tasks.notebook_task.notebook_path",
+		translateNotebookPath,
+	}
+}
+
+func transformSparkTask(resource any, dir string) *transformer {
+	task, ok := resource.(*jobs.Task)
+	if !ok || task.SparkPythonTask == nil {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&task.SparkPythonTask.PythonFile,
+		"tasks.spark_python_task.python_file",
+		translateFilePath,
+	}
+}
+
+func transformWhlLibrary(resource any, dir string) *transformer {
+	library, ok := resource.(*compute.Library)
+	if !ok || library.Whl == "" {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&library.Whl,
+		"libraries.whl",
+		translateNoOp,
+	}
+}
+
+func transformJarLibrary(resource any, dir string) *transformer {
+	library, ok := resource.(*compute.Library)
+	if !ok || library.Jar == "" {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&library.Jar,
+		"libraries.jar",
+		translateFilePath,
+	}
+}
+
+func applyJobTransformers(m *translatePaths, b *bundle.Bundle) error {
+	jobTransformers := []transformFunc{
+		transformNotebookTask,
+		transformSparkTask,
+		transformWhlLibrary,
+		transformJarLibrary,
+	}
+
+	for key, job := range b.Config.Resources.Jobs {
+		dir, err := job.ConfigFileDirectory()
+		if err != nil {
+			return fmt.Errorf("unable to determine directory for job %s: %w", key, err)
+		}
+
+		// Do not translate job task paths if using git source
+		if job.GitSource != nil {
+			continue
+		}
+
+		for i := 0; i < len(job.Tasks); i++ {
+			task := &job.Tasks[i]
+			err := m.applyTransformers(jobTransformers, b, task, dir)
+			if err != nil {
+				return err
+			}
+			for j := 0; j < len(task.Libraries); j++ {
+				library := &task.Libraries[j]
+				err := m.applyTransformers(jobTransformers, b, library, dir)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}

bundle/config/mutator/translate_paths_pipelines.go

@@ -0,0 +1,60 @@
+package mutator
+
+import (
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+)
+
+func transformLibraryNotebook(resource any, dir string) *transformer {
+	library, ok := resource.(*pipelines.PipelineLibrary)
+	if !ok || library.Notebook == nil {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&library.Notebook.Path,
+		"libraries.notebook.path",
+		translateNotebookPath,
+	}
+}
+
+func transformLibraryFile(resource any, dir string) *transformer {
+	library, ok := resource.(*pipelines.PipelineLibrary)
+	if !ok || library.File == nil {
+		return nil
+	}
+
+	return &transformer{
+		dir,
+		&library.File.Path,
+		"libraries.file.path",
+		translateFilePath,
+	}
+}
+
+func applyPipelineTransformers(m *translatePaths, b *bundle.Bundle) error {
+	pipelineTransformers := []transformFunc{
+		transformLibraryNotebook,
+		transformLibraryFile,
+	}
+
+	for key, pipeline := range b.Config.Resources.Pipelines {
+		dir, err := pipeline.ConfigFileDirectory()
+		if err != nil {
+			return fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err)
+		}
+
+		for i := 0; i < len(pipeline.Libraries); i++ {
+			library := &pipeline.Libraries[i]
+			err := m.applyTransformers(pipelineTransformers, b, library, dir)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

bundle/config/mutator/translate_paths_test.go

@@ -9,7 +9,9 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
 	"github.com/stretchr/testify/assert"
@@ -43,7 +45,7 @@ func TestTranslatePathsSkippedWithGitSource(t *testing.T) {
 				Jobs: map[string]*resources.Job{
 					"job": {
 
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -103,6 +105,7 @@ func TestTranslatePaths(t *testing.T) {
 	touchNotebookFile(t, filepath.Join(dir, "my_job_notebook.py"))
 	touchNotebookFile(t, filepath.Join(dir, "my_pipeline_notebook.py"))
 	touchEmptyFile(t, filepath.Join(dir, "my_python_file.py"))
+	touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar"))
 
 	bundle := &bundle.Bundle{
 		Config: config.Root{
@@ -113,7 +116,7 @@ func TestTranslatePaths(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -122,6 +125,9 @@ func TestTranslatePaths(t *testing.T) {
 									NotebookTask: &jobs.NotebookTask{
 										NotebookPath: "./my_job_notebook.py",
 									},
+									Libraries: []compute.Library{
+										{Whl: "./dist/task.whl"},
+									},
 								},
 								{
 									NotebookTask: &jobs.NotebookTask{
@@ -143,13 +149,29 @@ func TestTranslatePaths(t *testing.T) {
 										PythonFile: "./my_python_file.py",
 									},
 								},
+								{
+									SparkJarTask: &jobs.SparkJarTask{
+										MainClassName: "HelloWorld",
+									},
+									Libraries: []compute.Library{
+										{Jar: "./dist/task.jar"},
+									},
+								},
+								{
+									SparkJarTask: &jobs.SparkJarTask{
+										MainClassName: "HelloWorldRemote",
+									},
+									Libraries: []compute.Library{
+										{Jar: "dbfs:///bundle/dist/task_remote.jar"},
+									},
+								},
 							},
 						},
 					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						PipelineSpec: &pipelines.PipelineSpec{
@@ -194,6 +216,11 @@ func TestTranslatePaths(t *testing.T) {
 		"/bundle/my_job_notebook",
 		bundle.Config.Resources.Jobs["job"].Tasks[0].NotebookTask.NotebookPath,
 	)
+	assert.Equal(
+		t,
+		filepath.Join("dist", "task.whl"),
+		bundle.Config.Resources.Jobs["job"].Tasks[0].Libraries[0].Whl,
+	)
 	assert.Equal(
 		t,
 		"/Users/jane.doe@databricks.com/doesnt_exist.py",
@@ -209,6 +236,16 @@ func TestTranslatePaths(t *testing.T) {
 		"/bundle/my_python_file.py",
 		bundle.Config.Resources.Jobs["job"].Tasks[4].SparkPythonTask.PythonFile,
 	)
+	assert.Equal(
+		t,
+		"/bundle/dist/task.jar",
+		bundle.Config.Resources.Jobs["job"].Tasks[5].Libraries[0].Jar,
+	)
+	assert.Equal(
+		t,
+		"dbfs:///bundle/dist/task_remote.jar",
+		bundle.Config.Resources.Jobs["job"].Tasks[6].Libraries[0].Jar,
+	)
 
 	// Assert that the path in the libraries now refer to the artifact.
 	assert.Equal(
@@ -236,6 +273,7 @@ func TestTranslatePaths(t *testing.T) {
 func TestTranslatePathsInSubdirectories(t *testing.T) {
 	dir := t.TempDir()
 	touchEmptyFile(t, filepath.Join(dir, "job", "my_python_file.py"))
+	touchEmptyFile(t, filepath.Join(dir, "job", "dist", "task.jar"))
 	touchEmptyFile(t, filepath.Join(dir, "pipeline", "my_python_file.py"))
 
 	bundle := &bundle.Bundle{
@@ -247,7 +285,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "job/resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -257,13 +295,21 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 										PythonFile: "./my_python_file.py",
 									},
 								},
+								{
+									SparkJarTask: &jobs.SparkJarTask{
+										MainClassName: "HelloWorld",
+									},
+									Libraries: []compute.Library{
+										{Jar: "./dist/task.jar"},
+									},
+								},
 							},
 						},
 					},
 				},
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "pipeline/resource.yml"),
 						},
 
@@ -290,6 +336,11 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
 		"/bundle/job/my_python_file.py",
 		bundle.Config.Resources.Jobs["job"].Tasks[0].SparkPythonTask.PythonFile,
 	)
+	assert.Equal(
+		t,
+		"/bundle/job/dist/task.jar",
+		bundle.Config.Resources.Jobs["job"].Tasks[1].Libraries[0].Jar,
+	)
 
 	assert.Equal(
 		t,
@@ -310,7 +361,7 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "../resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -341,7 +392,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "fake.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -372,7 +423,7 @@ func TestJobFileDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "fake.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -403,7 +454,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "fake.yml"),
 						},
 						PipelineSpec: &pipelines.PipelineSpec{
@@ -434,7 +485,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "fake.yml"),
 						},
 						PipelineSpec: &pipelines.PipelineSpec{
@@ -469,7 +520,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -504,7 +555,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"job": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						JobSettings: &jobs.JobSettings{
@@ -539,7 +590,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						PipelineSpec: &pipelines.PipelineSpec{
@@ -574,7 +625,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) {
 			Resources: config.Resources{
 				Pipelines: map[string]*resources.Pipeline{
 					"pipeline": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: filepath.Join(dir, "resource.yml"),
 						},
 						PipelineSpec: &pipelines.PipelineSpec{

bundle/config/paths/paths.go

@@ -1,4 +1,4 @@
-package resources
+package paths
 
 import (
 	"fmt"

bundle/config/resources/job.go

@@ -1,6 +1,7 @@
 package resources
 
 import (
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/imdario/mergo"
 )
@@ -9,7 +10,7 @@ type Job struct {
 	ID          string       `json:"id,omitempty" bundle:"readonly"`
 	Permissions []Permission `json:"permissions,omitempty"`
 
-	Paths
+	paths.Paths
 
 	*jobs.JobSettings
 }

bundle/config/resources/mlflow_experiment.go

@@ -1,11 +1,14 @@
 package resources
 
-import "github.com/databricks/databricks-sdk-go/service/ml"
+import (
+	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/databricks-sdk-go/service/ml"
+)
 
 type MlflowExperiment struct {
 	Permissions []Permission `json:"permissions,omitempty"`
 
-	Paths
+	paths.Paths
 
 	*ml.Experiment
 }

bundle/config/resources/mlflow_model.go

@@ -1,11 +1,14 @@
 package resources
 
-import "github.com/databricks/databricks-sdk-go/service/ml"
+import (
+	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/databricks-sdk-go/service/ml"
+)
 
 type MlflowModel struct {
 	Permissions []Permission `json:"permissions,omitempty"`
 
-	Paths
+	paths.Paths
 
 	*ml.Model
 }

bundle/config/resources/pipeline.go

@@ -1,12 +1,15 @@
 package resources
 
-import "github.com/databricks/databricks-sdk-go/service/pipelines"
+import (
+	"github.com/databricks/cli/bundle/config/paths"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
+)
 
 type Pipeline struct {
 	ID          string       `json:"id,omitempty" bundle:"readonly"`
 	Permissions []Permission `json:"permissions,omitempty"`
 
-	Paths
+	paths.Paths
 
 	*pipelines.PipelineSpec
 }

bundle/config/resources_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"testing"
 
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/stretchr/testify/assert"
 )
@@ -11,21 +12,21 @@ func TestVerifyUniqueResourceIdentifiers(t *testing.T) {
 	r := Resources{
 		Jobs: map[string]*resources.Job{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo.yml",
 				},
 			},
 		},
 		Models: map[string]*resources.MlflowModel{
 			"bar": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "bar.yml",
 				},
 			},
 		},
 		Experiments: map[string]*resources.MlflowExperiment{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo2.yml",
 				},
 			},
@@ -39,14 +40,14 @@ func TestVerifySafeMerge(t *testing.T) {
 	r := Resources{
 		Jobs: map[string]*resources.Job{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo.yml",
 				},
 			},
 		},
 		Models: map[string]*resources.MlflowModel{
 			"bar": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "bar.yml",
 				},
 			},
@@ -55,7 +56,7 @@ func TestVerifySafeMerge(t *testing.T) {
 	other := Resources{
 		Pipelines: map[string]*resources.Pipeline{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo2.yml",
 				},
 			},
@@ -69,14 +70,14 @@ func TestVerifySafeMergeForSameResourceType(t *testing.T) {
 	r := Resources{
 		Jobs: map[string]*resources.Job{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo.yml",
 				},
 			},
 		},
 		Models: map[string]*resources.MlflowModel{
 			"bar": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "bar.yml",
 				},
 			},
@@ -85,7 +86,7 @@ func TestVerifySafeMergeForSameResourceType(t *testing.T) {
 	other := Resources{
 		Jobs: map[string]*resources.Job{
 			"foo": {
-				Paths: resources.Paths{
+				Paths: paths.Paths{
 					ConfigFilePath: "foo2.yml",
 				},
 			},

bundle/config/root.go

@@ -64,7 +64,7 @@ type Root struct {
 	Workspace Workspace `json:"workspace,omitempty"`
 
 	// Artifacts contains a description of all code artifacts in this bundle.
-	Artifacts map[string]*Artifact `json:"artifacts,omitempty"`
+	Artifacts Artifacts `json:"artifacts,omitempty"`
 
 	// Resources contains a description of all Databricks resources
 	// to deploy in this bundle (e.g. jobs, pipelines, etc.).
@@ -113,6 +113,10 @@ func Load(path string) (*Root, error) {
 // was loaded from in configuration leafs that require it.
 func (r *Root) SetConfigFilePath(path string) {
 	r.Resources.SetConfigFilePath(path)
+	if r.Artifacts != nil {
+		r.Artifacts.SetConfigFilePath(path)
+	}
+
 	if r.Targets != nil {
 		for _, env := range r.Targets {
 			if env == nil {
@@ -121,6 +125,9 @@ func (r *Root) SetConfigFilePath(path string) {
 			if env.Resources != nil {
 				env.Resources.SetConfigFilePath(path)
 			}
+			if env.Artifacts != nil {
+				env.Artifacts.SetConfigFilePath(path)
+			}
 		}
 	}
 }
@@ -175,11 +182,17 @@ func (r *Root) Load(path string) error {
 }
 
 func (r *Root) Merge(other *Root) error {
+	err := r.Sync.Merge(r, other)
+	if err != nil {
+		return err
+	}
+	other.Sync = Sync{}
+
 	// TODO: when hooking into merge semantics, disallow setting path on the target instance.
 	other.Path = ""
 
 	// Check for safe merge, protecting against duplicate resource identifiers
-	err := r.Resources.VerifySafeMerge(&other.Resources)
+	err = r.Resources.VerifySafeMerge(&other.Resources)
 	if err != nil {
 		return err
 	}

bundle/config/sync.go

@@ -1,5 +1,7 @@
 package config
 
+import "path/filepath"
+
 type Sync struct {
 	// Include contains a list of globs evaluated relative to the bundle root path
 	// to explicitly include files that were excluded by the user's gitignore.
@@ -11,3 +13,19 @@ type Sync struct {
 	// 2) the `Include` field above.
 	Exclude []string `json:"exclude,omitempty"`
 }
+
+func (s *Sync) Merge(root *Root, other *Root) error {
+	path, err := filepath.Rel(root.Path, other.Path)
+	if err != nil {
+		return err
+	}
+	for _, include := range other.Sync.Include {
+		s.Include = append(s.Include, filepath.Join(path, include))
+	}
+
+	for _, exclude := range other.Sync.Exclude {
+		s.Exclude = append(s.Exclude, filepath.Join(path, exclude))
+	}
+
+	return nil
+}

bundle/config/target.go

@@ -23,7 +23,7 @@ type Target struct {
 
 	Workspace *Workspace `json:"workspace,omitempty"`
 
-	Artifacts map[string]*Artifact `json:"artifacts,omitempty"`
+	Artifacts Artifacts `json:"artifacts,omitempty"`
 
 	Resources *Resources `json:"resources,omitempty"`
 

bundle/python/transform_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
+	"github.com/databricks/cli/bundle/config/paths"
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/stretchr/testify/require"
@@ -112,7 +113,7 @@ func TestNoPanicWithNoPythonWheelTasks(t *testing.T) {
 			Resources: config.Resources{
 				Jobs: map[string]*resources.Job{
 					"test": {
-						Paths: resources.Paths{
+						Paths: paths.Paths{
 							ConfigFilePath: tmpDir,
 						},
 						JobSettings: &jobs.JobSettings{

bundle/tests/relative_path_with_includes/bundle.yml

@@ -0,0 +1,25 @@
+bundle:
+  name: sync_include
+
+include:
+  - "*/*.yml"
+
+sync:
+  include:
+    - ./folder_a/*.*
+  exclude:
+    - ./folder_b/*.*
+
+artifacts:
+  test_a:
+    type: whl
+    path: ./artifact_a
+
+resources:
+  jobs:
+    job_a:
+      name: "job_a"
+      tasks:
+        - task_key: "task_a"
+          libraries:
+            - whl: ./dist/job_a.whl

bundle/tests/relative_path_with_includes/subfolder/include.yml

@@ -0,0 +1,20 @@
+sync:
+  include:
+    - ./folder_c/*.*
+  exclude:
+    - ./folder_d/*.*
+
+artifacts:
+  test_b:
+    type: whl
+    path: ./artifact_b
+
+
+resources:
+  jobs:
+    job_b:
+      name: "job_b"
+      tasks:
+        - task_key: "task_a"
+          libraries:
+            - whl: ./dist/job_b.whl

bundle/tests/relative_path_with_includes_test.go

@@ -0,0 +1,28 @@
+package config_tests
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/config/mutator"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRelativePathsWithIncludes(t *testing.T) {
+	b := load(t, "./relative_path_with_includes")
+
+	m := mutator.TranslatePaths()
+	err := bundle.Apply(context.Background(), b, m)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "artifact_a", b.Config.Artifacts["test_a"].Path)
+	assert.Equal(t, filepath.Join("subfolder", "artifact_b"), b.Config.Artifacts["test_b"].Path)
+
+	assert.ElementsMatch(t, []string{"./folder_a/*.*", filepath.Join("subfolder", "folder_c", "*.*")}, b.Config.Sync.Include)
+	assert.ElementsMatch(t, []string{"./folder_b/*.*", filepath.Join("subfolder", "folder_d", "*.*")}, b.Config.Sync.Exclude)
+
+	assert.Equal(t, filepath.Join("dist", "job_a.whl"), b.Config.Resources.Jobs["job_a"].Tasks[0].Libraries[0].Whl)
+	assert.Equal(t, filepath.Join("subfolder", "dist", "job_b.whl"), b.Config.Resources.Jobs["job_b"].Tasks[0].Libraries[0].Whl)
+}