diff --git a/bundle/bundle.go b/bundle/bundle.go index 02d0eaac..81fdfd4a 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -45,7 +45,11 @@ type Bundle struct { func Load(path string) (*Bundle, error) { bundle := &Bundle{} - err := bundle.Config.Load(filepath.Join(path, config.FileName)) + configFile, err := config.FileNames.FindInPath(path) + if err != nil { + return nil, err + } + err = bundle.Config.Load(configFile) if err != nil { return nil, err } diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 5a26d350..18550f4f 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -23,7 +23,7 @@ func TestLoadExists(t *testing.T) { func TestBundleCacheDir(t *testing.T) { projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "bundle.yml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) require.NoError(t, err) f1.Close() @@ -47,7 +47,7 @@ func TestBundleCacheDir(t *testing.T) { func TestBundleCacheDirOverride(t *testing.T) { projectDir := t.TempDir() bundleTmpDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "bundle.yml")) + f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) require.NoError(t, err) f1.Close() diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index 454e3a98..f3717ce0 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -27,8 +27,10 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) error var out []bundle.Mutator // Map with files we've already seen to avoid loading them twice. - var seen = map[string]bool{ - config.FileName: true, + var seen = map[string]bool{} + + for _, file := range config.FileNames { + seen[file] = true } // Maintain list of files in order of files being loaded. diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index c7d00d88..9ca5335a 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -61,7 +61,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - touch(t, bundle.Config.Path, "bundle.yml") + touch(t, bundle.Config.Path, "databricks.yml") touch(t, bundle.Config.Path, "a.yml") touch(t, bundle.Config.Path, "b.yml") diff --git a/bundle/config/root.go b/bundle/config/root.go index 5ee337d3..28b1a615 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -11,12 +11,39 @@ import ( "github.com/imdario/mergo" ) -// FileName is the name of bundle configuration file. -const FileName = "bundle.yml" +type ConfigFileNames []string + +// FileNames contains allowed names of bundle configuration files. +var FileNames = ConfigFileNames{"databricks.yml", "databricks.yaml", "bundle.yml", "bundle.yaml"} + +func (c ConfigFileNames) FindInPath(path string) (string, error) { + result := "" + var firstErr error + + for _, file := range c { + filePath := filepath.Join(path, file) + _, err := os.Stat(filePath) + if err == nil { + if result != "" { + return "", fmt.Errorf("multiple bundle root configuration files found in %s", path) + } + result = filePath + } else { + if firstErr == nil { + firstErr = err + } + } + } + + if result == "" { + return "", firstErr + } + return result, nil +} type Root struct { // Path contains the directory path to the root of the bundle. - // It is set when loading `bundle.yml`. + // It is set when loading `databricks.yml`. Path string `json:"-" bundle:"readonly"` // Contains user defined variables @@ -27,7 +54,7 @@ type Root struct { Bundle Bundle `json:"bundle"` // Include specifies a list of patterns of file names to load and - // merge into the this configuration. If not set in `bundle.yml`, + // merge into the this configuration. If not set in `databricks.yml`, // it defaults to loading `*.yml` and `*/*.yml`. // // Also see [mutator.DefineDefaultInclude]. @@ -62,7 +89,10 @@ func Load(path string) (*Root, error) { // If we were given a directory, assume this is the bundle root. if stat.IsDir() { - path = filepath.Join(path, FileName) + path, err = FileNames.FindInPath(path) + if err != nil { + return nil, err + } } if err := r.Load(path); err != nil { diff --git a/bundle/config/root_test.go b/bundle/config/root_test.go index 818e89a2..531ffcec 100644 --- a/bundle/config/root_test.go +++ b/bundle/config/root_test.go @@ -2,7 +2,11 @@ package config import ( "encoding/json" + "os" + "path/filepath" "reflect" + "runtime" + "strings" "testing" "github.com/databricks/cli/bundle/config/variable" @@ -26,7 +30,7 @@ func TestRootMarshalUnmarshal(t *testing.T) { func TestRootLoad(t *testing.T) { root := &Root{} - err := root.Load("../tests/basic/bundle.yml") + err := root.Load("../tests/basic/databricks.yml") require.NoError(t, err) assert.Equal(t, "basic", root.Bundle.Name) } @@ -78,13 +82,13 @@ func TestRootMergeMap(t *testing.T) { func TestDuplicateIdOnLoadReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_names_in_root/bundle.yml") - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/bundle.yml, pipeline at ./testdata/duplicate_resource_names_in_root/bundle.yml)") + err := root.Load("./testdata/duplicate_resource_names_in_root/databricks.yml") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_names_in_root/databricks.yml, pipeline at ./testdata/duplicate_resource_names_in_root/databricks.yml)") } func TestDuplicateIdOnMergeReturnsError(t *testing.T) { root := &Root{} - err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/bundle.yml") + err := root.Load("./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml") require.NoError(t, err) other := &Root{} @@ -92,7 +96,7 @@ func TestDuplicateIdOnMergeReturnsError(t *testing.T) { require.NoError(t, err) err = root.Merge(other) - assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/bundle.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") + assert.ErrorContains(t, err, "multiple resources named foo (job at ./testdata/duplicate_resource_name_in_subconfiguration/databricks.yml, pipeline at ./testdata/duplicate_resource_name_in_subconfiguration/resources.yml)") } func TestInitializeVariables(t *testing.T) { @@ -163,3 +167,62 @@ func TestRootMergeEnvironmentWithMode(t *testing.T) { require.NoError(t, root.MergeEnvironment(env)) assert.Equal(t, Development, root.Bundle.Mode) } + +func TestConfigFileNames_FindInPath(t *testing.T) { + testCases := []struct { + name string + files []string + expected string + err string + }{ + { + name: "file found", + files: []string{"databricks.yml"}, + expected: "BASE/databricks.yml", + err: "", + }, + { + name: "file found", + files: []string{"bundle.yml"}, + expected: "BASE/bundle.yml", + err: "", + }, + { + name: "multiple files found", + files: []string{"databricks.yaml", "bundle.yml"}, + expected: "", + err: "multiple bundle root configuration files found", + }, + { + name: "file not found", + files: []string{}, + expected: "", + err: "no such file or directory", + }, + } + + if runtime.GOOS == "windows" { + testCases[3].err = "The system cannot find the file specified." + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + projectDir := t.TempDir() + for _, file := range tc.files { + f1, _ := os.Create(filepath.Join(projectDir, file)) + f1.Close() + } + + result, err := FileNames.FindInPath(projectDir) + + expected := strings.Replace(tc.expected, "BASE/", projectDir+string(os.PathSeparator), 1) + assert.Equal(t, expected, result) + + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/bundle.yml b/bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_name_in_subconfiguration/bundle.yml rename to bundle/config/testdata/duplicate_resource_name_in_subconfiguration/databricks.yml diff --git a/bundle/config/testdata/duplicate_resource_names_in_root/bundle.yml b/bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml similarity index 100% rename from bundle/config/testdata/duplicate_resource_names_in_root/bundle.yml rename to bundle/config/testdata/duplicate_resource_names_in_root/databricks.yml diff --git a/bundle/root.go b/bundle/root.go index 70d778e1..46f63e13 100644 --- a/bundle/root.go +++ b/bundle/root.go @@ -36,11 +36,15 @@ func getRootWithTraversal() (string, error) { if err != nil { return "", err } - path, err := folders.FindDirWithLeaf(wd, config.FileName) - if err != nil { - return "", fmt.Errorf(`unable to locate bundle root: %s not found`, config.FileName) + + for _, file := range config.FileNames { + path, err := folders.FindDirWithLeaf(wd, file) + if err == nil { + return path, nil + } } - return path, nil + + return "", fmt.Errorf(`unable to locate bundle root: %s not found`, config.FileNames[0]) } // mustGetRoot returns a bundle root or an error if one cannot be found. diff --git a/bundle/root_test.go b/bundle/root_test.go index dab00225..2f830492 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -76,8 +76,8 @@ func TestRootLookup(t *testing.T) { chdir(t, t.TempDir()) - // Create bundle.yml file. - f, err := os.Create(config.FileName) + // Create databricks.yml file. + f, err := os.Create(config.FileNames[0]) require.NoError(t, err) defer f.Close() diff --git a/bundle/tests/autoload_git/bundle.yml b/bundle/tests/autoload_git/databricks.yml similarity index 100% rename from bundle/tests/autoload_git/bundle.yml rename to bundle/tests/autoload_git/databricks.yml diff --git a/bundle/tests/basic/bundle.yml b/bundle/tests/basic/databricks.yml similarity index 100% rename from bundle/tests/basic/bundle.yml rename to bundle/tests/basic/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/no_subconfigurations/bundle.yml b/bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/no_subconfigurations/bundle.yml rename to bundle/tests/conflicting_resource_ids/no_subconfigurations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/one_subconfiguration/bundle.yml b/bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/one_subconfiguration/bundle.yml rename to bundle/tests/conflicting_resource_ids/one_subconfiguration/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids/two_subconfigurations/bundle.yml b/bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml similarity index 100% rename from bundle/tests/conflicting_resource_ids/two_subconfigurations/bundle.yml rename to bundle/tests/conflicting_resource_ids/two_subconfigurations/databricks.yml diff --git a/bundle/tests/conflicting_resource_ids_test.go b/bundle/tests/conflicting_resource_ids_test.go index 12f460fd..b75e3753 100644 --- a/bundle/tests/conflicting_resource_ids_test.go +++ b/bundle/tests/conflicting_resource_ids_test.go @@ -14,7 +14,7 @@ import ( func TestConflictingResourceIdsNoSubconfig(t *testing.T) { _, err := bundle.Load("./conflicting_resource_ids/no_subconfigurations") - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/bundle.yml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/no_subconfigurations/databricks.yml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, bundleConfigPath)) } @@ -22,7 +22,7 @@ func TestConflictingResourceIdsOneSubconfig(t *testing.T) { b, err := bundle.Load("./conflicting_resource_ids/one_subconfiguration") require.NoError(t, err) err = bundle.Apply(context.Background(), b, bundle.Seq(mutator.DefaultMutators()...)) - bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/bundle.yml") + bundleConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/databricks.yml") resourcesConfigPath := filepath.FromSlash("conflicting_resource_ids/one_subconfiguration/resources.yml") assert.ErrorContains(t, err, fmt.Sprintf("multiple resources named foo (job at %s, pipeline at %s)", bundleConfigPath, resourcesConfigPath)) } diff --git a/bundle/tests/environment_empty/bundle.yml b/bundle/tests/environment_empty/databricks.yml similarity index 100% rename from bundle/tests/environment_empty/bundle.yml rename to bundle/tests/environment_empty/databricks.yml diff --git a/bundle/tests/environment_overrides/bundle.yml b/bundle/tests/environment_overrides/databricks.yml similarity index 100% rename from bundle/tests/environment_overrides/bundle.yml rename to bundle/tests/environment_overrides/databricks.yml diff --git a/bundle/tests/include_default/bundle.yml b/bundle/tests/include_default/databricks.yml similarity index 100% rename from bundle/tests/include_default/bundle.yml rename to bundle/tests/include_default/databricks.yml diff --git a/bundle/tests/include_invalid/bundle.yml b/bundle/tests/include_invalid/databricks.yml similarity index 100% rename from bundle/tests/include_invalid/bundle.yml rename to bundle/tests/include_invalid/databricks.yml diff --git a/bundle/tests/include_override/bundle.yml b/bundle/tests/include_override/databricks.yml similarity index 100% rename from bundle/tests/include_override/bundle.yml rename to bundle/tests/include_override/databricks.yml diff --git a/bundle/tests/include_with_glob/bundle.yml b/bundle/tests/include_with_glob/databricks.yml similarity index 100% rename from bundle/tests/include_with_glob/bundle.yml rename to bundle/tests/include_with_glob/databricks.yml diff --git a/bundle/tests/interpolation/bundle.yml b/bundle/tests/interpolation/databricks.yml similarity index 100% rename from bundle/tests/interpolation/bundle.yml rename to bundle/tests/interpolation/databricks.yml diff --git a/bundle/tests/job_and_pipeline/bundle.yml b/bundle/tests/job_and_pipeline/databricks.yml similarity index 100% rename from bundle/tests/job_and_pipeline/bundle.yml rename to bundle/tests/job_and_pipeline/databricks.yml diff --git a/bundle/tests/job_and_pipeline_test.go b/bundle/tests/job_and_pipeline_test.go index 775f415c..d92eabd3 100644 --- a/bundle/tests/job_and_pipeline_test.go +++ b/bundle/tests/job_and_pipeline_test.go @@ -15,7 +15,7 @@ func TestJobAndPipelineDevelopment(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.Equal(t, b.Config.Bundle.Mode, config.Development) assert.True(t, p.Development) require.Len(t, p.Libraries, 1) @@ -29,7 +29,7 @@ func TestJobAndPipelineStaging(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) @@ -42,14 +42,14 @@ func TestJobAndPipelineProduction(t *testing.T) { assert.Len(t, b.Config.Resources.Pipelines, 1) p := b.Config.Resources.Pipelines["nyc_taxi_pipeline"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(p.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(p.ConfigFilePath)) assert.False(t, p.Development) require.Len(t, p.Libraries, 1) assert.Equal(t, "./dlt/nyc_taxi_loader", p.Libraries[0].Notebook.Path) assert.Equal(t, "nyc_taxi_production", p.Target) j := b.Config.Resources.Jobs["pipeline_schedule"] - assert.Equal(t, "job_and_pipeline/bundle.yml", filepath.ToSlash(j.ConfigFilePath)) + assert.Equal(t, "job_and_pipeline/databricks.yml", filepath.ToSlash(j.ConfigFilePath)) assert.Equal(t, "Daily refresh of production pipeline", j.Name) require.Len(t, j.Tasks, 1) assert.NotEmpty(t, j.Tasks[0].PipelineTask.PipelineId) diff --git a/bundle/tests/variables/env_overrides/bundle.yml b/bundle/tests/variables/env_overrides/databricks.yml similarity index 100% rename from bundle/tests/variables/env_overrides/bundle.yml rename to bundle/tests/variables/env_overrides/databricks.yml diff --git a/bundle/tests/variables/vanilla/bundle.yml b/bundle/tests/variables/vanilla/databricks.yml similarity index 100% rename from bundle/tests/variables/vanilla/bundle.yml rename to bundle/tests/variables/vanilla/databricks.yml diff --git a/bundle/tests/yaml_anchors/bundle.yml b/bundle/tests/yaml_anchors/databricks.yml similarity index 100% rename from bundle/tests/yaml_anchors/bundle.yml rename to bundle/tests/yaml_anchors/databricks.yml diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index d13a85d0..51d71ea2 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -72,7 +72,7 @@ var syncCmd = &cobra.Command{ // // To be uncommented and used once our VS Code extension is bundle aware. - // Until then, this could interfere with extension usage where a `bundle.yml` file is present. + // Until then, this could interfere with extension usage where a `databricks.yml` file is present. // See https://github.com/databricks/cli/pull/207. // // b := bundle.GetOrNil(cmd.Context())