databricks-cli/bundle/config/root.go

package config

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/databricks/cli/bundle/config/variable"
	"github.com/databricks/databricks-sdk-go/service/jobs"
	"github.com/ghodss/yaml"
	"github.com/imdario/mergo"
)

type ConfigFileNames []string

// FileNames contains allowed names of bundle configuration files.
var FileNames = ConfigFileNames{"databricks.yml", "databricks.yaml", "bundle.yml", "bundle.yaml"}

func (c ConfigFileNames) FindInPath(path string) (string, error) {
	result := ""
	var firstErr error

	for _, file := range c {
		filePath := filepath.Join(path, file)
		_, err := os.Stat(filePath)
		if err == nil {
			if result != "" {
				return "", fmt.Errorf("multiple bundle root configuration files found in %s", path)
			}
			result = filePath
		} else {
			if firstErr == nil {
				firstErr = err
			}
		}
	}

	if result == "" {
		return "", firstErr
	}
	return result, nil
}

type Root struct {
	// Path contains the directory path to the root of the bundle.
	// It is set when loading `databricks.yml`.
	Path string `json:"-" bundle:"readonly"`

	// Contains user defined variables
	Variables map[string]*variable.Variable `json:"variables,omitempty"`

	// Bundle contains details about this bundle, such as its name,
	// version of the spec (TODO), default cluster, default warehouse, etc.
	Bundle Bundle `json:"bundle"`

	// Include specifies a list of patterns of file names to load and
	// merge into the this configuration. Only includes defined in the root
	// `databricks.yml` are processed. Defaults to an empty list.
	Include []string `json:"include,omitempty"`

	// Workspace contains details about the workspace to connect to
	// and paths in the workspace tree to use for this bundle.
	Workspace Workspace `json:"workspace,omitempty"`

	// Artifacts contains a description of all code artifacts in this bundle.
	Artifacts Artifacts `json:"artifacts,omitempty"`

	// Resources contains a description of all Databricks resources
	// to deploy in this bundle (e.g. jobs, pipelines, etc.).
	Resources Resources `json:"resources,omitempty"`

	// Targets can be used to differentiate settings and resources between
	// bundle deployment targets (e.g. development, staging, production).
	// If not specified, the code below initializes this field with a
	// single default-initialized target called "default".
	Targets map[string]*Target `json:"targets,omitempty"`

	// DEPRECATED. Left for backward compatibility with Targets
	Environments map[string]*Target `json:"environments,omitempty"`

	// Sync section specifies options for files synchronization
	Sync Sync `json:"sync"`

	// RunAs section allows to define an execution identity for jobs and pipelines runs
	RunAs *jobs.JobRunAs `json:"run_as,omitempty"`
}

func Load(path string) (*Root, error) {
	var r Root

	stat, err := os.Stat(path)
	if err != nil {
		return nil, err
	}

	// If we were given a directory, assume this is the bundle root.
	if stat.IsDir() {
		path, err = FileNames.FindInPath(path)
		if err != nil {
			return nil, err
		}
	}

	if err := r.Load(path); err != nil {
		return nil, err
	}

	return &r, nil
}

// SetConfigFilePath configures the path that its configuration
// was loaded from in configuration leafs that require it.
func (r *Root) SetConfigFilePath(path string) {
	r.Resources.SetConfigFilePath(path)
	if r.Artifacts != nil {
		r.Artifacts.SetConfigFilePath(path)
	}

	if r.Targets != nil {
		for _, env := range r.Targets {
			if env == nil {
				continue
			}
			if env.Resources != nil {
				env.Resources.SetConfigFilePath(path)
			}
			if env.Artifacts != nil {
				env.Artifacts.SetConfigFilePath(path)
			}
		}
	}
}

// Initializes variables using values passed from the command line flag
// Input has to be a string of the form `foo=bar`. In this case the variable with
// name `foo` is assigned the value `bar`
func (r *Root) InitializeVariables(vars []string) error {
	for _, variable := range vars {
		parsedVariable := strings.SplitN(variable, "=", 2)
		if len(parsedVariable) != 2 {
			return fmt.Errorf("unexpected flag value for variable assignment: %s", variable)
		}
		name := parsedVariable[0]
		val := parsedVariable[1]

		if _, ok := r.Variables[name]; !ok {
			return fmt.Errorf("variable %s has not been defined", name)
		}
		err := r.Variables[name].Set(val)
		if err != nil {
			return fmt.Errorf("failed to assign %s to %s: %s", val, name, err)
		}
	}
	return nil
}

func (r *Root) Load(path string) error {
	raw, err := os.ReadFile(path)
	if err != nil {
		return err
	}
	err = yaml.Unmarshal(raw, r)
	if err != nil {
		return fmt.Errorf("failed to load %s: %w", path, err)
	}

	if r.Environments != nil && r.Targets != nil {
		return fmt.Errorf("both 'environments' and 'targets' are specified, only 'targets' should be used: %s", path)
	}

	if r.Environments != nil {
		//TODO: add a command line notice that this is a deprecated option.
		r.Targets = r.Environments
	}

	r.Path = filepath.Dir(path)
	r.SetConfigFilePath(path)

	_, err = r.Resources.VerifyUniqueResourceIdentifiers()
	return err
}

func (r *Root) Merge(other *Root) error {
	err := r.Sync.Merge(r, other)
	if err != nil {
		return err
	}
	other.Sync = Sync{}

	// TODO: when hooking into merge semantics, disallow setting path on the target instance.
	other.Path = ""

	// Check for safe merge, protecting against duplicate resource identifiers
	err = r.Resources.VerifySafeMerge(&other.Resources)
	if err != nil {
		return err
	}

	// TODO: define and test semantics for merging.
	return mergo.Merge(r, other, mergo.WithOverride)
}

func (r *Root) MergeTargetOverrides(target *Target) error {
	var err error

	// Target may be nil if it's empty.
	if target == nil {
		return nil
	}

	if target.Bundle != nil {
		err = mergo.Merge(&r.Bundle, target.Bundle, mergo.WithOverride)
		if err != nil {
			return err
		}
	}

	if target.Workspace != nil {
		err = mergo.Merge(&r.Workspace, target.Workspace, mergo.WithOverride)
		if err != nil {
			return err
		}
	}

	if target.Artifacts != nil {
		err = mergo.Merge(&r.Artifacts, target.Artifacts, mergo.WithOverride, mergo.WithAppendSlice)
		if err != nil {
			return err
		}
	}

	if target.Resources != nil {
		err = mergo.Merge(&r.Resources, target.Resources, mergo.WithOverride, mergo.WithAppendSlice)
		if err != nil {
			return err
		}

		err = r.Resources.MergeJobClusters()
		if err != nil {
			return err
		}
	}

	if target.Variables != nil {
		for k, v := range target.Variables {
			variable, ok := r.Variables[k]
			if !ok {
				return fmt.Errorf("variable %s is not defined but is assigned a value", k)
			}
			// we only allow overrides of the default value for a variable
			defaultVal := v
			variable.Default = &defaultVal
		}
	}

	if target.RunAs != nil {
		r.RunAs = target.RunAs
	}

	if target.Mode != "" {
		r.Bundle.Mode = target.Mode
	}

	if target.ComputeID != "" {
		r.Bundle.ComputeID = target.ComputeID
	}

	git := &r.Bundle.Git
	if target.Git.Branch != "" {
		git.Branch = target.Git.Branch
		git.Inferred = false
	}
	if target.Git.Commit != "" {
		git.Commit = target.Git.Commit
	}
	if target.Git.OriginURL != "" {
		git.OriginURL = target.Git.OriginURL
	}

	return nil
}