databricks-cli/bundle/libraries/expand_glob_references.go

package libraries

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"

	"github.com/databricks/cli/bundle"
	"github.com/databricks/cli/libs/diag"
	"github.com/databricks/cli/libs/dyn"
)

type expand struct {
}

func matchError(p dyn.Path, l []dyn.Location, message string) diag.Diagnostic {
	return diag.Diagnostic{
		Severity:  diag.Error,
		Summary:   message,
		Locations: l,
		Paths:     []dyn.Path{p},
	}
}

func getLibDetails(v dyn.Value) (string, string, bool) {
	m := v.MustMap()
	whl, ok := m.GetByString("whl")
	if ok {
		return whl.MustString(), "whl", true
	}

	jar, ok := m.GetByString("jar")
	if ok {
		return jar.MustString(), "jar", true
	}

	return "", "", false
}

func findMatches(b *bundle.Bundle, path string) ([]string, error) {
	matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, path))
	if err != nil {
		return nil, err
	}

	if len(matches) == 0 {
		if isGlobPattern(path) {
			return nil, fmt.Errorf("no files match pattern: %s", path)
		} else {
			return nil, fmt.Errorf("file doesn't exist %s", path)
		}
	}

	// We make the matched path relative to the sync root path before storing it
	// to allow upload mutator to distinguish between local and remote paths
	for i, match := range matches {
		matches[i], err = filepath.Rel(b.SyncRootPath, match)
		if err != nil {
			return nil, err
		}
	}

	return matches, nil
}

// Checks if the path is a glob pattern
// It can contain *, [] or ? characters
func isGlobPattern(path string) bool {
	return strings.ContainsAny(path, "*?[")
}

func expandLibraries(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) {
	var output []dyn.Value
	var diags diag.Diagnostics

	libs := v.MustSequence()
	for i, lib := range libs {
		lp := p.Append(dyn.Index(i))
		path, libType, supported := getLibDetails(lib)
		if !supported || !IsLibraryLocal(path) {
			output = append(output, lib)
			continue
		}

		lp = lp.Append(dyn.Key(libType))

		matches, err := findMatches(b, path)
		if err != nil {
			diags = diags.Append(matchError(lp, lib.Locations(), err.Error()))
			continue
		}

		for _, match := range matches {
			output = append(output, dyn.NewValue(map[string]dyn.Value{
				libType: dyn.V(match),
			}, lib.Locations()))
		}
	}

	return diags, output
}

func expandEnvironmentDeps(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value) {
	var output []dyn.Value
	var diags diag.Diagnostics

	deps := v.MustSequence()
	for i, dep := range deps {
		lp := p.Append(dyn.Index(i))
		path := dep.MustString()
		if !IsLibraryLocal(path) {
			output = append(output, dep)
			continue
		}

		matches, err := findMatches(b, path)
		if err != nil {
			diags = diags.Append(matchError(lp, dep.Locations(), err.Error()))
			continue
		}

		for _, match := range matches {
			output = append(output, dyn.NewValue(match, dep.Locations()))
		}
	}

	return diags, output
}

type expandPattern struct {
	pattern dyn.Pattern
	fn      func(b *bundle.Bundle, p dyn.Path, v dyn.Value) (diag.Diagnostics, []dyn.Value)
}

var taskLibrariesPattern = dyn.NewPattern(
	dyn.Key("resources"),
	dyn.Key("jobs"),
	dyn.AnyKey(),
	dyn.Key("tasks"),
	dyn.AnyIndex(),
	dyn.Key("libraries"),
)

var forEachTaskLibrariesPattern = dyn.NewPattern(
	dyn.Key("resources"),
	dyn.Key("jobs"),
	dyn.AnyKey(),
	dyn.Key("tasks"),
	dyn.AnyIndex(),
	dyn.Key("for_each_task"),
	dyn.Key("task"),
	dyn.Key("libraries"),
)

var envDepsPattern = dyn.NewPattern(
	dyn.Key("resources"),
	dyn.Key("jobs"),
	dyn.AnyKey(),
	dyn.Key("environments"),
	dyn.AnyIndex(),
	dyn.Key("spec"),
	dyn.Key("dependencies"),
)

func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
	expanders := []expandPattern{
		{
			pattern: taskLibrariesPattern,
			fn:      expandLibraries,
		},
		{
			pattern: forEachTaskLibrariesPattern,
			fn:      expandLibraries,
		},
		{
			pattern: envDepsPattern,
			fn:      expandEnvironmentDeps,
		},
	}

	var diags diag.Diagnostics

	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
		var err error
		for _, expander := range expanders {
			v, err = dyn.MapByPattern(v, expander.pattern, func(p dyn.Path, lv dyn.Value) (dyn.Value, error) {
				d, output := expander.fn(b, p, lv)
				diags = diags.Extend(d)
				return dyn.V(output), nil
			})

			if err != nil {
				return dyn.InvalidValue, err
			}
		}

		return v, nil
	})

	if err != nil {
		diags = diags.Extend(diag.FromErr(err))
	}

	return diags
}

func (e *expand) Name() string {
	return "libraries.ExpandGlobReferences"
}

// ExpandGlobReferences expands any glob references in the libraries or environments section
// to corresponding local paths.
// We only expand local paths (i.e. paths that are relative to the sync root path).
// After expanding we make the paths relative to the sync root path to allow upload mutator later in the chain to
// distinguish between local and remote paths.
func ExpandGlobReferences() bundle.Mutator {
	return &expand{}
}