databricks-cli/bundle/bundle.go

// Package bundle is the top level package for Databricks Asset Bundles.
//
// A bundle is represented by the [Bundle] type. It consists of configuration
// and runtime state, such as a client to a Databricks workspace.
// Every mutation to a bundle's configuration or state is represented as a [Mutator].
// This interface makes every mutation observable and lets us reason about sequencing.
package bundle

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"sync"

	"github.com/databricks/cli/bundle/config"
	"github.com/databricks/cli/bundle/env"
	"github.com/databricks/cli/bundle/metadata"
	"github.com/databricks/cli/libs/fileset"
	"github.com/databricks/cli/libs/locker"
	"github.com/databricks/cli/libs/log"
	"github.com/databricks/cli/libs/tags"
	"github.com/databricks/cli/libs/terraform"
	"github.com/databricks/cli/libs/vfs"
	"github.com/databricks/databricks-sdk-go"
	sdkconfig "github.com/databricks/databricks-sdk-go/config"
	"github.com/hashicorp/terraform-exec/tfexec"
)

const internalFolder = ".internal"

type Bundle struct {
	// BundleRootPath is the local path to the root directory of the bundle.
	// It is set when we instantiate a new bundle instance.
	BundleRootPath string

	// BundleRoot is a virtual filesystem path to [BundleRootPath].
	// Exclusively use this field for filesystem operations.
	BundleRoot vfs.Path

	// SyncRootPath is the local path to the root directory of files that are synchronized to the workspace.
	// By default, it is the same as [BundleRootPath].
	// If it is different, it must be an ancestor to [BundleRootPath].
	// That is, [SyncRootPath] must contain [BundleRootPath].
	SyncRootPath string

	// SyncRoot is a virtual filesystem path to [SyncRootPath].
	// Exclusively use this field for filesystem operations.
	SyncRoot vfs.Path

	// Path to the root of git worktree containing the bundle.
	// This is the same as git repository root if worktrees are not used,
	// otherwise it's a descedant of git repository root.
	// https://git-scm.com/docs/git-worktree
	WorktreeRoot vfs.Path

	// Config contains the bundle configuration.
	// It is loaded from the bundle configuration files and mutators may update it.
	Config config.Root

	// Metadata about the bundle deployment. This is the interface Databricks services
	// rely on to integrate with bundles when they need additional information about
	// a bundle deployment.
	//
	// After deploy, a file containing the metadata (metadata.json) can be found
	// in the WSFS location containing the bundle state.
	Metadata metadata.Metadata

	// Store a pointer to the workspace client.
	// It can be initialized on demand after loading the configuration.
	clientOnce sync.Once
	client     *databricks.WorkspaceClient

	// Files that are synced to the workspace.file_path
	Files []fileset.File

	// Stores an initialized copy of this bundle's Terraform wrapper.
	Terraform *tfexec.Terraform

	// Stores the locker responsible for acquiring/releasing a deployment lock.
	Locker *locker.Locker

	Plan *terraform.Plan

	// if true, we skip approval checks for deploy, destroy resources and delete
	// files
	AutoApprove bool

	// Tagging is used to normalize tag keys and values.
	// The implementation depends on the cloud being targeted.
	Tagging tags.Cloud
}

func Load(ctx context.Context, path string) (*Bundle, error) {
	b := &Bundle{
		BundleRootPath: filepath.Clean(path),
		BundleRoot:     vfs.MustNew(path),
	}
	configFile, err := config.FileNames.FindInPath(path)
	if err != nil {
		return nil, err
	}
	log.Debugf(ctx, "Found bundle root at %s (file %s)", b.BundleRootPath, configFile)
	return b, nil
}

// MustLoad returns a bundle configuration.
// It returns an error if a bundle was not found or could not be loaded.
func MustLoad(ctx context.Context) (*Bundle, error) {
	root, err := mustGetRoot(ctx)
	if err != nil {
		return nil, err
	}

	return Load(ctx, root)
}

// TryLoad returns a bundle configuration if there is one, but doesn't fail if there isn't one.
// It returns an error if a bundle was found but could not be loaded.
// It returns a `nil` bundle if a bundle was not found.
func TryLoad(ctx context.Context) (*Bundle, error) {
	root, err := tryGetRoot(ctx)
	if err != nil {
		return nil, err
	}

	// No root is fine in this function.
	if root == "" {
		return nil, nil
	}

	return Load(ctx, root)
}

func (b *Bundle) InitializeWorkspaceClient() (*databricks.WorkspaceClient, error) {
	client, err := b.Config.Workspace.Client()
	if err != nil {
		return nil, fmt.Errorf("cannot resolve bundle auth configuration: %w", err)
	}
	return client, nil
}

func (b *Bundle) WorkspaceClient() *databricks.WorkspaceClient {
	b.clientOnce.Do(func() {
		var err error
		b.client, err = b.InitializeWorkspaceClient()
		if err != nil {
			panic(err)
		}
	})
	return b.client
}

// SetWorkpaceClient sets the workspace client for this bundle.
// This is used to inject a mock client for testing.
func (b *Bundle) SetWorkpaceClient(w *databricks.WorkspaceClient) {
	b.clientOnce.Do(func() {})
	b.client = w
}

// CacheDir returns directory to use for temporary files for this bundle.
// Scoped to the bundle's target.
func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) {
	if b.Config.Bundle.Target == "" {
		panic("target not set")
	}

	cacheDirName, exists := env.TempDir(ctx)
	if !exists || cacheDirName == "" {
		cacheDirName = filepath.Join(
			// Anchor at bundle root directory.
			b.BundleRootPath,
			// Static cache directory.
			".databricks",
			"bundle",
		)
	}

	// Fixed components of the result path.
	parts := []string{
		cacheDirName,
		// Scope with target name.
		b.Config.Bundle.Target,
	}

	// Append dynamic components of the result path.
	parts = append(parts, paths...)

	// Make directory if it doesn't exist yet.
	dir := filepath.Join(parts...)
	err := os.MkdirAll(dir, 0700)
	if err != nil {
		return "", err
	}

	return dir, nil
}

// This directory is used to store and automaticaly sync internal bundle files, such as, f.e
// notebook trampoline files for Python wheel and etc.
func (b *Bundle) InternalDir(ctx context.Context) (string, error) {
	cacheDir, err := b.CacheDir(ctx)
	if err != nil {
		return "", err
	}

	dir := filepath.Join(cacheDir, internalFolder)
	err = os.MkdirAll(dir, 0700)
	if err != nil {
		return dir, err
	}

	return dir, nil
}

// GetSyncIncludePatterns returns a list of user defined includes
// And also adds InternalDir folder to include list for sync command
// so this folder is always synced
func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) {
	internalDir, err := b.InternalDir(ctx)
	if err != nil {
		return nil, err
	}
	internalDirRel, err := filepath.Rel(b.BundleRootPath, internalDir)
	if err != nil {
		return nil, err
	}
	return append(b.Config.Sync.Include, filepath.ToSlash(filepath.Join(internalDirRel, "*.*"))), nil
}

// AuthEnv returns a map with environment variables and their values
// derived from the workspace client configuration that was resolved
// in the context of this bundle.
//
// This map can be used to configure authentication for tools that
// we call into from this bundle context.
func (b *Bundle) AuthEnv() (map[string]string, error) {
	if b.client == nil {
		return nil, fmt.Errorf("workspace client not initialized yet")
	}

	cfg := b.client.Config
	out := make(map[string]string)
	for _, attr := range sdkconfig.ConfigAttributes {
		// Ignore profile so that downstream tools don't try and reload
		// the profile even though we know the current configuration is valid.
		if attr.Name == "profile" {
			continue
		}
		if len(attr.EnvVars) == 0 {
			continue
		}
		if attr.IsZero(cfg) {
			continue
		}
		out[attr.EnvVars[0]] = attr.GetString(cfg)
	}

	return out, nil
}