Remove package project (#321)

## Changes
<!-- Summary of your changes that are easy to understand -->

This PR removes the project package and it's dependents in the bricks
repo

## Tests
<!-- How is this tested? -->

cmd/configure/configure.go

@@ -7,9 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/databricks/bricks/cmd/prompt"
 	"github.com/databricks/bricks/cmd/root"
-	"github.com/databricks/bricks/project"
 	"github.com/spf13/cobra"
 	"gopkg.in/ini.v1"
 )
@@ -20,7 +18,7 @@ type Configs struct {
 	Profile string `ini:"-"`
 }
 
-var noInteractive, tokenMode bool
+var tokenMode bool
 
 func (cfg *Configs) loadNonInteractive(cmd *cobra.Command) error {
 	host, err := cmd.Flags().GetString("host")
@@ -43,53 +41,6 @@ func (cfg *Configs) loadNonInteractive(cmd *cobra.Command) error {
 	return nil
 }
 
-func (cfg *Configs) loadInteractive(cmd *cobra.Command) error {
-	res := prompt.Results{}
-	questions := prompt.Questions{}
-
-	host, err := cmd.Flags().GetString("host")
-	if err != nil || host == "" {
-		questions = append(questions, prompt.Text{
-			Key:   "host",
-			Label: "Databricks Host",
-			Default: func(res prompt.Results) string {
-				return cfg.Host
-			},
-			Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) {
-				cfg.Host = ans.Value
-			},
-		})
-	} else {
-		cfg.Host = host
-	}
-
-	if tokenMode {
-		questions = append(questions, prompt.Text{
-			Key:   "token",
-			Label: "Databricks Token",
-			Default: func(res prompt.Results) string {
-				return cfg.Token
-			},
-			Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) {
-				cfg.Token = ans.Value
-			},
-		})
-	}
-
-	err = questions.Ask(res)
-	if err != nil {
-		return err
-	}
-
-	for _, answer := range res {
-		if answer.Callback != nil {
-			answer.Callback(answer, nil, res)
-		}
-	}
-
-	return nil
-}
-
 var configureCmd = &cobra.Command{
 	Use:   "configure",
 	Short: "Configure authentication",
@@ -135,11 +86,7 @@ var configureCmd = &cobra.Command{
 			return fmt.Errorf("unmarshal loaded config: %w", err)
 		}
 
-		if noInteractive {
+		err = cfg.loadNonInteractive(cmd)
-			err = cfg.loadNonInteractive(cmd)
-		} else {
-			err = cfg.loadInteractive(cmd)
-		}
 		if err != nil {
 			return fmt.Errorf("reading configs: %w", err)
 		}
@@ -171,7 +118,6 @@ var configureCmd = &cobra.Command{
 func init() {
 	root.RootCmd.AddCommand(configureCmd)
 	configureCmd.Flags().BoolVarP(&tokenMode, "token", "t", false, "Configure using Databricks Personal Access Token")
-	configureCmd.Flags().BoolVar(&noInteractive, "no-interactive", false, "Don't show interactive prompts for inputs. Read directly from stdin.")
 	configureCmd.Flags().String("host", "", "Host to connect to.")
 	configureCmd.Flags().String("profile", "DEFAULT", "CLI connection profile to use.")
 }

cmd/configure/configure_test.go

@@ -52,7 +52,7 @@ func TestDefaultConfigureNoInteractive(t *testing.T) {
 	})
 	os.Stdin = inp
 
-	root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host"})
+	root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host"})
 
 	err := root.RootCmd.ExecuteContext(ctx)
 	assert.NoError(t, err)
@@ -84,7 +84,7 @@ func TestConfigFileFromEnvNoInteractive(t *testing.T) {
 	t.Cleanup(func() { os.Stdin = oldStdin })
 	os.Stdin = inp
 
-	root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host"})
+	root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host"})
 
 	err := root.RootCmd.ExecuteContext(ctx)
 	assert.NoError(t, err)
@@ -112,7 +112,7 @@ func TestCustomProfileConfigureNoInteractive(t *testing.T) {
 	t.Cleanup(func() { os.Stdin = oldStdin })
 	os.Stdin = inp
 
-	root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host", "--profile", "CUSTOM"})
+	root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host", "--profile", "CUSTOM"})
 
 	err := root.RootCmd.ExecuteContext(ctx)
 	assert.NoError(t, err)

cmd/fs/fs.go

@@ -14,14 +14,4 @@ var fsCmd = &cobra.Command{
 
 func init() {
 	root.RootCmd.AddCommand(fsCmd)
-
-	// Here you will define your flags and configuration settings.
-
-	// Cobra supports Persistent Flags which will work for this command
-	// and all subcommands, e.g.:
-	// fsCmd.PersistentFlags().String("foo", "", "A help for foo")
-
-	// Cobra supports local flags which will only run when this command
-	// is called directly, e.g.:
-	// fsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 }

cmd/fs/ls.go

@@ -3,7 +3,6 @@ package fs
 import (
 	"fmt"
 
-	"github.com/databricks/bricks/project"
 	"github.com/spf13/cobra"
 )
 
@@ -12,25 +11,12 @@ var lsCmd = &cobra.Command{
 	Use:   "ls <dir-name>",
 	Short: "Lists files",
 	Long:  `Lists files`,
-	Args:  cobra.ExactArgs(1),
 
-	PreRunE: project.Configure,
+	RunE: func(cmd *cobra.Command, args []string) error {
-	Run: func(cmd *cobra.Command, args []string) {
+		return fmt.Errorf("TODO")
-		wsc := project.Get(cmd.Context()).WorkspacesClient()
-		listStatusResponse, err := wsc.Dbfs.ListByPath(cmd.Context(), args[0])
-		if err != nil {
-			panic(err)
-		}
-		files := listStatusResponse.Files
-		// TODO: output formatting: JSON, CSV, tables and default
-		for _, v := range files {
-			fmt.Printf("[-] %s (%d, %v)\n", v.Path, v.FileSize, v.IsDir)
-		}
 	},
 }
 
 func init() {
-	// TODO: pietern: conditionally register commands
-	// fabianj: don't do it
 	fsCmd.AddCommand(lsCmd)
 }

cmd/init/init.go

@@ -1,128 +0,0 @@
-package init
-
-import (
-	"embed"
-	"fmt"
-	"os"
-	"path"
-
-	"github.com/databricks/bricks/cmd/prompt"
-	"github.com/databricks/bricks/cmd/root"
-	"github.com/databricks/bricks/project"
-	"github.com/ghodss/yaml"
-	"github.com/spf13/cobra"
-)
-
-//go:embed templates
-var templates embed.FS
-
-// initCmd represents the init command
-var initCmd = &cobra.Command{
-	Use:   "init",
-	Short: "Project starter templates",
-	Long:  `Generate project templates`,
-
-	RunE: func(cmd *cobra.Command, args []string) error {
-		if project.IsDatabricksProject() {
-			return fmt.Errorf("this path is already a Databricks project")
-		}
-		profileChoice, err := getConnectionProfile()
-		if err != nil {
-			return err
-		}
-		wd, _ := os.Getwd()
-		q := prompt.Questions{
-			prompt.Text{
-				Key:   "name",
-				Label: "Project name",
-				Default: func(res prompt.Results) string {
-					return path.Base(wd)
-				},
-				Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) {
-					config.Name = ans.Value
-				},
-			},
-			*profileChoice,
-			prompt.Choice{Key: "language", Label: "Project language", Answers: prompt.Answers{
-				{
-					Value:    "Python",
-					Details:  "Machine learning and data engineering focused projects",
-					Callback: nil,
-				},
-				{
-					Value:    "Scala",
-					Details:  "Data engineering focused projects with strong typing",
-					Callback: nil,
-				},
-			}},
-			prompt.Choice{Key: "isolation", Label: "Deployment isolation", Answers: prompt.Answers{
-				{
-					Value:    "None",
-					Details:  "Use shared Databricks workspace resources for all project team members",
-					Callback: nil,
-				},
-				{
-					Value:   "Soft",
-					Details: "Prepend prefixes to each team member's deployment",
-					Callback: func(
-						ans prompt.Answer, config *project.Config, res prompt.Results) {
-						config.Isolation = project.Soft
-					},
-				},
-			}},
-			// DBR selection
-			// Choice{"cloud", "Cloud", Answers{
-			// 	{"AWS", "Amazon Web Services", nil},
-			// 	{"Azure", "Microsoft Azure Cloud", nil},
-			// 	{"GCP", "Google Cloud Platform", nil},
-			// }},
-			// Choice{"ci", "Continuous Integration", Answers{
-			// 	{"None", "Do not create continuous integration configuration", nil},
-			// 	{"GitHub Actions", "Create .github/workflows/push.yml configuration", nil},
-			// 	{"Azure DevOps", "Create basic build and test pipelines", nil},
-			// }},
-			// Choice{"ide", "Integrated Development Environment", Answers{
-			// 	{"None", "Do not create templates for IDE", nil},
-			// 	{"VSCode", "Create .devcontainer and other useful things", nil},
-			// 	{"PyCharm", "Create project conf and other things", nil},
-			// }},
-		}
-		res := prompt.Results{}
-		err = q.Ask(res)
-		if err != nil {
-			return err
-		}
-		var config project.Config
-		for _, ans := range res {
-			if ans.Callback == nil {
-				continue
-			}
-			ans.Callback(ans, &config, res)
-		}
-		raw, err := yaml.Marshal(config)
-		if err != nil {
-			return err
-		}
-		newConfig, err := os.Create(fmt.Sprintf("%s/%s", wd, project.ConfigFile))
-		if err != nil {
-			return err
-		}
-		_, err = newConfig.Write(raw)
-		if err != nil {
-			return err
-		}
-		d, err := templates.ReadDir(".")
-		if err != nil {
-			return err
-		}
-		for _, v := range d {
-			cmd.Printf("template found: %v", v.Name())
-		}
-		cmd.Print("Config initialized!")
-		return err
-	},
-}
-
-func init() {
-	root.RootCmd.AddCommand(initCmd)
-}

cmd/init/legacy-cli.go

@@ -1,57 +0,0 @@
-package init
-
-import (
-	"fmt"
-
-	"github.com/databricks/bricks/cmd/prompt"
-	"github.com/databricks/bricks/project"
-	"github.com/mitchellh/go-homedir"
-	"gopkg.in/ini.v1"
-)
-
-func loadCliProfiles() (profiles []prompt.Answer, err error) {
-	file, err := homedir.Expand("~/.databrickscfg")
-	if err != nil {
-		return
-	}
-	gitConfig, err := ini.Load(file)
-	if err != nil {
-		return
-	}
-	for _, v := range gitConfig.Sections() {
-		host, err := v.GetKey("host")
-		if err != nil {
-			// invalid profile
-			continue
-		}
-		// TODO: verify these tokens to work, becaus they may be expired
-		profiles = append(profiles, prompt.Answer{
-			Value:   v.Name(),
-			Details: fmt.Sprintf(`Connecting to "%s" workspace`, host),
-			Callback: func(ans prompt.Answer, config *project.Config, _ prompt.Results) {
-				if config.Environments == nil {
-					config.Environments = make(map[string]project.Environment)
-				}
-				config.Environments[project.DefaultEnvironment] = project.Environment{
-					Workspace: project.Workspace{
-						Profile: ans.Value,
-					},
-				}
-			},
-		})
-	}
-	return
-}
-
-func getConnectionProfile() (*prompt.Choice, error) {
-	profiles, err := loadCliProfiles()
-	if err != nil {
-		return nil, err
-	}
-	// TODO: propmt for password and create ~/.databrickscfg
-	return &prompt.Choice{
-		Key:     "profile",
-		Label:   "Databricks CLI profile",
-		Answers: profiles,
-	}, err
-}

cmd/init/templates/python-notebook/Hello.py

@@ -1,4 +0,0 @@
-# Databricks notebook source
-
-# this was automatically generated
-display(spark.tables())

cmd/prompt/prompt.go

@@ -1,89 +0,0 @@
-package prompt
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/databricks/bricks/project"
-	"github.com/manifoldco/promptui"
-)
-
-type Results map[string]Answer
-
-type Question interface {
-	Ask(res Results) (key string, ans Answer, err error)
-}
-
-type Questions []Question
-
-func (qq Questions) Ask(res Results) error {
-	for _, v := range qq {
-		key, ans, err := v.Ask(res)
-		if err != nil {
-			return err
-		}
-		res[key] = ans
-	}
-	return nil
-}
-
-type Text struct {
-	Key      string
-	Label    string
-	Default  func(res Results) string
-	Callback AnswerCallback
-	Stdin    io.ReadCloser
-}
-
-func (t Text) Ask(res Results) (string, Answer, error) {
-	def := ""
-	if t.Default != nil {
-		def = t.Default(res)
-	}
-	v, err := (&promptui.Prompt{
-		Label:   t.Label,
-		Default: def,
-		Stdin:   t.Stdin,
-	}).Run()
-	return t.Key, Answer{
-		Value:    v,
-		Callback: t.Callback,
-	}, err
-}
-
-type Choice struct {
-	Key     string
-	Label   string
-	Answers []Answer
-	Stdin   io.ReadCloser
-}
-
-func (q Choice) Ask(res Results) (string, Answer, error) {
-	// TODO: validate and re-ask
-	prompt := promptui.Select{
-		Label: q.Label,
-		Items: q.Answers,
-		Templates: &promptui.SelectTemplates{
-			Label:    `{{ .Value }}`,
-			Details:  `{{ .Details | green }}`,
-			Selected: fmt.Sprintf(`{{ "%s" | faint }}: {{ .Value | bold }}`, q.Label),
-		},
-		Stdin: q.Stdin,
-	}
-	i, _, err := prompt.Run()
-	return q.Key, q.Answers[i], err
-}
-
-type Answers []Answer
-
-type AnswerCallback func(ans Answer, config *project.Config, res Results)
-
-type Answer struct {
-	Value    string
-	Details  string
-	Callback AnswerCallback
-}
-
-func (a Answer) String() string {
-	return a.Value
-}

go.mod

@@ -6,8 +6,6 @@ require (
 	github.com/atotto/clipboard v0.1.4
 	github.com/databricks/databricks-sdk-go v0.7.0
 	github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
-	github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license
-	github.com/mitchellh/go-homedir v1.1.0 // MIT
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT
 	github.com/spf13/cobra v1.7.0 // Apache 2.0
@@ -41,7 +39,6 @@ require (
 require (
 	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.3 // indirect

go.sum

@@ -12,12 +12,6 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkE
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
@@ -89,15 +83,11 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
-github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
-github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
@@ -159,7 +149,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

main.go

@@ -6,9 +6,7 @@ import (
 	_ "github.com/databricks/bricks/cmd/bundle"
 	_ "github.com/databricks/bricks/cmd/bundle/debug"
 	_ "github.com/databricks/bricks/cmd/bundle/debug/deploy"
-	_ "github.com/databricks/bricks/cmd/configure"
 	_ "github.com/databricks/bricks/cmd/fs"
-	_ "github.com/databricks/bricks/cmd/init"
 	"github.com/databricks/bricks/cmd/root"
 	_ "github.com/databricks/bricks/cmd/sync"
 	_ "github.com/databricks/bricks/cmd/version"

project/README.md

@@ -1,6 +0,0 @@
-Project Configuration
----
-
-_Good implicit defaults is better than explicit complex configuration._
-
-Regardless of current working directory, `bricks` finds project root with `databricks.yml` file up the directory tree. Technically, there might be couple of different Databricks Projects in the same Git repository, but the recommended scenario is to have just one `databricks.yml` in the root of Git repo.

project/config.go

@@ -1,137 +0,0 @@
-package project
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"reflect"
-
-	"github.com/databricks/bricks/folders"
-	"github.com/databricks/databricks-sdk-go/service/clusters"
-
-	"github.com/ghodss/yaml"
-)
-
-type Isolation string
-
-const (
-	None Isolation = ""
-	Soft Isolation = "soft"
-)
-
-// ConfigFile is the name of project configuration file
-const ConfigFile = "databricks.yml"
-
-type Assertions struct {
-	Groups            []string `json:"groups,omitempty"`
-	Secrets           []string `json:"secrets,omitempty"`
-	ServicePrincipals []string `json:"service_principals,omitempty"`
-}
-
-type Config struct {
-	Name      string    `json:"name"` // or do default from folder name?..
-	Isolation Isolation `json:"isolation,omitempty"`
-
-	// development-time vs deployment-time resources
-	DevCluster *clusters.ClusterInfo `json:"dev_cluster,omitempty"`
-
-	// Assertions defines a list of configurations expected to be applied
-	// to the workspace by a higher-privileged user (or service principal)
-	// in order for the deploy command to work, as individual project teams
-	// in almost all the cases don’t have admin privileges on Databricks
-	// workspaces.
-	//
-	// This configuration simplifies the flexibility of individual project
-	// teams, make jobs deployment easier and portable across environments.
-	// This configuration block would contain the following entities to be
-	// created by administrator users or admin-level automation, like Terraform
-	// and/or SCIM provisioning.
-	Assertions *Assertions `json:"assertions,omitempty"`
-
-	// Environments contain this project's defined environments.
-	// They can be used to differentiate settings and resources between
-	// development, staging, production, etc.
-	// If not specified, the code below initializes this field with a
-	// single default-initialized environment called "development".
-	Environments map[string]Environment `json:"environments"`
-}
-
-func (c Config) IsDevClusterDefined() bool {
-	return reflect.ValueOf(c.DevCluster).IsZero()
-}
-
-// IsDevClusterJustReference denotes reference-only clusters.
-// This conflicts with Soft isolation. Happens for cost-restricted projects,
-// where there's only a single Shared Autoscaling cluster per workspace and
-// general users have no ability to create other iteractive clusters.
-func (c *Config) IsDevClusterJustReference() bool {
-	if c.DevCluster.ClusterName == "" {
-		return false
-	}
-	return reflect.DeepEqual(c.DevCluster, &clusters.ClusterInfo{
-		ClusterName: c.DevCluster.ClusterName,
-	})
-}
-
-// IsDatabricksProject returns true for folders with `databricks.yml`
-// in the parent tree
-func IsDatabricksProject() bool {
-	_, err := findProjectRoot()
-	return err == nil
-}
-
-func loadProjectConf(root string) (c Config, err error) {
-	configFilePath := filepath.Join(root, ConfigFile)
-
-	if _, err := os.Stat(configFilePath); errors.Is(err, os.ErrNotExist) {
-		baseDir := filepath.Base(root)
-		// If bricks config file is missing we assume the project root dir name
-		// as the name of the project
-		return validateAndApplyProjectDefaults(Config{Name: baseDir})
-	}
-
-	config, err := os.Open(configFilePath)
-	if err != nil {
-		return
-	}
-	defer config.Close()
-	raw, err := io.ReadAll(config)
-	if err != nil {
-		return
-	}
-	err = yaml.Unmarshal(raw, &c)
-	if err != nil {
-		return
-	}
-	return validateAndApplyProjectDefaults(c)
-}
-
-func validateAndApplyProjectDefaults(c Config) (Config, error) {
-	// If no environments are specified, define default environment under default name.
-	if c.Environments == nil {
-		c.Environments = make(map[string]Environment)
-		c.Environments[DefaultEnvironment] = Environment{}
-	}
-	// defaultCluster := clusters.ClusterInfo{
-	// 	NodeTypeID: "smallest",
-	// 	SparkVersion: "latest",
-	// 	AutoterminationMinutes: 30,
-	// }
-	return c, nil
-}
-
-func findProjectRoot() (string, error) {
-	wd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	dir, err := folders.FindDirWithLeaf(wd, ConfigFile)
-	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
-			return "", fmt.Errorf("cannot find %s anywhere", ConfigFile)
-		}
-	}
-	return dir, nil
-}

project/config_test.go

@@ -1,14 +0,0 @@
-package project
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestLoadProjectConf(t *testing.T) {
-	prj, err := loadProjectConf("./testdata")
-	assert.NoError(t, err)
-	assert.Equal(t, "dev", prj.Name)
-	assert.True(t, prj.IsDevClusterJustReference())
-}

project/environment.go

@@ -1,41 +0,0 @@
-package project
-
-import (
-	"os"
-
-	"github.com/spf13/cobra"
-)
-
-const bricksEnv = "BRICKS_ENV"
-
-const DefaultEnvironment = "development"
-
-// Workspace defines configurables at the workspace level.
-type Workspace struct {
-	Profile string `json:"profile,omitempty"`
-}
-
-// Environment defines all configurables for a single environment.
-type Environment struct {
-	Workspace Workspace `json:"workspace"`
-}
-
-// getEnvironment returns the name of the environment to operate in.
-func getEnvironment(cmd *cobra.Command) (value string) {
-	// The command line flag takes precedence.
-	flag := cmd.Flag("environment")
-	if flag != nil {
-		value = flag.Value.String()
-		if value != "" {
-			return
-		}
-	}
-
-	// If it's not set, use the environment variable.
-	value = os.Getenv(bricksEnv)
-	if value != "" {
-		return
-	}
-
-	return DefaultEnvironment
-}

project/environment_test.go

@@ -1,38 +0,0 @@
-package project
-
-import (
-	"testing"
-
-	"github.com/spf13/cobra"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestEnvironmentFromCommand(t *testing.T) {
-	var cmd cobra.Command
-	cmd.Flags().String("environment", "", "specify environment")
-	cmd.Flags().Set("environment", "env-from-arg")
-	t.Setenv(bricksEnv, "")
-
-	value := getEnvironment(&cmd)
-	assert.Equal(t, "env-from-arg", value)
-}
-
-func TestEnvironmentFromEnvironment(t *testing.T) {
-	var cmd cobra.Command
-	cmd.Flags().String("environment", "", "specify environment")
-	cmd.Flags().Set("environment", "")
-	t.Setenv(bricksEnv, "env-from-env")
-
-	value := getEnvironment(&cmd)
-	assert.Equal(t, "env-from-env", value)
-}
-
-func TestEnvironmentDefault(t *testing.T) {
-	var cmd cobra.Command
-	cmd.Flags().String("environment", "", "specify environment")
-	cmd.Flags().Set("environment", "")
-	t.Setenv(bricksEnv, "")
-
-	value := getEnvironment(&cmd)
-	assert.Equal(t, DefaultEnvironment, value)
-}

project/flavor.go

@@ -1,81 +0,0 @@
-package project
-
-// type Flavor interface {
-// 	// Name returns a tuple of flavor key and readable name
-// 	Name() (string, string)
-
-// 	// Detected returns true on successful metadata checks
-// 	Detected() bool
-
-// 	// Build triggers packaging subprocesses
-// 	Build(context.Context) error
-// 	// TODO: Init() Questions
-// 	// TODO: Deploy(context.Context) error
-// }
-
-// var _ Flavor = PythonWheel{}
-
-// type PythonWheel struct{}
-
-// func (pw PythonWheel) Name() (string, string) {
-// 	return "wheel", "Python Wheel"
-// }
-
-// func (pw PythonWheel) Detected() bool {
-// 	root, err := findProjectRoot()
-// 	if err != nil {
-// 		return false
-// 	}
-// 	_, err = os.Stat(fmt.Sprintf("%s/setup.py", root))
-// 	return err == nil
-// }
-
-// func (pw PythonWheel) Build(ctx context.Context) error {
-// 	defer toTheRootAndBack()()
-// 	// do subprocesses or https://github.com/go-python/cpy3
-// 	// it all depends on complexity and binary size
-// 	// TODO: detect if there's an .venv here and call setup.py with ENV vars of it
-// 	// TODO: where.exe python (WIN) / which python (UNIX)
-// 	cmd := exec.CommandContext(ctx, "python", "setup.py", "bdist-wheel")
-// 	err := cmd.Run()
-// 	if err != nil {
-// 		return err
-// 	}
-// 	return nil
-// }
-
-// func toTheRootAndBack() func() {
-// 	wd, _ := os.Getwd()
-// 	root, _ := findProjectRoot()
-// 	os.Chdir(root)
-// 	return func() {
-// 		os.Chdir(wd)
-// 	}
-// }
-
-// var _ Flavor = PythonNotebooks{}
-
-// type PythonNotebooks struct{}
-
-// func (n PythonNotebooks) Name() (string, string) {
-// 	// or just "notebooks", as we might shuffle in scala?...
-// 	return "python-notebooks", "Python Notebooks"
-// }
-
-// func (n PythonNotebooks) Detected() bool {
-// 	// TODO: Steps:
-// 	// - get all filenames
-// 	// - read first X bytes from random 10 files and check
-// 	// if they're "Databricks Notebook Source"
-// 	return false
-// }
-
-// func (n PythonNotebooks) Build(ctx context.Context) error {
-// 	// TODO: perhaps some linting?..
-// 	return nil
-// }
-
-// func (n PythonNotebooks) Deploy(ctx context.Context) error {
-// 	// TODO: recursively upload notebooks to a given workspace path
-// 	return nil
-// }

project/project.go

@@ -1,271 +0,0 @@
-package project
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"sync"
-
-	"github.com/databricks/bricks/libs/git"
-	"github.com/databricks/databricks-sdk-go"
-	"github.com/databricks/databricks-sdk-go/service/commands"
-	"github.com/databricks/databricks-sdk-go/service/scim"
-	"github.com/spf13/cobra"
-)
-
-const CacheDirName = ".databricks"
-
-type project struct {
-	mu sync.Mutex
-
-	root string
-	env  string
-
-	config      *Config
-	environment *Environment
-	wsc         *databricks.WorkspaceClient
-	me          *scim.User
-	fileSet     *git.FileSet
-}
-
-// Configure is used as a PreRunE function for all commands that
-// require a project to be configured. If a project could successfully
-// be found and loaded, it is set on the command's context object.
-func Configure(cmd *cobra.Command, args []string) error {
-	root, err := getRoot()
-	if err != nil {
-		return err
-	}
-
-	ctx, err := Initialize(cmd.Context(), root, getEnvironment(cmd))
-	if err != nil {
-		return err
-	}
-
-	cmd.SetContext(ctx)
-	return nil
-}
-
-// Placeholder to use as unique key in context.Context.
-var projectKey int
-
-// Initialize loads a project configuration given a root and environment.
-// It stores the project on a new context.
-// The project is available through the `Get()` function.
-func Initialize(ctx context.Context, root, env string) (context.Context, error) {
-	config, err := loadProjectConf(root)
-	if err != nil {
-		return nil, err
-	}
-
-	// Confirm that the specified environment is valid.
-	environment, ok := config.Environments[env]
-	if !ok {
-		return nil, fmt.Errorf("environment [%s] not defined", env)
-	}
-
-	fileSet, err := git.NewFileSet(root)
-	if err != nil {
-		return nil, err
-	}
-	err = fileSet.EnsureValidGitIgnoreExists()
-	if err != nil {
-		return nil, err
-	}
-
-	p := project{
-		root: root,
-		env:  env,
-
-		config:      &config,
-		environment: &environment,
-		fileSet:     fileSet,
-	}
-
-	p.initializeWorkspacesClient(ctx)
-	return context.WithValue(ctx, &projectKey, &p), nil
-}
-
-func (p *project) initializeWorkspacesClient(ctx context.Context) {
-	var config databricks.Config
-
-	// If the config specifies a profile, or other authentication related properties,
-	// pass them along to the SDK here. If nothing is defined, the SDK will figure
-	// out which autentication mechanism to use using enviroment variables.
-	if p.environment.Workspace.Profile != "" {
-		config.Profile = p.environment.Workspace.Profile
-	}
-
-	p.wsc = databricks.Must(databricks.NewWorkspaceClient(&config))
-}
-
-// Get returns the project as configured on the context.
-// It panics if it isn't configured.
-func Get(ctx context.Context) *project {
-	project, ok := ctx.Value(&projectKey).(*project)
-	if !ok {
-		panic(`context not configured with project`)
-	}
-	return project
-}
-
-// Make sure to initialize the workspaces client on project init
-func (p *project) WorkspacesClient() *databricks.WorkspaceClient {
-	return p.wsc
-}
-
-func (p *project) Root() string {
-	return p.root
-}
-
-func (p *project) GetFileSet() *git.FileSet {
-	return p.fileSet
-}
-
-// This cache dir will contain any state, state overrides (per user overrides
-// to the project config) or any generated artifacts (eg: sync snapshots)
-// that should never be checked into Git.
-//
-// We enfore that cache dir (.databricks) is added to .gitignore
-// because it contains per-user overrides that we do not want users to
-// accidentally check into git
-func (p *project) CacheDir() (string, error) {
-	// assert cache dir is present in git ignore
-	ign, err := p.fileSet.IgnoreDirectory(fmt.Sprintf("/%s/", CacheDirName))
-	if err != nil {
-		return "", fmt.Errorf("failed to check if directory %s is ignored: %w", CacheDirName, err)
-	}
-	if !ign {
-		return "", fmt.Errorf("please add /%s/ to .gitignore", CacheDirName)
-	}
-
-	cacheDirPath := filepath.Join(p.root, CacheDirName)
-	// create cache dir if it does not exist
-	if _, err := os.Stat(cacheDirPath); os.IsNotExist(err) {
-		err = os.Mkdir(cacheDirPath, os.ModeDir|os.ModePerm)
-		if err != nil {
-			return "", fmt.Errorf("failed to create cache directory %s with error: %s", cacheDirPath, err)
-		}
-	}
-	return cacheDirPath, nil
-}
-
-func (p *project) Config() Config {
-	return *p.config
-}
-
-func (p *project) Environment() Environment {
-	return *p.environment
-}
-
-func (p *project) Me() (*scim.User, error) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if p.me != nil {
-		return p.me, nil
-	}
-	me, err := p.wsc.CurrentUser.Me(context.Background())
-	if err != nil {
-		return nil, err
-	}
-	p.me = me
-	return me, nil
-}
-
-func (p *project) DeploymentIsolationPrefix() string {
-	if p.config.Isolation == None {
-		return p.config.Name
-	}
-	if p.config.Isolation == Soft {
-		me, err := p.Me()
-		if err != nil {
-			panic(err)
-		}
-		return fmt.Sprintf("%s/%s", p.config.Name, me.UserName)
-	}
-	panic(fmt.Errorf("unknow project isolation: %s", p.config.Isolation))
-}
-
-func getClusterIdFromClusterName(ctx context.Context,
-	wsc *databricks.WorkspaceClient,
-	clusterName string,
-) (clusterId string, err error) {
-	clusterInfo, err := wsc.Clusters.GetByClusterName(ctx, clusterName)
-	if err != nil {
-		return "", err
-	}
-	return clusterInfo.ClusterId, nil
-}
-
-// Old version of getting development cluster details with isolation implemented.
-// Kept just for reference. Remove once isolation is implemented properly
-/*
-func (p *project) DevelopmentCluster(ctx context.Context) (cluster clusters.ClusterInfo, err error) {
-	api := clusters.NewClustersAPI(ctx, p.Client()) // TODO: rewrite with normal SDK
-	if p.project.DevCluster == nil {
-		p.project.DevCluster = &clusters.Cluster{}
-	}
-	dc := p.project.DevCluster
-	if p.project.Isolation == Soft {
-		if p.project.IsDevClusterJustReference() {
-			err = fmt.Errorf("projects with soft isolation cannot have named clusters")
-			return
-		}
-		dc.ClusterName = fmt.Sprintf("dev/%s", p.DeploymentIsolationPrefix())
-	}
-	if dc.ClusterName == "" {
-		err = fmt.Errorf("please either pick `isolation: soft` or specify a shared cluster name")
-		return
-	}
-	return app.GetOrCreateRunningCluster(dc.ClusterName, *dc)
-}
-
-func runCommandOnDev(ctx context.Context, language, command string) common.CommandResults {
-	cluster, err := Current.DevelopmentCluster(ctx)
-	exec := Current.Client().CommandExecutor(ctx)
-	if err != nil {
-		return common.CommandResults{
-			ResultType: "error",
-			Summary:    err.Error(),
-		}
-	}
-	return exec.Execute(cluster.ClusterID, language, command)
-}
-
-func RunPythonOnDev(ctx context.Context, command string) common.CommandResults {
-	return runCommandOnDev(ctx, "python", command)
-}
-*/
-
-// TODO: Add safe access to p.project and p.project.DevCluster that throws errors if
-// the fields are not defined properly
-func (p *project) GetDevelopmentClusterId(ctx context.Context) (clusterId string, err error) {
-	clusterId = p.config.DevCluster.ClusterId
-	clusterName := p.config.DevCluster.ClusterName
-	if clusterId != "" {
-		return
-	} else if clusterName != "" {
-		// Add workspaces client on init
-		return getClusterIdFromClusterName(ctx, p.wsc, clusterName)
-	} else {
-		// TODO: Add the project config file location used to error message
-		err = fmt.Errorf("please define either development cluster's cluster_id or cluster_name in your project config")
-		return
-	}
-}
-
-func runCommandOnDev(ctx context.Context, language, command string) commands.Results {
-	clusterId, err := Get(ctx).GetDevelopmentClusterId(ctx)
-	if err != nil {
-		return commands.Results{
-			ResultType: "error",
-			Summary:    err.Error(),
-		}
-	}
-	return Get(ctx).wsc.CommandExecutor.Execute(ctx, clusterId, language, command)
-}
-
-func RunPythonOnDev(ctx context.Context, command string) commands.Results {
-	return runCommandOnDev(ctx, "python", command)
-}

project/project_test.go

@@ -1,117 +0,0 @@
-package project
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestProjectInitialize(t *testing.T) {
-	ctx, err := Initialize(context.Background(), "./testdata", DefaultEnvironment)
-	require.NoError(t, err)
-	assert.Equal(t, Get(ctx).config.Name, "dev")
-}
-
-func TestProjectInitializationCreatesGitIgnoreIfAbsent(t *testing.T) {
-	// create project root with databricks.yml
-	projectDir := t.TempDir()
-	f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
-	assert.NoError(t, err)
-	defer f1.Close()
-
-	ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment)
-	assert.NoError(t, err)
-
-	gitIgnorePath := filepath.Join(projectDir, ".gitignore")
-	assert.FileExists(t, gitIgnorePath)
-	fileBytes, err := os.ReadFile(gitIgnorePath)
-	assert.NoError(t, err)
-	assert.Contains(t, string(fileBytes), ".databricks")
-
-	prj := Get(ctx)
-	_, err = prj.CacheDir()
-	assert.NoError(t, err)
-}
-
-func TestProjectInitializationAddsCacheDirToGitIgnore(t *testing.T) {
-	// create project root with databricks.yml
-	projectDir := t.TempDir()
-	f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
-	assert.NoError(t, err)
-	f1.Close()
-
-	gitIgnorePath := filepath.Join(projectDir, ".gitignore")
-	f2, err := os.Create(gitIgnorePath)
-	assert.NoError(t, err)
-	f2.Close()
-
-	ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment)
-	assert.NoError(t, err)
-
-	fileBytes, err := os.ReadFile(gitIgnorePath)
-	assert.NoError(t, err)
-	assert.Contains(t, string(fileBytes), ".databricks")
-
-	// Muck with mtime of this file manually because in GitHub Actions runners the
-	// mtime isn't updated on write automatically (probably to save I/Os).
-	// We perform a reload of .gitignore files only if their mtime has changed.
-	// Add a minute to ensure it is different if the value is truncated to full seconds.
-	future := time.Now().Add(time.Minute)
-	err = os.Chtimes(gitIgnorePath, future, future)
-	require.NoError(t, err)
-
-	prj := Get(ctx)
-	_, err = prj.CacheDir()
-	assert.NoError(t, err)
-}
-
-func TestProjectInitializationDoesNotAddCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) {
-	// create project root with databricks.yml
-	projectDir := t.TempDir()
-	f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
-	assert.NoError(t, err)
-	f1.Close()
-
-	gitIgnorePath := filepath.Join(projectDir, ".gitignore")
-
-	err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644)
-	assert.NoError(t, err)
-
-	_, err = Initialize(context.Background(), projectDir, DefaultEnvironment)
-	assert.NoError(t, err)
-
-	fileBytes, err := os.ReadFile(gitIgnorePath)
-	assert.NoError(t, err)
-
-	assert.Equal(t, 1, strings.Count(string(fileBytes), ".databricks"))
-}
-
-func TestProjectCacheDir(t *testing.T) {
-	// create project root with databricks.yml
-	projectDir := t.TempDir()
-	f1, err := os.Create(filepath.Join(projectDir, "databricks.yml"))
-	assert.NoError(t, err)
-	f1.Close()
-
-	// create .gitignore with the .databricks dir in it
-	f2, err := os.Create(filepath.Join(projectDir, ".gitignore"))
-	assert.NoError(t, err)
-	content := []byte("/.databricks/")
-	_, err = f2.Write(content)
-	assert.NoError(t, err)
-	f2.Close()
-
-	ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment)
-	assert.NoError(t, err)
-
-	prj := Get(ctx)
-	cacheDir, err := prj.CacheDir()
-	assert.NoError(t, err)
-	assert.Equal(t, filepath.Join(projectDir, ".databricks"), cacheDir)
-}

project/root.go

@@ -1,37 +0,0 @@
-package project
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/databricks/bricks/folders"
-)
-
-const bricksRoot = "BRICKS_ROOT"
-
-// getRoot returns the project root.
-// If the `BRICKS_ROOT` environment variable is set, we assume its value
-// to be a valid project root. Otherwise we try to find it by traversing
-// the path and looking for a project configuration file.
-func getRoot() (string, error) {
-	path, ok := os.LookupEnv(bricksRoot)
-	if ok {
-		stat, err := os.Stat(path)
-		if err == nil && !stat.IsDir() {
-			err = fmt.Errorf("not a directory")
-		}
-		if err != nil {
-			return "", fmt.Errorf(`invalid project root %s="%s": %w`, bricksRoot, path, err)
-		}
-		return path, nil
-	}
-	wd, err := os.Getwd()
-	if err != nil {
-		return "", err
-	}
-	path, err = folders.FindDirWithLeaf(wd, ConfigFile)
-	if err != nil {
-		return "", fmt.Errorf(`unable to locate project root`)
-	}
-	return path, nil
-}

project/root_test.go

@@ -1,92 +0,0 @@
-package project
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-// Changes into specified directory for the duration of the test.
-// Returns the current working directory.
-func chdir(t *testing.T, dir string) string {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
-
-	abs, err := filepath.Abs(dir)
-	require.NoError(t, err)
-
-	err = os.Chdir(abs)
-	require.NoError(t, err)
-
-	t.Cleanup(func() {
-		err := os.Chdir(wd)
-		require.NoError(t, err)
-	})
-
-	return wd
-}
-
-func TestRootFromEnv(t *testing.T) {
-	dir := t.TempDir()
-	t.Setenv(bricksRoot, dir)
-
-	// It should pull the root from the environment variable.
-	root, err := getRoot()
-	require.NoError(t, err)
-	require.Equal(t, root, dir)
-}
-
-func TestRootFromEnvDoesntExist(t *testing.T) {
-	dir := t.TempDir()
-	t.Setenv(bricksRoot, filepath.Join(dir, "doesntexist"))
-
-	// It should pull the root from the environment variable.
-	_, err := getRoot()
-	require.Errorf(t, err, "invalid project root")
-}
-
-func TestRootFromEnvIsFile(t *testing.T) {
-	dir := t.TempDir()
-	f, err := os.Create(filepath.Join(dir, "invalid"))
-	require.NoError(t, err)
-	f.Close()
-	t.Setenv(bricksRoot, f.Name())
-
-	// It should pull the root from the environment variable.
-	_, err = getRoot()
-	require.Errorf(t, err, "invalid project root")
-}
-
-func TestRootIfEnvIsEmpty(t *testing.T) {
-	dir := ""
-	t.Setenv(bricksRoot, dir)
-
-	// It should pull the root from the environment variable.
-	_, err := getRoot()
-	require.Errorf(t, err, "invalid project root")
-}
-
-func TestRootLookup(t *testing.T) {
-	// Have to set then unset to allow the testing package to revert it to its original value.
-	t.Setenv(bricksRoot, "")
-	os.Unsetenv(bricksRoot)
-
-	// It should find the project root from $PWD.
-	wd := chdir(t, "./testdata/a/b/c")
-	root, err := getRoot()
-	require.NoError(t, err)
-	require.Equal(t, root, filepath.Join(wd, "testdata"))
-}
-
-func TestRootLookupError(t *testing.T) {
-	// Have to set then unset to allow the testing package to revert it to its original value.
-	t.Setenv(bricksRoot, "")
-	os.Unsetenv(bricksRoot)
-
-	// It can't find a project root from a temporary directory.
-	_ = chdir(t, t.TempDir())
-	_, err := getRoot()
-	require.ErrorContains(t, err, "unable to locate project root")
-}

project/testdata/.gitignore

@@ -1,2 +0,0 @@
-
-.databricks

project/testdata/databricks.yml

@@ -1,4 +0,0 @@
-name: dev
-profile: demo
-dev_cluster:
-  cluster_name: Shared Autoscaling

python/wheel.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/databricks/bricks/libs/log"
-	"github.com/databricks/bricks/project"
+	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/dbfs"
 )
 
@@ -63,7 +63,10 @@ func UploadWheelToDBFSWithPEP503(ctx context.Context, dir string) (string, error
 	// extra index URLs. See more pointers at https://stackoverflow.com/q/30889494/277035
 	dbfsLoc := fmt.Sprintf("%s/%s/%s", DBFSWheelLocation, dist.NormalizedName(), path.Base(wheel))
 
-	wsc := project.Get(ctx).WorkspacesClient()
+	wsc, err := databricks.NewWorkspaceClient(&databricks.Config{})
+	if err != nil {
+		return "", err
+	}
 	wf, err := os.Open(wheel)
 	if err != nil {
 		return "", err

terraform/runner.go

@@ -1,195 +0,0 @@
-/*
-How to simplify terraform configuration for the project?
----
-
-Solve the following adoption slowers:
-
-- remove the need for `required_providers` block
-- authenticate Databricks provider with the same DatabricksClient
-- skip downloading and locking Databricks provider every time (few seconds)
-- users won't have to copy-paste these into their configs:
-
-```hcl
-
-	terraform {
-	  required_providers {
-	    databricks = {
-	      source  = "databrickslabs/databricks"
-	    }
-	  }
-	}
-
-provider "databricks" {
-}
-```
-
-Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find
-details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short:
-
-  - init provider isntance
-  - start terraform plugin GRPC server
-  - "reattach" providers and specify the `tfexec.Reattach` options, which essentially
-    forward GRPC address to terraform subprocess.
-  - this can be done by either adding a source depenency on Databricks provider
-    or adding a special launch mode to it.
-
-For now
----
-Let's see how far we can get without GRPC magic.
-*/
-package terraform
-
-const DeploymentStateRemoteLocation = "dbfs:/FileStore/deployment-state"
-
-// type TerraformDeployer struct {
-// 	WorkDir string
-// 	CopyTfs bool
-
-// 	tf *tfexec.Terraform
-// }
-
-// func (d *TerraformDeployer) Init(ctx context.Context) error {
-// 	if d.CopyTfs {
-// 		panic("copying tf configuration files to a temporary dir not yet implemented")
-// 	}
-// 	// TODO: most likely merge the methods
-// 	exec, err := newTerraform(ctx, d.WorkDir, map[string]string{})
-// 	if err != nil {
-// 		return err
-// 	}
-// 	d.tf = exec
-// 	return nil
-// }
-
-// // returns location of terraform state on DBFS based on project's deployment isolation level.
-// func (d *TerraformDeployer) remoteTfstateLoc() string {
-// 	prefix := project.Current.DeploymentIsolationPrefix()
-// 	return fmt.Sprintf("%s/%s/terraform.tfstate", DeploymentStateRemoteLocation, prefix)
-// }
-
-// // returns structured representation of terraform state on DBFS.
-// func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) {
-// 	raw, err := utilities.ReadDbfsFile(ctx,
-// 		project.Current.WorkspacesClient(),
-// 		d.remoteTfstateLoc(),
-// 	)
-// 	if err != nil {
-// 		return nil, 0, err
-// 	}
-// 	return d.tfstateFromReader(bytes.NewBuffer(raw))
-// }
-
-// // opens file handle for local-backend terraform state, that has to be closed in the calling
-// // methods. this file alone is not the authoritative state of deployment and has to properly
-// // be synced with remote counterpart.
-// func (d *TerraformDeployer) openLocalState() (*os.File, error) {
-// 	return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir))
-// }
-
-// // returns structured representation of terraform state on local machine. as part of
-// // the optimistic concurrency control, please make sure to always compare the serial
-// // number of local and remote states before proceeding with deployment.
-// func (d *TerraformDeployer) localState() (*tfjson.State, int, error) {
-// 	local, err := d.openLocalState()
-// 	if err != nil {
-// 		return nil, 0, err
-// 	}
-// 	defer local.Close()
-// 	return d.tfstateFromReader(local)
-// }
-
-// // converts input stream into structured representation of terraform state and deployment
-// // serial number, that helps controlling versioning and synchronisation via optimistic locking.
-// func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) {
-// 	var state tfjson.State
-// 	state.UseJSONNumber(true)
-// 	decoder := json.NewDecoder(reader)
-// 	decoder.UseNumber()
-// 	err := decoder.Decode(&state)
-// 	if err != nil {
-// 		return nil, 0, err
-// 	}
-// 	err = state.Validate()
-// 	if err != nil {
-// 		return nil, 0, err
-// 	}
-// 	var serialWrapper struct {
-// 		Serial int `json:"serial,omitempty"`
-// 	}
-// 	// TODO: use byte buffer if this decoder fails on double reading
-// 	err = decoder.Decode(&serialWrapper)
-// 	if err != nil {
-// 		return nil, 0, err
-// 	}
-// 	return &state, serialWrapper.Serial, nil
-// }
-
-// // uploads terraform state from local directory to designated DBFS location.
-// func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error {
-// 	local, err := d.openLocalState()
-// 	if err != nil {
-// 		return err
-// 	}
-// 	defer local.Close()
-// 	raw, err := io.ReadAll(local)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	// TODO: make sure that deployment locks are implemented
-// 	return utilities.CreateDbfsFile(ctx,
-// 		project.Current.WorkspacesClient(),
-// 		d.remoteTfstateLoc(),
-// 		raw,
-// 		true,
-// 	)
-// }
-
-// // downloads terraform state from DBFS to local working directory.
-// func (d *TerraformDeployer) downloadTfstate(ctx context.Context) error {
-// 	remote, serialDeployed, err := d.remoteState(ctx)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	log.Debugf(ctx, "remote serial is %d", serialDeployed)
-// 	local, err := d.openLocalState()
-// 	if err != nil {
-// 		return err
-// 	}
-// 	defer local.Close()
-// 	raw, err := json.Marshal(remote)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	_, err = io.Copy(local, bytes.NewBuffer(raw))
-// 	return err
-// }
-
-// // installs terraform to a temporary directory (for now)
-// func installTerraform(ctx context.Context) (string, error) {
-// 	// TODO: let configuration and/or environment variable specify
-// 	// terraform binary. Or detect if terraform is installed in the $PATH
-// 	installer := &releases.ExactVersion{
-// 		Product: product.Terraform,
-// 		Version: version.Must(version.NewVersion("1.1.0")),
-// 	}
-// 	return installer.Install(ctx)
-// }
-
-// func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) {
-// 	execPath, err := installTerraform(ctx)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-// 	// TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users
-// 	// one of the options: take entire working directory with *.tf files and move them to tmpdir.
-// 	// make it optional, of course, otherwise debugging may become super hard.
-// 	tf, err := tfexec.NewTerraform(workDir, execPath)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-// 	err = tf.SetEnv(env)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-// 	return tf, err
-// }

terraform/runner_test.go

@@ -1,44 +0,0 @@
-package terraform
-
-// func TestSomething(t *testing.T) {
-// 	ctx := context.Background()
-// 	tf, err := newTerraform(ctx, "testdata/simplest", map[string]string{
-// 		"DATABRICKS_HOST":  "..",
-// 		"DATABRICKS_TOKEN": "..",
-// 	})
-// 	assert.NoError(t, err)
-
-// 	err = tf.Init(ctx)
-// 	assert.NoError(t, err)
-
-// 	planLoc := path.Join(t.TempDir(), "tfplan.zip")
-// 	hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc))
-// 	assert.True(t, hasChanges)
-// 	assert.NoError(t, err)
-
-// 	plan, err := tf.ShowPlanFile(ctx, planLoc)
-// 	assert.NoError(t, err)
-// 	assert.NotNil(t, plan)
-
-// 	found := false
-// 	for _, r := range plan.Config.RootModule.Resources {
-// 		// TODO: add validator to prevent non-Databricks resources in *.tf files, as
-// 		// we're not replacing terraform, we're wrapping it up for the average user.
-// 		if r.Type != "databricks_job" {
-// 			continue
-// 		}
-// 		// TODO: validate that libraries on jobs defined in *.tf and libraries
-// 		// in `install_requires` defined in setup.py are the same. Exist with
-// 		// the explanatory error otherwise.
-// 		found = true
-// 		// resource "databricks_job" "this"
-// 		assert.Equal(t, "this", r.Name)
-// 		// this is just a PoC to show how to retrieve DBR version from definitions.
-// 		// production code should perform rigorous nil checks...
-// 		nc := r.Expressions["new_cluster"]
-// 		firstBlock := nc.NestedBlocks[0]
-// 		ver := firstBlock["spark_version"].ConstantValue.(string)
-// 		assert.Equal(t, "10.0.1", ver)
-// 	}
-// 	assert.True(t, found)
-// }

terraform/testdata/simplest/demo.tf

@@ -1,41 +0,0 @@
-terraform {
-  required_providers {
-    databricks = {
-      source  = "databrickslabs/databricks"
-    }
-  }
-}
-
-provider "databricks" {
-}
-
-# This file cannot be used for tests until `InsecureSkipVerify` is exposed though env var
-# data "databricks_current_user" "me" {}
-# data "databricks_spark_version" "latest" {}
-# data "databricks_node_type" "smallest" {
-#   local_disk = true
-# }
-
-resource "databricks_notebook" "this" {
-  path     = "/Users/me@example.com/Terraform"
-  language = "PYTHON"
-  content_base64 = base64encode(<<-EOT
-    # created from ${abspath(path.module)}
-    display(spark.range(10))
-    EOT
-  )
-}
-
-resource "databricks_job" "this" {
-  name = "Terraform Demo (me@example.com)"
-
-  new_cluster {
-    num_workers   = 1
-    spark_version = "10.0.1"
-    node_type_id  = "i3.xlarge"
-  }
-
-  notebook_task {
-    notebook_path = databricks_notebook.this.path
-  }
-}