diff --git a/cmd/configure/configure.go b/cmd/configure/configure.go index ed1bae01..ebb653e0 100644 --- a/cmd/configure/configure.go +++ b/cmd/configure/configure.go @@ -7,9 +7,7 @@ import ( "os" "path/filepath" - "github.com/databricks/bricks/cmd/prompt" "github.com/databricks/bricks/cmd/root" - "github.com/databricks/bricks/project" "github.com/spf13/cobra" "gopkg.in/ini.v1" ) @@ -20,7 +18,7 @@ type Configs struct { Profile string `ini:"-"` } -var noInteractive, tokenMode bool +var tokenMode bool func (cfg *Configs) loadNonInteractive(cmd *cobra.Command) error { host, err := cmd.Flags().GetString("host") @@ -43,53 +41,6 @@ func (cfg *Configs) loadNonInteractive(cmd *cobra.Command) error { return nil } -func (cfg *Configs) loadInteractive(cmd *cobra.Command) error { - res := prompt.Results{} - questions := prompt.Questions{} - - host, err := cmd.Flags().GetString("host") - if err != nil || host == "" { - questions = append(questions, prompt.Text{ - Key: "host", - Label: "Databricks Host", - Default: func(res prompt.Results) string { - return cfg.Host - }, - Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) { - cfg.Host = ans.Value - }, - }) - } else { - cfg.Host = host - } - - if tokenMode { - questions = append(questions, prompt.Text{ - Key: "token", - Label: "Databricks Token", - Default: func(res prompt.Results) string { - return cfg.Token - }, - Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) { - cfg.Token = ans.Value - }, - }) - } - - err = questions.Ask(res) - if err != nil { - return err - } - - for _, answer := range res { - if answer.Callback != nil { - answer.Callback(answer, nil, res) - } - } - - return nil -} - var configureCmd = &cobra.Command{ Use: "configure", Short: "Configure authentication", @@ -135,11 +86,7 @@ var configureCmd = &cobra.Command{ return fmt.Errorf("unmarshal loaded config: %w", err) } - if noInteractive { - err = cfg.loadNonInteractive(cmd) - } else { - err = cfg.loadInteractive(cmd) - } + err = cfg.loadNonInteractive(cmd) if err != nil { return fmt.Errorf("reading configs: %w", err) } @@ -171,7 +118,6 @@ var configureCmd = &cobra.Command{ func init() { root.RootCmd.AddCommand(configureCmd) configureCmd.Flags().BoolVarP(&tokenMode, "token", "t", false, "Configure using Databricks Personal Access Token") - configureCmd.Flags().BoolVar(&noInteractive, "no-interactive", false, "Don't show interactive prompts for inputs. Read directly from stdin.") configureCmd.Flags().String("host", "", "Host to connect to.") configureCmd.Flags().String("profile", "DEFAULT", "CLI connection profile to use.") } diff --git a/cmd/configure/configure_test.go b/cmd/configure/configure_test.go index d168352c..34add438 100644 --- a/cmd/configure/configure_test.go +++ b/cmd/configure/configure_test.go @@ -52,7 +52,7 @@ func TestDefaultConfigureNoInteractive(t *testing.T) { }) os.Stdin = inp - root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host"}) + root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host"}) err := root.RootCmd.ExecuteContext(ctx) assert.NoError(t, err) @@ -84,7 +84,7 @@ func TestConfigFileFromEnvNoInteractive(t *testing.T) { t.Cleanup(func() { os.Stdin = oldStdin }) os.Stdin = inp - root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host"}) + root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host"}) err := root.RootCmd.ExecuteContext(ctx) assert.NoError(t, err) @@ -112,7 +112,7 @@ func TestCustomProfileConfigureNoInteractive(t *testing.T) { t.Cleanup(func() { os.Stdin = oldStdin }) os.Stdin = inp - root.RootCmd.SetArgs([]string{"configure", "--token", "--no-interactive", "--host", "host", "--profile", "CUSTOM"}) + root.RootCmd.SetArgs([]string{"configure", "--token", "--host", "host", "--profile", "CUSTOM"}) err := root.RootCmd.ExecuteContext(ctx) assert.NoError(t, err) diff --git a/cmd/fs/fs.go b/cmd/fs/fs.go index 600bfd9c..3aaca70f 100644 --- a/cmd/fs/fs.go +++ b/cmd/fs/fs.go @@ -14,14 +14,4 @@ var fsCmd = &cobra.Command{ func init() { root.RootCmd.AddCommand(fsCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // fsCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // fsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index 1f5e878b..1df30959 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -3,7 +3,6 @@ package fs import ( "fmt" - "github.com/databricks/bricks/project" "github.com/spf13/cobra" ) @@ -12,25 +11,12 @@ var lsCmd = &cobra.Command{ Use: "ls ", Short: "Lists files", Long: `Lists files`, - Args: cobra.ExactArgs(1), - PreRunE: project.Configure, - Run: func(cmd *cobra.Command, args []string) { - wsc := project.Get(cmd.Context()).WorkspacesClient() - listStatusResponse, err := wsc.Dbfs.ListByPath(cmd.Context(), args[0]) - if err != nil { - panic(err) - } - files := listStatusResponse.Files - // TODO: output formatting: JSON, CSV, tables and default - for _, v := range files { - fmt.Printf("[-] %s (%d, %v)\n", v.Path, v.FileSize, v.IsDir) - } + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("TODO") }, } func init() { - // TODO: pietern: conditionally register commands - // fabianj: don't do it fsCmd.AddCommand(lsCmd) } diff --git a/cmd/init/init.go b/cmd/init/init.go deleted file mode 100644 index ea67238d..00000000 --- a/cmd/init/init.go +++ /dev/null @@ -1,128 +0,0 @@ -package init - -import ( - "embed" - "fmt" - "os" - "path" - - "github.com/databricks/bricks/cmd/prompt" - "github.com/databricks/bricks/cmd/root" - "github.com/databricks/bricks/project" - "github.com/ghodss/yaml" - "github.com/spf13/cobra" -) - -//go:embed templates -var templates embed.FS - -// initCmd represents the init command -var initCmd = &cobra.Command{ - Use: "init", - Short: "Project starter templates", - Long: `Generate project templates`, - - RunE: func(cmd *cobra.Command, args []string) error { - if project.IsDatabricksProject() { - return fmt.Errorf("this path is already a Databricks project") - } - profileChoice, err := getConnectionProfile() - if err != nil { - return err - } - wd, _ := os.Getwd() - q := prompt.Questions{ - prompt.Text{ - Key: "name", - Label: "Project name", - Default: func(res prompt.Results) string { - return path.Base(wd) - }, - Callback: func(ans prompt.Answer, config *project.Config, res prompt.Results) { - config.Name = ans.Value - }, - }, - *profileChoice, - prompt.Choice{Key: "language", Label: "Project language", Answers: prompt.Answers{ - { - Value: "Python", - Details: "Machine learning and data engineering focused projects", - Callback: nil, - }, - { - Value: "Scala", - Details: "Data engineering focused projects with strong typing", - Callback: nil, - }, - }}, - prompt.Choice{Key: "isolation", Label: "Deployment isolation", Answers: prompt.Answers{ - { - Value: "None", - Details: "Use shared Databricks workspace resources for all project team members", - Callback: nil, - }, - { - Value: "Soft", - Details: "Prepend prefixes to each team member's deployment", - Callback: func( - ans prompt.Answer, config *project.Config, res prompt.Results) { - config.Isolation = project.Soft - }, - }, - }}, - // DBR selection - // Choice{"cloud", "Cloud", Answers{ - // {"AWS", "Amazon Web Services", nil}, - // {"Azure", "Microsoft Azure Cloud", nil}, - // {"GCP", "Google Cloud Platform", nil}, - // }}, - // Choice{"ci", "Continuous Integration", Answers{ - // {"None", "Do not create continuous integration configuration", nil}, - // {"GitHub Actions", "Create .github/workflows/push.yml configuration", nil}, - // {"Azure DevOps", "Create basic build and test pipelines", nil}, - // }}, - // Choice{"ide", "Integrated Development Environment", Answers{ - // {"None", "Do not create templates for IDE", nil}, - // {"VSCode", "Create .devcontainer and other useful things", nil}, - // {"PyCharm", "Create project conf and other things", nil}, - // }}, - } - res := prompt.Results{} - err = q.Ask(res) - if err != nil { - return err - } - var config project.Config - for _, ans := range res { - if ans.Callback == nil { - continue - } - ans.Callback(ans, &config, res) - } - raw, err := yaml.Marshal(config) - if err != nil { - return err - } - newConfig, err := os.Create(fmt.Sprintf("%s/%s", wd, project.ConfigFile)) - if err != nil { - return err - } - _, err = newConfig.Write(raw) - if err != nil { - return err - } - d, err := templates.ReadDir(".") - if err != nil { - return err - } - for _, v := range d { - cmd.Printf("template found: %v", v.Name()) - } - cmd.Print("Config initialized!") - return err - }, -} - -func init() { - root.RootCmd.AddCommand(initCmd) -} diff --git a/cmd/init/legacy-cli.go b/cmd/init/legacy-cli.go deleted file mode 100644 index 5391169a..00000000 --- a/cmd/init/legacy-cli.go +++ /dev/null @@ -1,57 +0,0 @@ -package init - -import ( - "fmt" - - "github.com/databricks/bricks/cmd/prompt" - "github.com/databricks/bricks/project" - "github.com/mitchellh/go-homedir" - "gopkg.in/ini.v1" -) - -func loadCliProfiles() (profiles []prompt.Answer, err error) { - file, err := homedir.Expand("~/.databrickscfg") - if err != nil { - return - } - gitConfig, err := ini.Load(file) - if err != nil { - return - } - for _, v := range gitConfig.Sections() { - host, err := v.GetKey("host") - if err != nil { - // invalid profile - continue - } - // TODO: verify these tokens to work, becaus they may be expired - profiles = append(profiles, prompt.Answer{ - Value: v.Name(), - Details: fmt.Sprintf(`Connecting to "%s" workspace`, host), - Callback: func(ans prompt.Answer, config *project.Config, _ prompt.Results) { - if config.Environments == nil { - config.Environments = make(map[string]project.Environment) - } - config.Environments[project.DefaultEnvironment] = project.Environment{ - Workspace: project.Workspace{ - Profile: ans.Value, - }, - } - }, - }) - } - return -} - -func getConnectionProfile() (*prompt.Choice, error) { - profiles, err := loadCliProfiles() - if err != nil { - return nil, err - } - // TODO: propmt for password and create ~/.databrickscfg - return &prompt.Choice{ - Key: "profile", - Label: "Databricks CLI profile", - Answers: profiles, - }, err -} diff --git a/cmd/init/templates/python-notebook/Hello.py b/cmd/init/templates/python-notebook/Hello.py deleted file mode 100644 index 1910e099..00000000 --- a/cmd/init/templates/python-notebook/Hello.py +++ /dev/null @@ -1,4 +0,0 @@ -# Databricks notebook source - -# this was automatically generated -display(spark.tables()) \ No newline at end of file diff --git a/cmd/prompt/prompt.go b/cmd/prompt/prompt.go deleted file mode 100644 index 428fb9d2..00000000 --- a/cmd/prompt/prompt.go +++ /dev/null @@ -1,89 +0,0 @@ -package prompt - -import ( - "fmt" - "io" - - "github.com/databricks/bricks/project" - "github.com/manifoldco/promptui" -) - -type Results map[string]Answer - -type Question interface { - Ask(res Results) (key string, ans Answer, err error) -} - -type Questions []Question - -func (qq Questions) Ask(res Results) error { - for _, v := range qq { - key, ans, err := v.Ask(res) - if err != nil { - return err - } - res[key] = ans - } - return nil -} - -type Text struct { - Key string - Label string - Default func(res Results) string - Callback AnswerCallback - Stdin io.ReadCloser -} - -func (t Text) Ask(res Results) (string, Answer, error) { - def := "" - if t.Default != nil { - def = t.Default(res) - } - v, err := (&promptui.Prompt{ - Label: t.Label, - Default: def, - Stdin: t.Stdin, - }).Run() - return t.Key, Answer{ - Value: v, - Callback: t.Callback, - }, err -} - -type Choice struct { - Key string - Label string - Answers []Answer - Stdin io.ReadCloser -} - -func (q Choice) Ask(res Results) (string, Answer, error) { - // TODO: validate and re-ask - prompt := promptui.Select{ - Label: q.Label, - Items: q.Answers, - Templates: &promptui.SelectTemplates{ - Label: `{{ .Value }}`, - Details: `{{ .Details | green }}`, - Selected: fmt.Sprintf(`{{ "%s" | faint }}: {{ .Value | bold }}`, q.Label), - }, - Stdin: q.Stdin, - } - i, _, err := prompt.Run() - return q.Key, q.Answers[i], err -} - -type Answers []Answer - -type AnswerCallback func(ans Answer, config *project.Config, res Results) - -type Answer struct { - Value string - Details string - Callback AnswerCallback -} - -func (a Answer) String() string { - return a.Value -} diff --git a/go.mod b/go.mod index ebb85223..5f4ed518 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,6 @@ require ( github.com/atotto/clipboard v0.1.4 github.com/databricks/databricks-sdk-go v0.7.0 github.com/ghodss/yaml v1.0.0 // MIT + NOTICE - github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause license - github.com/mitchellh/go-homedir v1.1.0 // MIT github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // BSD-2-Clause github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT github.com/spf13/cobra v1.7.0 // Apache 2.0 @@ -41,7 +39,6 @@ require ( require ( cloud.google.com/go/compute v1.19.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.3 // indirect diff --git a/go.sum b/go.sum index da9d63f5..9edd47c7 100644 --- a/go.sum +++ b/go.sum @@ -12,12 +12,6 @@ github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkE github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -89,15 +83,11 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= -github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= @@ -159,7 +149,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/main.go b/main.go index b0fc5d7e..3d76d32f 100644 --- a/main.go +++ b/main.go @@ -6,9 +6,7 @@ import ( _ "github.com/databricks/bricks/cmd/bundle" _ "github.com/databricks/bricks/cmd/bundle/debug" _ "github.com/databricks/bricks/cmd/bundle/debug/deploy" - _ "github.com/databricks/bricks/cmd/configure" _ "github.com/databricks/bricks/cmd/fs" - _ "github.com/databricks/bricks/cmd/init" "github.com/databricks/bricks/cmd/root" _ "github.com/databricks/bricks/cmd/sync" _ "github.com/databricks/bricks/cmd/version" diff --git a/project/README.md b/project/README.md deleted file mode 100644 index 1c39bbc7..00000000 --- a/project/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Project Configuration ---- - -_Good implicit defaults is better than explicit complex configuration._ - -Regardless of current working directory, `bricks` finds project root with `databricks.yml` file up the directory tree. Technically, there might be couple of different Databricks Projects in the same Git repository, but the recommended scenario is to have just one `databricks.yml` in the root of Git repo. \ No newline at end of file diff --git a/project/config.go b/project/config.go deleted file mode 100644 index f8df78dc..00000000 --- a/project/config.go +++ /dev/null @@ -1,137 +0,0 @@ -package project - -import ( - "errors" - "fmt" - "io" - "os" - "path/filepath" - "reflect" - - "github.com/databricks/bricks/folders" - "github.com/databricks/databricks-sdk-go/service/clusters" - - "github.com/ghodss/yaml" -) - -type Isolation string - -const ( - None Isolation = "" - Soft Isolation = "soft" -) - -// ConfigFile is the name of project configuration file -const ConfigFile = "databricks.yml" - -type Assertions struct { - Groups []string `json:"groups,omitempty"` - Secrets []string `json:"secrets,omitempty"` - ServicePrincipals []string `json:"service_principals,omitempty"` -} - -type Config struct { - Name string `json:"name"` // or do default from folder name?.. - Isolation Isolation `json:"isolation,omitempty"` - - // development-time vs deployment-time resources - DevCluster *clusters.ClusterInfo `json:"dev_cluster,omitempty"` - - // Assertions defines a list of configurations expected to be applied - // to the workspace by a higher-privileged user (or service principal) - // in order for the deploy command to work, as individual project teams - // in almost all the cases don’t have admin privileges on Databricks - // workspaces. - // - // This configuration simplifies the flexibility of individual project - // teams, make jobs deployment easier and portable across environments. - // This configuration block would contain the following entities to be - // created by administrator users or admin-level automation, like Terraform - // and/or SCIM provisioning. - Assertions *Assertions `json:"assertions,omitempty"` - - // Environments contain this project's defined environments. - // They can be used to differentiate settings and resources between - // development, staging, production, etc. - // If not specified, the code below initializes this field with a - // single default-initialized environment called "development". - Environments map[string]Environment `json:"environments"` -} - -func (c Config) IsDevClusterDefined() bool { - return reflect.ValueOf(c.DevCluster).IsZero() -} - -// IsDevClusterJustReference denotes reference-only clusters. -// This conflicts with Soft isolation. Happens for cost-restricted projects, -// where there's only a single Shared Autoscaling cluster per workspace and -// general users have no ability to create other iteractive clusters. -func (c *Config) IsDevClusterJustReference() bool { - if c.DevCluster.ClusterName == "" { - return false - } - return reflect.DeepEqual(c.DevCluster, &clusters.ClusterInfo{ - ClusterName: c.DevCluster.ClusterName, - }) -} - -// IsDatabricksProject returns true for folders with `databricks.yml` -// in the parent tree -func IsDatabricksProject() bool { - _, err := findProjectRoot() - return err == nil -} - -func loadProjectConf(root string) (c Config, err error) { - configFilePath := filepath.Join(root, ConfigFile) - - if _, err := os.Stat(configFilePath); errors.Is(err, os.ErrNotExist) { - baseDir := filepath.Base(root) - // If bricks config file is missing we assume the project root dir name - // as the name of the project - return validateAndApplyProjectDefaults(Config{Name: baseDir}) - } - - config, err := os.Open(configFilePath) - if err != nil { - return - } - defer config.Close() - raw, err := io.ReadAll(config) - if err != nil { - return - } - err = yaml.Unmarshal(raw, &c) - if err != nil { - return - } - return validateAndApplyProjectDefaults(c) -} - -func validateAndApplyProjectDefaults(c Config) (Config, error) { - // If no environments are specified, define default environment under default name. - if c.Environments == nil { - c.Environments = make(map[string]Environment) - c.Environments[DefaultEnvironment] = Environment{} - } - // defaultCluster := clusters.ClusterInfo{ - // NodeTypeID: "smallest", - // SparkVersion: "latest", - // AutoterminationMinutes: 30, - // } - return c, nil -} - -func findProjectRoot() (string, error) { - wd, err := os.Getwd() - if err != nil { - return "", err - } - dir, err := folders.FindDirWithLeaf(wd, ConfigFile) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return "", fmt.Errorf("cannot find %s anywhere", ConfigFile) - } - } - return dir, nil -} diff --git a/project/config_test.go b/project/config_test.go deleted file mode 100644 index 83b347da..00000000 --- a/project/config_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package project - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLoadProjectConf(t *testing.T) { - prj, err := loadProjectConf("./testdata") - assert.NoError(t, err) - assert.Equal(t, "dev", prj.Name) - assert.True(t, prj.IsDevClusterJustReference()) -} diff --git a/project/environment.go b/project/environment.go deleted file mode 100644 index 77af9a38..00000000 --- a/project/environment.go +++ /dev/null @@ -1,41 +0,0 @@ -package project - -import ( - "os" - - "github.com/spf13/cobra" -) - -const bricksEnv = "BRICKS_ENV" - -const DefaultEnvironment = "development" - -// Workspace defines configurables at the workspace level. -type Workspace struct { - Profile string `json:"profile,omitempty"` -} - -// Environment defines all configurables for a single environment. -type Environment struct { - Workspace Workspace `json:"workspace"` -} - -// getEnvironment returns the name of the environment to operate in. -func getEnvironment(cmd *cobra.Command) (value string) { - // The command line flag takes precedence. - flag := cmd.Flag("environment") - if flag != nil { - value = flag.Value.String() - if value != "" { - return - } - } - - // If it's not set, use the environment variable. - value = os.Getenv(bricksEnv) - if value != "" { - return - } - - return DefaultEnvironment -} diff --git a/project/environment_test.go b/project/environment_test.go deleted file mode 100644 index eb5c0187..00000000 --- a/project/environment_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package project - -import ( - "testing" - - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" -) - -func TestEnvironmentFromCommand(t *testing.T) { - var cmd cobra.Command - cmd.Flags().String("environment", "", "specify environment") - cmd.Flags().Set("environment", "env-from-arg") - t.Setenv(bricksEnv, "") - - value := getEnvironment(&cmd) - assert.Equal(t, "env-from-arg", value) -} - -func TestEnvironmentFromEnvironment(t *testing.T) { - var cmd cobra.Command - cmd.Flags().String("environment", "", "specify environment") - cmd.Flags().Set("environment", "") - t.Setenv(bricksEnv, "env-from-env") - - value := getEnvironment(&cmd) - assert.Equal(t, "env-from-env", value) -} - -func TestEnvironmentDefault(t *testing.T) { - var cmd cobra.Command - cmd.Flags().String("environment", "", "specify environment") - cmd.Flags().Set("environment", "") - t.Setenv(bricksEnv, "") - - value := getEnvironment(&cmd) - assert.Equal(t, DefaultEnvironment, value) -} diff --git a/project/flavor.go b/project/flavor.go deleted file mode 100644 index 4c859926..00000000 --- a/project/flavor.go +++ /dev/null @@ -1,81 +0,0 @@ -package project - -// type Flavor interface { -// // Name returns a tuple of flavor key and readable name -// Name() (string, string) - -// // Detected returns true on successful metadata checks -// Detected() bool - -// // Build triggers packaging subprocesses -// Build(context.Context) error -// // TODO: Init() Questions -// // TODO: Deploy(context.Context) error -// } - -// var _ Flavor = PythonWheel{} - -// type PythonWheel struct{} - -// func (pw PythonWheel) Name() (string, string) { -// return "wheel", "Python Wheel" -// } - -// func (pw PythonWheel) Detected() bool { -// root, err := findProjectRoot() -// if err != nil { -// return false -// } -// _, err = os.Stat(fmt.Sprintf("%s/setup.py", root)) -// return err == nil -// } - -// func (pw PythonWheel) Build(ctx context.Context) error { -// defer toTheRootAndBack()() -// // do subprocesses or https://github.com/go-python/cpy3 -// // it all depends on complexity and binary size -// // TODO: detect if there's an .venv here and call setup.py with ENV vars of it -// // TODO: where.exe python (WIN) / which python (UNIX) -// cmd := exec.CommandContext(ctx, "python", "setup.py", "bdist-wheel") -// err := cmd.Run() -// if err != nil { -// return err -// } -// return nil -// } - -// func toTheRootAndBack() func() { -// wd, _ := os.Getwd() -// root, _ := findProjectRoot() -// os.Chdir(root) -// return func() { -// os.Chdir(wd) -// } -// } - -// var _ Flavor = PythonNotebooks{} - -// type PythonNotebooks struct{} - -// func (n PythonNotebooks) Name() (string, string) { -// // or just "notebooks", as we might shuffle in scala?... -// return "python-notebooks", "Python Notebooks" -// } - -// func (n PythonNotebooks) Detected() bool { -// // TODO: Steps: -// // - get all filenames -// // - read first X bytes from random 10 files and check -// // if they're "Databricks Notebook Source" -// return false -// } - -// func (n PythonNotebooks) Build(ctx context.Context) error { -// // TODO: perhaps some linting?.. -// return nil -// } - -// func (n PythonNotebooks) Deploy(ctx context.Context) error { -// // TODO: recursively upload notebooks to a given workspace path -// return nil -// } diff --git a/project/project.go b/project/project.go deleted file mode 100644 index 585b15a8..00000000 --- a/project/project.go +++ /dev/null @@ -1,271 +0,0 @@ -package project - -import ( - "context" - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/databricks/bricks/libs/git" - "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/service/commands" - "github.com/databricks/databricks-sdk-go/service/scim" - "github.com/spf13/cobra" -) - -const CacheDirName = ".databricks" - -type project struct { - mu sync.Mutex - - root string - env string - - config *Config - environment *Environment - wsc *databricks.WorkspaceClient - me *scim.User - fileSet *git.FileSet -} - -// Configure is used as a PreRunE function for all commands that -// require a project to be configured. If a project could successfully -// be found and loaded, it is set on the command's context object. -func Configure(cmd *cobra.Command, args []string) error { - root, err := getRoot() - if err != nil { - return err - } - - ctx, err := Initialize(cmd.Context(), root, getEnvironment(cmd)) - if err != nil { - return err - } - - cmd.SetContext(ctx) - return nil -} - -// Placeholder to use as unique key in context.Context. -var projectKey int - -// Initialize loads a project configuration given a root and environment. -// It stores the project on a new context. -// The project is available through the `Get()` function. -func Initialize(ctx context.Context, root, env string) (context.Context, error) { - config, err := loadProjectConf(root) - if err != nil { - return nil, err - } - - // Confirm that the specified environment is valid. - environment, ok := config.Environments[env] - if !ok { - return nil, fmt.Errorf("environment [%s] not defined", env) - } - - fileSet, err := git.NewFileSet(root) - if err != nil { - return nil, err - } - err = fileSet.EnsureValidGitIgnoreExists() - if err != nil { - return nil, err - } - - p := project{ - root: root, - env: env, - - config: &config, - environment: &environment, - fileSet: fileSet, - } - - p.initializeWorkspacesClient(ctx) - return context.WithValue(ctx, &projectKey, &p), nil -} - -func (p *project) initializeWorkspacesClient(ctx context.Context) { - var config databricks.Config - - // If the config specifies a profile, or other authentication related properties, - // pass them along to the SDK here. If nothing is defined, the SDK will figure - // out which autentication mechanism to use using enviroment variables. - if p.environment.Workspace.Profile != "" { - config.Profile = p.environment.Workspace.Profile - } - - p.wsc = databricks.Must(databricks.NewWorkspaceClient(&config)) -} - -// Get returns the project as configured on the context. -// It panics if it isn't configured. -func Get(ctx context.Context) *project { - project, ok := ctx.Value(&projectKey).(*project) - if !ok { - panic(`context not configured with project`) - } - return project -} - -// Make sure to initialize the workspaces client on project init -func (p *project) WorkspacesClient() *databricks.WorkspaceClient { - return p.wsc -} - -func (p *project) Root() string { - return p.root -} - -func (p *project) GetFileSet() *git.FileSet { - return p.fileSet -} - -// This cache dir will contain any state, state overrides (per user overrides -// to the project config) or any generated artifacts (eg: sync snapshots) -// that should never be checked into Git. -// -// We enfore that cache dir (.databricks) is added to .gitignore -// because it contains per-user overrides that we do not want users to -// accidentally check into git -func (p *project) CacheDir() (string, error) { - // assert cache dir is present in git ignore - ign, err := p.fileSet.IgnoreDirectory(fmt.Sprintf("/%s/", CacheDirName)) - if err != nil { - return "", fmt.Errorf("failed to check if directory %s is ignored: %w", CacheDirName, err) - } - if !ign { - return "", fmt.Errorf("please add /%s/ to .gitignore", CacheDirName) - } - - cacheDirPath := filepath.Join(p.root, CacheDirName) - // create cache dir if it does not exist - if _, err := os.Stat(cacheDirPath); os.IsNotExist(err) { - err = os.Mkdir(cacheDirPath, os.ModeDir|os.ModePerm) - if err != nil { - return "", fmt.Errorf("failed to create cache directory %s with error: %s", cacheDirPath, err) - } - } - return cacheDirPath, nil -} - -func (p *project) Config() Config { - return *p.config -} - -func (p *project) Environment() Environment { - return *p.environment -} - -func (p *project) Me() (*scim.User, error) { - p.mu.Lock() - defer p.mu.Unlock() - if p.me != nil { - return p.me, nil - } - me, err := p.wsc.CurrentUser.Me(context.Background()) - if err != nil { - return nil, err - } - p.me = me - return me, nil -} - -func (p *project) DeploymentIsolationPrefix() string { - if p.config.Isolation == None { - return p.config.Name - } - if p.config.Isolation == Soft { - me, err := p.Me() - if err != nil { - panic(err) - } - return fmt.Sprintf("%s/%s", p.config.Name, me.UserName) - } - panic(fmt.Errorf("unknow project isolation: %s", p.config.Isolation)) -} - -func getClusterIdFromClusterName(ctx context.Context, - wsc *databricks.WorkspaceClient, - clusterName string, -) (clusterId string, err error) { - clusterInfo, err := wsc.Clusters.GetByClusterName(ctx, clusterName) - if err != nil { - return "", err - } - return clusterInfo.ClusterId, nil -} - -// Old version of getting development cluster details with isolation implemented. -// Kept just for reference. Remove once isolation is implemented properly -/* -func (p *project) DevelopmentCluster(ctx context.Context) (cluster clusters.ClusterInfo, err error) { - api := clusters.NewClustersAPI(ctx, p.Client()) // TODO: rewrite with normal SDK - if p.project.DevCluster == nil { - p.project.DevCluster = &clusters.Cluster{} - } - dc := p.project.DevCluster - if p.project.Isolation == Soft { - if p.project.IsDevClusterJustReference() { - err = fmt.Errorf("projects with soft isolation cannot have named clusters") - return - } - dc.ClusterName = fmt.Sprintf("dev/%s", p.DeploymentIsolationPrefix()) - } - if dc.ClusterName == "" { - err = fmt.Errorf("please either pick `isolation: soft` or specify a shared cluster name") - return - } - return app.GetOrCreateRunningCluster(dc.ClusterName, *dc) -} - -func runCommandOnDev(ctx context.Context, language, command string) common.CommandResults { - cluster, err := Current.DevelopmentCluster(ctx) - exec := Current.Client().CommandExecutor(ctx) - if err != nil { - return common.CommandResults{ - ResultType: "error", - Summary: err.Error(), - } - } - return exec.Execute(cluster.ClusterID, language, command) -} - -func RunPythonOnDev(ctx context.Context, command string) common.CommandResults { - return runCommandOnDev(ctx, "python", command) -} -*/ - -// TODO: Add safe access to p.project and p.project.DevCluster that throws errors if -// the fields are not defined properly -func (p *project) GetDevelopmentClusterId(ctx context.Context) (clusterId string, err error) { - clusterId = p.config.DevCluster.ClusterId - clusterName := p.config.DevCluster.ClusterName - if clusterId != "" { - return - } else if clusterName != "" { - // Add workspaces client on init - return getClusterIdFromClusterName(ctx, p.wsc, clusterName) - } else { - // TODO: Add the project config file location used to error message - err = fmt.Errorf("please define either development cluster's cluster_id or cluster_name in your project config") - return - } -} - -func runCommandOnDev(ctx context.Context, language, command string) commands.Results { - clusterId, err := Get(ctx).GetDevelopmentClusterId(ctx) - if err != nil { - return commands.Results{ - ResultType: "error", - Summary: err.Error(), - } - } - return Get(ctx).wsc.CommandExecutor.Execute(ctx, clusterId, language, command) -} - -func RunPythonOnDev(ctx context.Context, command string) commands.Results { - return runCommandOnDev(ctx, "python", command) -} diff --git a/project/project_test.go b/project/project_test.go deleted file mode 100644 index 1c575030..00000000 --- a/project/project_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package project - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestProjectInitialize(t *testing.T) { - ctx, err := Initialize(context.Background(), "./testdata", DefaultEnvironment) - require.NoError(t, err) - assert.Equal(t, Get(ctx).config.Name, "dev") -} - -func TestProjectInitializationCreatesGitIgnoreIfAbsent(t *testing.T) { - // create project root with databricks.yml - projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) - assert.NoError(t, err) - defer f1.Close() - - ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment) - assert.NoError(t, err) - - gitIgnorePath := filepath.Join(projectDir, ".gitignore") - assert.FileExists(t, gitIgnorePath) - fileBytes, err := os.ReadFile(gitIgnorePath) - assert.NoError(t, err) - assert.Contains(t, string(fileBytes), ".databricks") - - prj := Get(ctx) - _, err = prj.CacheDir() - assert.NoError(t, err) -} - -func TestProjectInitializationAddsCacheDirToGitIgnore(t *testing.T) { - // create project root with databricks.yml - projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) - assert.NoError(t, err) - f1.Close() - - gitIgnorePath := filepath.Join(projectDir, ".gitignore") - f2, err := os.Create(gitIgnorePath) - assert.NoError(t, err) - f2.Close() - - ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment) - assert.NoError(t, err) - - fileBytes, err := os.ReadFile(gitIgnorePath) - assert.NoError(t, err) - assert.Contains(t, string(fileBytes), ".databricks") - - // Muck with mtime of this file manually because in GitHub Actions runners the - // mtime isn't updated on write automatically (probably to save I/Os). - // We perform a reload of .gitignore files only if their mtime has changed. - // Add a minute to ensure it is different if the value is truncated to full seconds. - future := time.Now().Add(time.Minute) - err = os.Chtimes(gitIgnorePath, future, future) - require.NoError(t, err) - - prj := Get(ctx) - _, err = prj.CacheDir() - assert.NoError(t, err) -} - -func TestProjectInitializationDoesNotAddCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) { - // create project root with databricks.yml - projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) - assert.NoError(t, err) - f1.Close() - - gitIgnorePath := filepath.Join(projectDir, ".gitignore") - - err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644) - assert.NoError(t, err) - - _, err = Initialize(context.Background(), projectDir, DefaultEnvironment) - assert.NoError(t, err) - - fileBytes, err := os.ReadFile(gitIgnorePath) - assert.NoError(t, err) - - assert.Equal(t, 1, strings.Count(string(fileBytes), ".databricks")) -} - -func TestProjectCacheDir(t *testing.T) { - // create project root with databricks.yml - projectDir := t.TempDir() - f1, err := os.Create(filepath.Join(projectDir, "databricks.yml")) - assert.NoError(t, err) - f1.Close() - - // create .gitignore with the .databricks dir in it - f2, err := os.Create(filepath.Join(projectDir, ".gitignore")) - assert.NoError(t, err) - content := []byte("/.databricks/") - _, err = f2.Write(content) - assert.NoError(t, err) - f2.Close() - - ctx, err := Initialize(context.Background(), projectDir, DefaultEnvironment) - assert.NoError(t, err) - - prj := Get(ctx) - cacheDir, err := prj.CacheDir() - assert.NoError(t, err) - assert.Equal(t, filepath.Join(projectDir, ".databricks"), cacheDir) -} diff --git a/project/root.go b/project/root.go deleted file mode 100644 index 326ddf4b..00000000 --- a/project/root.go +++ /dev/null @@ -1,37 +0,0 @@ -package project - -import ( - "fmt" - "os" - - "github.com/databricks/bricks/folders" -) - -const bricksRoot = "BRICKS_ROOT" - -// getRoot returns the project root. -// If the `BRICKS_ROOT` environment variable is set, we assume its value -// to be a valid project root. Otherwise we try to find it by traversing -// the path and looking for a project configuration file. -func getRoot() (string, error) { - path, ok := os.LookupEnv(bricksRoot) - if ok { - stat, err := os.Stat(path) - if err == nil && !stat.IsDir() { - err = fmt.Errorf("not a directory") - } - if err != nil { - return "", fmt.Errorf(`invalid project root %s="%s": %w`, bricksRoot, path, err) - } - return path, nil - } - wd, err := os.Getwd() - if err != nil { - return "", err - } - path, err = folders.FindDirWithLeaf(wd, ConfigFile) - if err != nil { - return "", fmt.Errorf(`unable to locate project root`) - } - return path, nil -} diff --git a/project/root_test.go b/project/root_test.go deleted file mode 100644 index 9fc87ea9..00000000 --- a/project/root_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package project - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// Changes into specified directory for the duration of the test. -// Returns the current working directory. -func chdir(t *testing.T, dir string) string { - wd, err := os.Getwd() - require.NoError(t, err) - - abs, err := filepath.Abs(dir) - require.NoError(t, err) - - err = os.Chdir(abs) - require.NoError(t, err) - - t.Cleanup(func() { - err := os.Chdir(wd) - require.NoError(t, err) - }) - - return wd -} - -func TestRootFromEnv(t *testing.T) { - dir := t.TempDir() - t.Setenv(bricksRoot, dir) - - // It should pull the root from the environment variable. - root, err := getRoot() - require.NoError(t, err) - require.Equal(t, root, dir) -} - -func TestRootFromEnvDoesntExist(t *testing.T) { - dir := t.TempDir() - t.Setenv(bricksRoot, filepath.Join(dir, "doesntexist")) - - // It should pull the root from the environment variable. - _, err := getRoot() - require.Errorf(t, err, "invalid project root") -} - -func TestRootFromEnvIsFile(t *testing.T) { - dir := t.TempDir() - f, err := os.Create(filepath.Join(dir, "invalid")) - require.NoError(t, err) - f.Close() - t.Setenv(bricksRoot, f.Name()) - - // It should pull the root from the environment variable. - _, err = getRoot() - require.Errorf(t, err, "invalid project root") -} - -func TestRootIfEnvIsEmpty(t *testing.T) { - dir := "" - t.Setenv(bricksRoot, dir) - - // It should pull the root from the environment variable. - _, err := getRoot() - require.Errorf(t, err, "invalid project root") -} - -func TestRootLookup(t *testing.T) { - // Have to set then unset to allow the testing package to revert it to its original value. - t.Setenv(bricksRoot, "") - os.Unsetenv(bricksRoot) - - // It should find the project root from $PWD. - wd := chdir(t, "./testdata/a/b/c") - root, err := getRoot() - require.NoError(t, err) - require.Equal(t, root, filepath.Join(wd, "testdata")) -} - -func TestRootLookupError(t *testing.T) { - // Have to set then unset to allow the testing package to revert it to its original value. - t.Setenv(bricksRoot, "") - os.Unsetenv(bricksRoot) - - // It can't find a project root from a temporary directory. - _ = chdir(t, t.TempDir()) - _, err := getRoot() - require.ErrorContains(t, err, "unable to locate project root") -} diff --git a/project/testdata/.gitignore b/project/testdata/.gitignore deleted file mode 100644 index de811f11..00000000 --- a/project/testdata/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ - -.databricks diff --git a/project/testdata/a/b/c/.gitkeep b/project/testdata/a/b/c/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/project/testdata/c/__init__.py b/project/testdata/c/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/project/testdata/databricks.yml b/project/testdata/databricks.yml deleted file mode 100644 index 2b286d17..00000000 --- a/project/testdata/databricks.yml +++ /dev/null @@ -1,4 +0,0 @@ -name: dev -profile: demo -dev_cluster: - cluster_name: Shared Autoscaling diff --git a/python/wheel.go b/python/wheel.go index 987a536b..76672ee4 100644 --- a/python/wheel.go +++ b/python/wheel.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/databricks/bricks/libs/log" - "github.com/databricks/bricks/project" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/dbfs" ) @@ -63,7 +63,10 @@ func UploadWheelToDBFSWithPEP503(ctx context.Context, dir string) (string, error // extra index URLs. See more pointers at https://stackoverflow.com/q/30889494/277035 dbfsLoc := fmt.Sprintf("%s/%s/%s", DBFSWheelLocation, dist.NormalizedName(), path.Base(wheel)) - wsc := project.Get(ctx).WorkspacesClient() + wsc, err := databricks.NewWorkspaceClient(&databricks.Config{}) + if err != nil { + return "", err + } wf, err := os.Open(wheel) if err != nil { return "", err diff --git a/terraform/runner.go b/terraform/runner.go deleted file mode 100644 index a666cbbe..00000000 --- a/terraform/runner.go +++ /dev/null @@ -1,195 +0,0 @@ -/* -How to simplify terraform configuration for the project? ---- - -Solve the following adoption slowers: - -- remove the need for `required_providers` block -- authenticate Databricks provider with the same DatabricksClient -- skip downloading and locking Databricks provider every time (few seconds) -- users won't have to copy-paste these into their configs: - -```hcl - - terraform { - required_providers { - databricks = { - source = "databrickslabs/databricks" - } - } - } - -provider "databricks" { -} -``` - -Terraform Plugin SDK v2 is using similar techniques for testing providers. One may find -details in github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource/plugin.go. In short: - - - init provider isntance - - start terraform plugin GRPC server - - "reattach" providers and specify the `tfexec.Reattach` options, which essentially - forward GRPC address to terraform subprocess. - - this can be done by either adding a source depenency on Databricks provider - or adding a special launch mode to it. - -For now ---- -Let's see how far we can get without GRPC magic. -*/ -package terraform - -const DeploymentStateRemoteLocation = "dbfs:/FileStore/deployment-state" - -// type TerraformDeployer struct { -// WorkDir string -// CopyTfs bool - -// tf *tfexec.Terraform -// } - -// func (d *TerraformDeployer) Init(ctx context.Context) error { -// if d.CopyTfs { -// panic("copying tf configuration files to a temporary dir not yet implemented") -// } -// // TODO: most likely merge the methods -// exec, err := newTerraform(ctx, d.WorkDir, map[string]string{}) -// if err != nil { -// return err -// } -// d.tf = exec -// return nil -// } - -// // returns location of terraform state on DBFS based on project's deployment isolation level. -// func (d *TerraformDeployer) remoteTfstateLoc() string { -// prefix := project.Current.DeploymentIsolationPrefix() -// return fmt.Sprintf("%s/%s/terraform.tfstate", DeploymentStateRemoteLocation, prefix) -// } - -// // returns structured representation of terraform state on DBFS. -// func (d *TerraformDeployer) remoteState(ctx context.Context) (*tfjson.State, int, error) { -// raw, err := utilities.ReadDbfsFile(ctx, -// project.Current.WorkspacesClient(), -// d.remoteTfstateLoc(), -// ) -// if err != nil { -// return nil, 0, err -// } -// return d.tfstateFromReader(bytes.NewBuffer(raw)) -// } - -// // opens file handle for local-backend terraform state, that has to be closed in the calling -// // methods. this file alone is not the authoritative state of deployment and has to properly -// // be synced with remote counterpart. -// func (d *TerraformDeployer) openLocalState() (*os.File, error) { -// return os.Open(fmt.Sprintf("%s/terraform.tfstate", d.WorkDir)) -// } - -// // returns structured representation of terraform state on local machine. as part of -// // the optimistic concurrency control, please make sure to always compare the serial -// // number of local and remote states before proceeding with deployment. -// func (d *TerraformDeployer) localState() (*tfjson.State, int, error) { -// local, err := d.openLocalState() -// if err != nil { -// return nil, 0, err -// } -// defer local.Close() -// return d.tfstateFromReader(local) -// } - -// // converts input stream into structured representation of terraform state and deployment -// // serial number, that helps controlling versioning and synchronisation via optimistic locking. -// func (d *TerraformDeployer) tfstateFromReader(reader io.Reader) (*tfjson.State, int, error) { -// var state tfjson.State -// state.UseJSONNumber(true) -// decoder := json.NewDecoder(reader) -// decoder.UseNumber() -// err := decoder.Decode(&state) -// if err != nil { -// return nil, 0, err -// } -// err = state.Validate() -// if err != nil { -// return nil, 0, err -// } -// var serialWrapper struct { -// Serial int `json:"serial,omitempty"` -// } -// // TODO: use byte buffer if this decoder fails on double reading -// err = decoder.Decode(&serialWrapper) -// if err != nil { -// return nil, 0, err -// } -// return &state, serialWrapper.Serial, nil -// } - -// // uploads terraform state from local directory to designated DBFS location. -// func (d *TerraformDeployer) uploadTfstate(ctx context.Context) error { -// local, err := d.openLocalState() -// if err != nil { -// return err -// } -// defer local.Close() -// raw, err := io.ReadAll(local) -// if err != nil { -// return err -// } -// // TODO: make sure that deployment locks are implemented -// return utilities.CreateDbfsFile(ctx, -// project.Current.WorkspacesClient(), -// d.remoteTfstateLoc(), -// raw, -// true, -// ) -// } - -// // downloads terraform state from DBFS to local working directory. -// func (d *TerraformDeployer) downloadTfstate(ctx context.Context) error { -// remote, serialDeployed, err := d.remoteState(ctx) -// if err != nil { -// return err -// } -// log.Debugf(ctx, "remote serial is %d", serialDeployed) -// local, err := d.openLocalState() -// if err != nil { -// return err -// } -// defer local.Close() -// raw, err := json.Marshal(remote) -// if err != nil { -// return err -// } -// _, err = io.Copy(local, bytes.NewBuffer(raw)) -// return err -// } - -// // installs terraform to a temporary directory (for now) -// func installTerraform(ctx context.Context) (string, error) { -// // TODO: let configuration and/or environment variable specify -// // terraform binary. Or detect if terraform is installed in the $PATH -// installer := &releases.ExactVersion{ -// Product: product.Terraform, -// Version: version.Must(version.NewVersion("1.1.0")), -// } -// return installer.Install(ctx) -// } - -// func newTerraform(ctx context.Context, workDir string, env map[string]string) (*tfexec.Terraform, error) { -// execPath, err := installTerraform(ctx) -// if err != nil { -// return nil, err -// } -// // TODO: figure out how to cleanup/skip `.terraform*` files and dirs, not to confuse users -// // one of the options: take entire working directory with *.tf files and move them to tmpdir. -// // make it optional, of course, otherwise debugging may become super hard. -// tf, err := tfexec.NewTerraform(workDir, execPath) -// if err != nil { -// return nil, err -// } -// err = tf.SetEnv(env) -// if err != nil { -// return nil, err -// } -// return tf, err -// } diff --git a/terraform/runner_test.go b/terraform/runner_test.go deleted file mode 100644 index af89cbf2..00000000 --- a/terraform/runner_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package terraform - -// func TestSomething(t *testing.T) { -// ctx := context.Background() -// tf, err := newTerraform(ctx, "testdata/simplest", map[string]string{ -// "DATABRICKS_HOST": "..", -// "DATABRICKS_TOKEN": "..", -// }) -// assert.NoError(t, err) - -// err = tf.Init(ctx) -// assert.NoError(t, err) - -// planLoc := path.Join(t.TempDir(), "tfplan.zip") -// hasChanges, err := tf.Plan(ctx, tfexec.Out(planLoc)) -// assert.True(t, hasChanges) -// assert.NoError(t, err) - -// plan, err := tf.ShowPlanFile(ctx, planLoc) -// assert.NoError(t, err) -// assert.NotNil(t, plan) - -// found := false -// for _, r := range plan.Config.RootModule.Resources { -// // TODO: add validator to prevent non-Databricks resources in *.tf files, as -// // we're not replacing terraform, we're wrapping it up for the average user. -// if r.Type != "databricks_job" { -// continue -// } -// // TODO: validate that libraries on jobs defined in *.tf and libraries -// // in `install_requires` defined in setup.py are the same. Exist with -// // the explanatory error otherwise. -// found = true -// // resource "databricks_job" "this" -// assert.Equal(t, "this", r.Name) -// // this is just a PoC to show how to retrieve DBR version from definitions. -// // production code should perform rigorous nil checks... -// nc := r.Expressions["new_cluster"] -// firstBlock := nc.NestedBlocks[0] -// ver := firstBlock["spark_version"].ConstantValue.(string) -// assert.Equal(t, "10.0.1", ver) -// } -// assert.True(t, found) -// } diff --git a/terraform/testdata/simplest/demo.tf b/terraform/testdata/simplest/demo.tf deleted file mode 100644 index f4a8db57..00000000 --- a/terraform/testdata/simplest/demo.tf +++ /dev/null @@ -1,41 +0,0 @@ -terraform { - required_providers { - databricks = { - source = "databrickslabs/databricks" - } - } -} - -provider "databricks" { -} - -# This file cannot be used for tests until `InsecureSkipVerify` is exposed though env var -# data "databricks_current_user" "me" {} -# data "databricks_spark_version" "latest" {} -# data "databricks_node_type" "smallest" { -# local_disk = true -# } - -resource "databricks_notebook" "this" { - path = "/Users/me@example.com/Terraform" - language = "PYTHON" - content_base64 = base64encode(<<-EOT - # created from ${abspath(path.module)} - display(spark.range(10)) - EOT - ) -} - -resource "databricks_job" "this" { - name = "Terraform Demo (me@example.com)" - - new_cluster { - num_workers = 1 - spark_version = "10.0.1" - node_type_id = "i3.xlarge" - } - - notebook_task { - notebook_path = databricks_notebook.this.path - } -} \ No newline at end of file