From 2dab552829a1b4260598ac82655c0df0efedc32c Mon Sep 17 00:00:00 2001 From: Serge Smertin Date: Fri, 13 May 2022 17:43:54 +0200 Subject: [PATCH] Added `launch` command and release pipeline --- .github/workflows/release.yml | 31 +++++++++++++++++++++++ .gitignore | 1 + .goreleaser | 47 +++++++++++++++++++++++++++++++++++ README.md | 25 ++++++++++++++++--- cmd/launch.go | 33 ++++++++++++++++++++++++ cmd/root.go | 40 ++++++++++++++++------------- cmd/test.go | 2 +- go.mod | 1 - test.py | 1 + 9 files changed, 159 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser create mode 100644 cmd/launch.go create mode 100644 test.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..4b79711a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: release +on: + push: + tags: + - "v*" +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Unshallow + run: git fetch --prune --unshallow + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + # - name: Import GPG key + # id: import_gpg + # uses: hashicorp/ghaction-import-gpg@v2.1.0 + # env: + # GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + # PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + # GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 21ce9dec..7d19b3bf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ bricks # Dependency directories (remove the comment below to include it) vendor/ +dist/ \ No newline at end of file diff --git a/.goreleaser b/.goreleaser new file mode 100644 index 00000000..bcf16e53 --- /dev/null +++ b/.goreleaser @@ -0,0 +1,47 @@ + +hooks: + - go mod tidy +builds: +- env: + - CGO_ENABLED=0 + mod_timestamp: '{{ .CommitTimestamp }}' + flags: + - -trimpath + ldflags: + - '-s -w' + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - '386' + - arm + - arm64 + ignore: + - goos: darwin + goarch: '386' + binary: '{{ .ProjectName }}_v{{ .Version }}' +archives: +- format: zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' +checksum: + name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS' + algorithm: sha256 +snapshot: + name_template: '{{ incpatch .Version }}-devel' +# signs: +# - artifacts: checksum +# args: +# - "--local-user" +# - "{{ .Env.GPG_FINGERPRINT }}" +# - "--output" +# - "${signature}" +# - "--detach-sign" +# - "${artifact}" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' diff --git a/README.md b/README.md index 8efac932..1cdd4982 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,26 @@ # Bricks! -`make build` +This is an early PoC at this stage! -`./bricks test` +`make build` (or download artifacts) -the rest will follow someday. \ No newline at end of file +Reuses authentication from Databricks CLI. And terraform provider. See details here: https://registry.terraform.io/providers/databrickslabs/databricks/latest/docs#environment-variables + +Supports: +* Databricks CLI +* Databricks CLI Profiles +* Azure CLI Auth +* Azure MSI Auth +* Azure SPN Auth +* Google OIDC Auth +* Direct `DATABRICKS_HOST`, `DATABRICKS_TOKEN` or `DATABRICKS_USERNAME` + `DATABRICKS_PASSWORD` variables. + +What works: + +* `./bricks fs ls /` +* `./bricks test` +* `./bricks run test.py` + +What doesn't work: + +* Everything else. \ No newline at end of file diff --git a/cmd/launch.go b/cmd/launch.go new file mode 100644 index 00000000..12251d7e --- /dev/null +++ b/cmd/launch.go @@ -0,0 +1,33 @@ +package cmd + +import ( + "fmt" + "log" + "os" + + "github.com/databricks/bricks/project" + "github.com/spf13/cobra" +) + +// launchCmd represents the launch command +var launchCmd = &cobra.Command{ + Use: "launch", + Short: "Launches a notebook on development cluster", + Long: `Reads a file and executes it on dev cluster`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + contents, err := os.ReadFile(args[0]) + if err != nil { + log.Fatal(err) + } + results := project.RunPythonOnDev(cmd.Context(), string(contents)) + if results.Failed() { + log.Fatal(results.Error()) + } + fmt.Println(results.Text()) + }, +} + +func init() { + rootCmd.AddCommand(launchCmd) +} diff --git a/cmd/root.go b/cmd/root.go index 7ce517eb..da773b4b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,14 +2,12 @@ package cmd import ( "context" + "log" "os" - "time" + "strings" "github.com/databricks/bricks/project" "github.com/spf13/cobra" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" ) // rootCmd represents the base command when called without any subcommands @@ -19,12 +17,30 @@ var rootCmd = &cobra.Command{ Long: `Where's "data"? Secured by the unity catalog. Projects build lifecycle is secured by bricks`, // Uncomment the following line if your bare application // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, +} + +// TODO: replace with zerolog +type levelWriter []string + +var logLevel = levelWriter{"[INFO]", "[ERROR]", "[WARN]"} +var verbose bool + +func (lw *levelWriter) Write(p []byte) (n int, err error) { + a := string(p) + for _, l := range *lw { + if strings.Contains(a, l) { + return os.Stdout.Write(p) + } + } + return } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { + if verbose { + logLevel = append(logLevel, "[DEBUG]") + } ctx := project.Authenticate(context.Background()) err := rootCmd.ExecuteContext(ctx) if err != nil { @@ -33,16 +49,6 @@ func Execute() { } func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.bricks.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) - zerolog.DurationFieldUnit = time.Second + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "print debug logs") + log.SetOutput(&logLevel) } diff --git a/cmd/test.go b/cmd/test.go index b3c1aecb..ff58c4e4 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -13,7 +13,7 @@ var testCmd = &cobra.Command{ Short: "run tests for the project", Long: `This is longer description of the command`, Run: func(cmd *cobra.Command, args []string) { - results := project.RunPythonOnDev(cmd.Context(), `print("hello, world!")`) + results := project.RunPythonOnDev(cmd.Context(), `return 1`) if results.Failed() { log.Fatal(results.Error()) } diff --git a/go.mod b/go.mod index 7b5d440e..1089f8a6 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,5 @@ go 1.16 require ( github.com/databrickslabs/terraform-provider-databricks v0.5.7 - github.com/rs/zerolog v1.26.1 github.com/spf13/cobra v1.4.0 ) diff --git a/test.py b/test.py new file mode 100644 index 00000000..9329274a --- /dev/null +++ b/test.py @@ -0,0 +1 @@ +spark.sql('show tables').show() \ No newline at end of file