steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge d846c91f96 into 24ac8d8d59

This commit is contained in:
Hari Selvarajan 2025-02-11 21:30:35 +05:30 committed by GitHub
parent 24ac8d8d59 d846c91f96
commit c30ddb034f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 107 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/settings.json vendored
View File

@ -9,7 +9,7 @@
],
"go.useLanguageServer": true,
"gopls": {
"formatting.gofumpt": true
"formatting.gofumpt": true
},
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true,

26
cmd/labs/github/releases.go
View File

@ -13,25 +13,31 @@ const cacheTTL = 1 * time.Hour
// NewReleaseCache creates a release cache for a repository in the GitHub org.
// Caller has to provide different cache directories for different repositories.
func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache {
func NewReleaseCache(org, repo, cacheDir string, offlineInstall bool) *ReleaseCache {
pattern := fmt.Sprintf("%s-%s-releases", org, repo)
return &ReleaseCache{
cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
Org: org,
Repo: repo,
cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
Org: org,
Repo: repo,
Offline: offlineInstall,
}
}
type ReleaseCache struct {
cache localcache.LocalCache[Versions]
Org string
Repo string
cache localcache.LocalCache[Versions]
Org string
Repo string
Offline bool
}
func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) {
return r.cache.Load(ctx, func() (Versions, error) {
return getVersions(ctx, r.Org, r.Repo)
})
if !r.Offline {
return r.cache.Load(ctx, func() (Versions, error) {
return getVersions(ctx, r.Org, r.Repo)
})
}
cached, err := r.cache.LoadCache()
return cached.Data, err
}
// getVersions is considered to be a private API, as we want the usage go through a cache

2
cmd/labs/github/releases_test.go
View File

@ -26,7 +26,7 @@ func TestLoadsReleasesForCLI(t *testing.T) {
ctx := context.Background()
ctx = WithApiOverride(ctx, server.URL)
r := NewReleaseCache("databricks", "cli", t.TempDir())
r := NewReleaseCache("databricks", "cli", t.TempDir(), false)
all, err := r.Load(ctx)
assert.NoError(t, err)
assert.Len(t, all, 2)

26
cmd/labs/install.go
View File

@ -7,16 +7,20 @@ import (
)
func newInstallCommand() *cobra.Command {
return &cobra.Command{
Use: "install NAME",
Args: root.ExactArgs(1),
Short: "Installs project",
RunE: func(cmd *cobra.Command, args []string) error {
inst, err := project.NewInstaller(cmd, args[0])
if err != nil {
return err
}
return inst.Install(cmd.Context())
},
cmd := &cobra.Command{}
var offlineInstall bool
cmd.Flags().BoolVar(&offlineInstall, "offline", offlineInstall, `If installing in offline mode, set this flag to true.`)
cmd.Use = "install NAME"
cmd.Args = root.ExactArgs(1)
cmd.Short = "Installs project"
cmd.RunE = func(cmd *cobra.Command, args []string) error {
inst, err := project.NewInstaller(cmd, args[0], offlineInstall)
if err != nil {
return err
}
return inst.Install(cmd.Context(), offlineInstall)
}
return cmd
}

4
cmd/labs/localcache/jsonfile.go
View File

@ -35,7 +35,7 @@ type LocalCache[T any] struct {
}
func (r *LocalCache[T]) Load(ctx context.Context, refresh func() (T, error)) (T, error) {
cached, err := r.loadCache()
cached, err := r.LoadCache()
if errors.Is(err, fs.ErrNotExist) {
return r.refreshCache(ctx, refresh, r.zero)
} else if err != nil {
@ -96,7 +96,7 @@ func (r *LocalCache[T]) FileName() string {
return filepath.Join(r.dir, r.name+".json")
}
func (r *LocalCache[T]) loadCache() (*cached[T], error) {
func (r *LocalCache[T]) LoadCache() (*cached[T], error) {
jsonFile := r.FileName()
raw, err := os.ReadFile(r.FileName())
if err != nil {

38
cmd/labs/project/fetcher.go
View File

@ -15,7 +15,7 @@ import (
)
type installable interface {
Install(ctx context.Context) error
Install(ctx context.Context, offlineInstall bool) error
}
type devInstallation struct {
@ -23,7 +23,7 @@ type devInstallation struct {
*cobra.Command
}
func (d *devInstallation) Install(ctx context.Context) error {
func (d *devInstallation) Install(ctx context.Context, offlineInstall bool) error {
if d.Installer == nil {
return nil
}
@ -54,7 +54,7 @@ func (d *devInstallation) Install(ctx context.Context) error {
return d.Installer.runHook(d.Command)
}
func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
func NewInstaller(cmd *cobra.Command, name string, offlineInstall bool) (installable, error) {
if name == "." {
wd, err := os.Getwd()
if err != nil {
@ -75,14 +75,17 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
version = "latest"
}
f := &fetcher{name}
version, err := f.checkReleasedVersions(cmd, version)
version, err := f.checkReleasedVersions(cmd, version, offlineInstall)
if err != nil {
return nil, fmt.Errorf("version: %w", err)
}
prj, err := f.loadRemoteProjectDefinition(cmd, version)
prj, err := f.loadRemoteProjectDefinition(cmd, version, offlineInstall)
if err != nil {
return nil, fmt.Errorf("remote: %w", err)
}
return &installer{
Project: prj,
version: version,
@ -92,11 +95,11 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
func NewUpgrader(cmd *cobra.Command, name string) (*installer, error) {
f := &fetcher{name}
version, err := f.checkReleasedVersions(cmd, "latest")
version, err := f.checkReleasedVersions(cmd, "latest", false)
if err != nil {
return nil, fmt.Errorf("version: %w", err)
}
prj, err := f.loadRemoteProjectDefinition(cmd, version)
prj, err := f.loadRemoteProjectDefinition(cmd, version, false)
if err != nil {
return nil, fmt.Errorf("remote: %w", err)
}
@ -115,7 +118,7 @@ type fetcher struct {
name string
}
func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (string, error) {
func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string, offlineInstall bool) (string, error) {
ctx := cmd.Context()
cacheDir, err := PathInLabs(ctx, f.name, "cache")
if err != nil {
@ -123,7 +126,8 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
}
// `databricks labs isntall X` doesn't know which exact version to fetch, so first
// we fetch all versions and then pick the latest one dynamically.
versions, err := github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx)
var versions github.Versions
versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir, offlineInstall).Load(ctx)
if err != nil {
return "", fmt.Errorf("versions: %w", err)
}
@ -140,9 +144,21 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
return version, nil
}
func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string) (*Project, error) {
func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string, offlineInstall bool) (*Project, error) {
ctx := cmd.Context()
raw, err := github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
var raw []byte
var err error
if !offlineInstall {
raw, err = github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
} else {
libDir, file_err := PathInLabs(ctx, i.name, "lib")
if file_err != nil {
return nil, file_err
}
fileName := filepath.Join(libDir, "labs.yml")
raw, err = os.ReadFile(fileName)
}
if err != nil {
return nil, fmt.Errorf("read labs.yml from GitHub: %w", err)
}

14
cmd/labs/project/installer.go
View File

@ -79,7 +79,7 @@ type installer struct {
cmd *cobra.Command
}
func (i *installer) Install(ctx context.Context) error {
func (i *installer) Install(ctx context.Context, offlineInstall bool) error {
err := i.EnsureFoldersExist()
if err != nil {
return fmt.Errorf("folders: %w", err)
@ -101,9 +101,15 @@ func (i *installer) Install(ctx context.Context) error {
} else if err != nil {
return fmt.Errorf("login: %w", err)
}
err = i.downloadLibrary(ctx)
if err != nil {
return fmt.Errorf("lib: %w", err)
if !offlineInstall {
err = i.downloadLibrary(ctx)
if err != nil {
return fmt.Errorf("lib: %w", err)
}
}
if _, err := os.Stat(i.LibDir()); os.IsNotExist(err) {
return fmt.Errorf("no local installation found: %w", err)
}
err = i.setupPythonVirtualEnvironment(ctx, w)
if err != nil {

34
cmd/labs/project/installer_test.go
View File

@ -241,6 +241,40 @@ func TestInstallerWorksForReleases(t *testing.T) {
r.RunAndExpectOutput("setting up important infrastructure")
}
func TestOfflineInstallerWorksForReleases(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/2.1/clusters/get" {
respondWithJSON(t, w, &compute.ClusterDetails{
State: compute.StateRunning,
})
return
}
t.Logf("Requested: %s", r.URL.Path)
t.FailNow()
}))
defer server.Close()
ctx := installerContext(t, server)
newHome := copyTestdata(t, "testdata/installed-in-home")
ctx = env.WithUserHomeDir(ctx, newHome)
ctx, stub := process.WithStub(ctx)
stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5")
// on Unix, we call `python3`, but on Windows it is `python.exe`
stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]")
stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]")
stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure")
// simulate the case of GitHub Actions
ctx = env.Set(ctx, "DATABRICKS_HOST", server.URL)
ctx = env.Set(ctx, "DATABRICKS_TOKEN", "...")
ctx = env.Set(ctx, "DATABRICKS_CLUSTER_ID", "installer-cluster")
ctx = env.Set(ctx, "DATABRICKS_WAREHOUSE_ID", "installer-warehouse")
r := testcli.NewRunner(t, ctx, "labs", "install", "blueprint", "--offline=true", "--debug")
r.RunAndExpectOutput("setting up important infrastructure")
}
func TestInstallerWorksForDevelopment(t *testing.T) {
defer func() {
if !t.Failed() {

2
cmd/labs/project/project.go
View File

@ -307,7 +307,7 @@ func (p *Project) checkUpdates(cmd *cobra.Command) error {
// might not be installed yet
return nil
}
r := github.NewReleaseCache("databrickslabs", p.Name, p.CacheDir())
r := github.NewReleaseCache("databrickslabs", p.Name, p.CacheDir(), false)
versions, err := r.Load(ctx)
if err != nil {
return err