steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

commit for offlineinstall

This commit is contained in:
hari-selvarajan_data 2024-12-27 08:15:01 +00:00
parent 98bdd8d0b1
commit c514cca929
6 changed files with 97 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cmd/labs/github/releases.go
View File

@ -16,24 +16,29 @@ const cacheTTL = 1 * time.Hour
func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache {
pattern := fmt.Sprintf("%s-%s-releases", org, repo)
return &ReleaseCache{
cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
Cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
Org: org,
Repo: repo,
}
}
type ReleaseCache struct {
cache localcache.LocalCache[Versions]
Cache localcache.LocalCache[Versions]
Org string
Repo string
}
func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) {
return r.cache.Load(ctx, func() (Versions, error) {
return r.Cache.Load(ctx, func() (Versions, error) {
return getVersions(ctx, r.Org, r.Repo)
})
}
func (r *ReleaseCache) LoadCache(ctx context.Context) (Versions, error) {
cached, err := r.Cache.LoadCache()
return cached.Data, err
}
// getVersions is considered to be a private API, as we want the usage go through a cache
func getVersions(ctx context.Context, org, repo string) (Versions, error) {
var releases Versions

26
cmd/labs/install.go
View File

@ -7,16 +7,20 @@ import (
)
func newInstallCommand() *cobra.Command {
return &cobra.Command{
Use: "install NAME",
Args: root.ExactArgs(1),
Short: "Installs project",
RunE: func(cmd *cobra.Command, args []string) error {
inst, err := project.NewInstaller(cmd, args[0])
if err != nil {
return err
}
return inst.Install(cmd.Context())
},
cmd := &cobra.Command{}
var offlineInstall bool
cmd.Flags().BoolVar(&offlineInstall, "offline-install", offlineInstall, `If installing in offline mode, set this flag to true.`)
cmd.Use = "install NAME"
cmd.Args = root.ExactArgs(1)
cmd.Short = "Installs project"
cmd.RunE = func(cmd *cobra.Command, args []string) error {
inst, err := project.NewInstaller(cmd, args[0], offlineInstall)
if err != nil {
return err
}
return inst.Install(cmd.Context(), offlineInstall)
}
return cmd
}

4
cmd/labs/localcache/jsonfile.go
View File

@ -35,7 +35,7 @@ type LocalCache[T any] struct {
}
func (r *LocalCache[T]) Load(ctx context.Context, refresh func() (T, error)) (T, error) {
cached, err := r.loadCache()
cached, err := r.LoadCache()
if errors.Is(err, fs.ErrNotExist) {
return r.refreshCache(ctx, refresh, r.zero)
} else if err != nil {
@ -96,7 +96,7 @@ func (r *LocalCache[T]) FileName() string {
return filepath.Join(r.dir, fmt.Sprintf("%s.json", r.name))
}
func (r *LocalCache[T]) loadCache() (*cached[T], error) {
func (r *LocalCache[T]) LoadCache() (*cached[T], error) {
jsonFile := r.FileName()
raw, err := os.ReadFile(r.FileName())
if err != nil {

43
cmd/labs/project/fetcher.go
View File

@ -15,7 +15,7 @@ import (
)
type installable interface {
Install(ctx context.Context) error
Install(ctx context.Context, offlineInstall bool) error
}
type devInstallation struct {
@ -23,7 +23,7 @@ type devInstallation struct {
*cobra.Command
}
func (d *devInstallation) Install(ctx context.Context) error {
func (d *devInstallation) Install(ctx context.Context, offlineInstall bool) error {
if d.Installer == nil {
return nil
}
@ -54,7 +54,7 @@ func (d *devInstallation) Install(ctx context.Context) error {
return d.Installer.runHook(d.Command)
}
func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
func NewInstaller(cmd *cobra.Command, name string, offlineInstall bool) (installable, error) {
if name == "." {
wd, err := os.Getwd()
if err != nil {
@ -75,14 +75,17 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
version = "latest"
}
f := &fetcher{name}
version, err := f.checkReleasedVersions(cmd, version)
version, err := f.checkReleasedVersions(cmd, version, offlineInstall)
if err != nil {
return nil, fmt.Errorf("version: %w", err)
}
prj, err := f.loadRemoteProjectDefinition(cmd, version)
prj, err := f.loadRemoteProjectDefinition(cmd, version, offlineInstall)
if err != nil {
return nil, fmt.Errorf("remote: %w", err)
}
return &installer{
Project: prj,
version: version,
@ -92,11 +95,11 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
func NewUpgrader(cmd *cobra.Command, name string) (*installer, error) {
f := &fetcher{name}
version, err := f.checkReleasedVersions(cmd, "latest")
version, err := f.checkReleasedVersions(cmd, "latest", false)
if err != nil {
return nil, fmt.Errorf("version: %w", err)
}
prj, err := f.loadRemoteProjectDefinition(cmd, version)
prj, err := f.loadRemoteProjectDefinition(cmd, version, false)
if err != nil {
return nil, fmt.Errorf("remote: %w", err)
}
@ -115,7 +118,7 @@ type fetcher struct {
name string
}
func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (string, error) {
func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string, offlineInstall bool) (string, error) {
ctx := cmd.Context()
cacheDir, err := PathInLabs(ctx, f.name, "cache")
if err != nil {
@ -123,7 +126,13 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
}
// `databricks labs isntall X` doesn't know which exact version to fetch, so first
// we fetch all versions and then pick the latest one dynamically.
versions, err := github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx)
var versions github.Versions
if offlineInstall {
versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).LoadCache(ctx)
} else {
versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx)
}
if err != nil {
return "", fmt.Errorf("versions: %w", err)
}
@ -140,9 +149,21 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
return version, nil
}
func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string) (*Project, error) {
func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string, offlineInstall bool) (*Project, error) {
ctx := cmd.Context()
raw, err := github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
var raw []byte
var err error
if !offlineInstall {
raw, err = github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
} else {
libDir, file_err := PathInLabs(ctx, i.name, "lib")
if file_err != nil {
return nil, file_err
}
fileName := filepath.Join(libDir, "labs.yml")
raw, err = os.ReadFile(fileName)
}
if err != nil {
return nil, fmt.Errorf("read labs.yml from GitHub: %w", err)
}

10
cmd/labs/project/installer.go
View File

@ -79,7 +79,7 @@ type installer struct {
cmd *cobra.Command
}
func (i *installer) Install(ctx context.Context) error {
func (i *installer) Install(ctx context.Context, offlineInstall bool) error {
err := i.EnsureFoldersExist()
if err != nil {
return fmt.Errorf("folders: %w", err)
@ -101,9 +101,11 @@ func (i *installer) Install(ctx context.Context) error {
} else if err != nil {
return fmt.Errorf("login: %w", err)
}
err = i.downloadLibrary(ctx)
if err != nil {
return fmt.Errorf("lib: %w", err)
if !offlineInstall {
err = i.downloadLibrary(ctx)
if err != nil {
return fmt.Errorf("lib: %w", err)
}
}
err = i.setupPythonVirtualEnvironment(ctx, w)
if err != nil {

34
cmd/labs/project/installer_test.go
View File

@ -240,6 +240,40 @@ func TestInstallerWorksForReleases(t *testing.T) {
r.RunAndExpectOutput("setting up important infrastructure")
}
func TestOfflineInstallerWorksForReleases(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/2.1/clusters/get" {
respondWithJSON(t, w, &compute.ClusterDetails{
State: compute.StateRunning,
})
return
}
t.Logf("Requested: %s", r.URL.Path)
t.FailNow()
}))
defer server.Close()
ctx := installerContext(t, server)
newHome := copyTestdata(t, "testdata/installed-in-home")
ctx = env.WithUserHomeDir(ctx, newHome)
ctx, stub := process.WithStub(ctx)
stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5")
// on Unix, we call `python3`, but on Windows it is `python.exe`
stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]")
stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]")
stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure")
// simulate the case of GitHub Actions
ctx = env.Set(ctx, "DATABRICKS_HOST", server.URL)
ctx = env.Set(ctx, "DATABRICKS_TOKEN", "...")
ctx = env.Set(ctx, "DATABRICKS_CLUSTER_ID", "installer-cluster")
ctx = env.Set(ctx, "DATABRICKS_WAREHOUSE_ID", "installer-warehouse")
r := testcli.NewRunner(t, ctx, "labs", "install", "blueprint", "--offline-install=true", "--debug")
r.RunAndExpectOutput("setting up important infrastructure")
}
func TestInstallerWorksForDevelopment(t *testing.T) {
defer func() {
if !t.Failed() {