diff --git a/cmd/labs/github/releases.go b/cmd/labs/github/releases.go index 0dae0317d..1737ffbf4 100644 --- a/cmd/labs/github/releases.go +++ b/cmd/labs/github/releases.go @@ -16,24 +16,29 @@ const cacheTTL = 1 * time.Hour func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache { pattern := fmt.Sprintf("%s-%s-releases", org, repo) return &ReleaseCache{ - cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL), + Cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL), Org: org, Repo: repo, } } type ReleaseCache struct { - cache localcache.LocalCache[Versions] + Cache localcache.LocalCache[Versions] Org string Repo string } func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) { - return r.cache.Load(ctx, func() (Versions, error) { + return r.Cache.Load(ctx, func() (Versions, error) { return getVersions(ctx, r.Org, r.Repo) }) } +func (r *ReleaseCache) LoadCache(ctx context.Context) (Versions, error) { + cached, err := r.Cache.LoadCache() + return cached.Data, err +} + // getVersions is considered to be a private API, as we want the usage go through a cache func getVersions(ctx context.Context, org, repo string) (Versions, error) { var releases Versions diff --git a/cmd/labs/install.go b/cmd/labs/install.go index 6ed6b2e91..978916bb8 100644 --- a/cmd/labs/install.go +++ b/cmd/labs/install.go @@ -7,16 +7,20 @@ import ( ) func newInstallCommand() *cobra.Command { - return &cobra.Command{ - Use: "install NAME", - Args: root.ExactArgs(1), - Short: "Installs project", - RunE: func(cmd *cobra.Command, args []string) error { - inst, err := project.NewInstaller(cmd, args[0]) - if err != nil { - return err - } - return inst.Install(cmd.Context()) - }, + cmd := &cobra.Command{} + var offlineInstall bool + + cmd.Flags().BoolVar(&offlineInstall, "offline-install", offlineInstall, `If installing in offline mode, set this flag to true.`) + + cmd.Use = "install NAME" + cmd.Args = root.ExactArgs(1) + cmd.Short = "Installs project" + cmd.RunE = func(cmd *cobra.Command, args []string) error { + inst, err := project.NewInstaller(cmd, args[0], offlineInstall) + if err != nil { + return err + } + return inst.Install(cmd.Context(), offlineInstall) } + return cmd } diff --git a/cmd/labs/localcache/jsonfile.go b/cmd/labs/localcache/jsonfile.go index 6540e4ac2..5f6d35359 100644 --- a/cmd/labs/localcache/jsonfile.go +++ b/cmd/labs/localcache/jsonfile.go @@ -35,7 +35,7 @@ type LocalCache[T any] struct { } func (r *LocalCache[T]) Load(ctx context.Context, refresh func() (T, error)) (T, error) { - cached, err := r.loadCache() + cached, err := r.LoadCache() if errors.Is(err, fs.ErrNotExist) { return r.refreshCache(ctx, refresh, r.zero) } else if err != nil { @@ -96,7 +96,7 @@ func (r *LocalCache[T]) FileName() string { return filepath.Join(r.dir, fmt.Sprintf("%s.json", r.name)) } -func (r *LocalCache[T]) loadCache() (*cached[T], error) { +func (r *LocalCache[T]) LoadCache() (*cached[T], error) { jsonFile := r.FileName() raw, err := os.ReadFile(r.FileName()) if err != nil { diff --git a/cmd/labs/project/fetcher.go b/cmd/labs/project/fetcher.go index 8f4fafde6..efc654c8d 100644 --- a/cmd/labs/project/fetcher.go +++ b/cmd/labs/project/fetcher.go @@ -15,7 +15,7 @@ import ( ) type installable interface { - Install(ctx context.Context) error + Install(ctx context.Context, offlineInstall bool) error } type devInstallation struct { @@ -23,7 +23,7 @@ type devInstallation struct { *cobra.Command } -func (d *devInstallation) Install(ctx context.Context) error { +func (d *devInstallation) Install(ctx context.Context, offlineInstall bool) error { if d.Installer == nil { return nil } @@ -54,7 +54,7 @@ func (d *devInstallation) Install(ctx context.Context) error { return d.Installer.runHook(d.Command) } -func NewInstaller(cmd *cobra.Command, name string) (installable, error) { +func NewInstaller(cmd *cobra.Command, name string, offlineInstall bool) (installable, error) { if name == "." { wd, err := os.Getwd() if err != nil { @@ -75,14 +75,17 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) { version = "latest" } f := &fetcher{name} - version, err := f.checkReleasedVersions(cmd, version) + + version, err := f.checkReleasedVersions(cmd, version, offlineInstall) if err != nil { return nil, fmt.Errorf("version: %w", err) } - prj, err := f.loadRemoteProjectDefinition(cmd, version) + + prj, err := f.loadRemoteProjectDefinition(cmd, version, offlineInstall) if err != nil { return nil, fmt.Errorf("remote: %w", err) } + return &installer{ Project: prj, version: version, @@ -92,11 +95,11 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) { func NewUpgrader(cmd *cobra.Command, name string) (*installer, error) { f := &fetcher{name} - version, err := f.checkReleasedVersions(cmd, "latest") + version, err := f.checkReleasedVersions(cmd, "latest", false) if err != nil { return nil, fmt.Errorf("version: %w", err) } - prj, err := f.loadRemoteProjectDefinition(cmd, version) + prj, err := f.loadRemoteProjectDefinition(cmd, version, false) if err != nil { return nil, fmt.Errorf("remote: %w", err) } @@ -115,7 +118,7 @@ type fetcher struct { name string } -func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (string, error) { +func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string, offlineInstall bool) (string, error) { ctx := cmd.Context() cacheDir, err := PathInLabs(ctx, f.name, "cache") if err != nil { @@ -123,7 +126,13 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str } // `databricks labs isntall X` doesn't know which exact version to fetch, so first // we fetch all versions and then pick the latest one dynamically. - versions, err := github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx) + var versions github.Versions + if offlineInstall { + versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).LoadCache(ctx) + } else { + versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx) + } + if err != nil { return "", fmt.Errorf("versions: %w", err) } @@ -140,9 +149,21 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str return version, nil } -func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string) (*Project, error) { +func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string, offlineInstall bool) (*Project, error) { ctx := cmd.Context() - raw, err := github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml") + var raw []byte + var err error + if !offlineInstall { + raw, err = github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml") + } else { + libDir, file_err := PathInLabs(ctx, i.name, "lib") + if file_err != nil { + return nil, file_err + } + fileName := filepath.Join(libDir, "labs.yml") + raw, err = os.ReadFile(fileName) + } + if err != nil { return nil, fmt.Errorf("read labs.yml from GitHub: %w", err) } diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 041415964..10540f904 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -79,7 +79,7 @@ type installer struct { cmd *cobra.Command } -func (i *installer) Install(ctx context.Context) error { +func (i *installer) Install(ctx context.Context, offlineInstall bool) error { err := i.EnsureFoldersExist() if err != nil { return fmt.Errorf("folders: %w", err) @@ -101,9 +101,11 @@ func (i *installer) Install(ctx context.Context) error { } else if err != nil { return fmt.Errorf("login: %w", err) } - err = i.downloadLibrary(ctx) - if err != nil { - return fmt.Errorf("lib: %w", err) + if !offlineInstall { + err = i.downloadLibrary(ctx) + if err != nil { + return fmt.Errorf("lib: %w", err) + } } err = i.setupPythonVirtualEnvironment(ctx, w) if err != nil { diff --git a/cmd/labs/project/installer_test.go b/cmd/labs/project/installer_test.go index a69389b31..d8c8e0085 100644 --- a/cmd/labs/project/installer_test.go +++ b/cmd/labs/project/installer_test.go @@ -240,6 +240,40 @@ func TestInstallerWorksForReleases(t *testing.T) { r.RunAndExpectOutput("setting up important infrastructure") } +func TestOfflineInstallerWorksForReleases(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/2.1/clusters/get" { + respondWithJSON(t, w, &compute.ClusterDetails{ + State: compute.StateRunning, + }) + return + } + t.Logf("Requested: %s", r.URL.Path) + t.FailNow() + })) + defer server.Close() + + ctx := installerContext(t, server) + newHome := copyTestdata(t, "testdata/installed-in-home") + ctx = env.WithUserHomeDir(ctx, newHome) + + ctx, stub := process.WithStub(ctx) + stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5") + // on Unix, we call `python3`, but on Windows it is `python.exe` + stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]") + stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]") + stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure") + + // simulate the case of GitHub Actions + ctx = env.Set(ctx, "DATABRICKS_HOST", server.URL) + ctx = env.Set(ctx, "DATABRICKS_TOKEN", "...") + ctx = env.Set(ctx, "DATABRICKS_CLUSTER_ID", "installer-cluster") + ctx = env.Set(ctx, "DATABRICKS_WAREHOUSE_ID", "installer-warehouse") + + r := testcli.NewRunner(t, ctx, "labs", "install", "blueprint", "--offline-install=true", "--debug") + r.RunAndExpectOutput("setting up important infrastructure") +} + func TestInstallerWorksForDevelopment(t *testing.T) { defer func() { if !t.Failed() {