commit for offlineinstall

cmd/labs/github/releases.go

@@ -16,24 +16,29 @@ const cacheTTL = 1 * time.Hour
 func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache {
 	pattern := fmt.Sprintf("%s-%s-releases", org, repo)
 	return &ReleaseCache{
-		cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
+		Cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
 		Org:   org,
 		Repo:  repo,
 	}
 }
 
 type ReleaseCache struct {
-	cache localcache.LocalCache[Versions]
+	Cache localcache.LocalCache[Versions]
 	Org   string
 	Repo  string
 }
 
 func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) {
-	return r.cache.Load(ctx, func() (Versions, error) {
+	return r.Cache.Load(ctx, func() (Versions, error) {
 		return getVersions(ctx, r.Org, r.Repo)
 	})
 }
 
+func (r *ReleaseCache) LoadCache(ctx context.Context) (Versions, error) {
+	cached, err := r.Cache.LoadCache()
+	return cached.Data, err
+}
+
 // getVersions is considered to be a private API, as we want the usage go through a cache
 func getVersions(ctx context.Context, org, repo string) (Versions, error) {
 	var releases Versions

cmd/labs/install.go

@@ -7,16 +7,20 @@ import (
 )
 
 func newInstallCommand() *cobra.Command {
-	return &cobra.Command{
+	cmd := &cobra.Command{}
-		Use:   "install NAME",
+	var offlineInstall bool
-		Args:  root.ExactArgs(1),
+
-		Short: "Installs project",
+	cmd.Flags().BoolVar(&offlineInstall, "offline-install", offlineInstall, `If installing in offline mode, set this flag to true.`)
-		RunE: func(cmd *cobra.Command, args []string) error {
+
-			inst, err := project.NewInstaller(cmd, args[0])
+	cmd.Use = "install NAME"
-			if err != nil {
+	cmd.Args = root.ExactArgs(1)
-				return err
+	cmd.Short = "Installs project"
-			}
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
-			return inst.Install(cmd.Context())
+		inst, err := project.NewInstaller(cmd, args[0], offlineInstall)
-		},
+		if err != nil {
+			return err
+		}
+		return inst.Install(cmd.Context(), offlineInstall)
 	}
+	return cmd
 }

cmd/labs/localcache/jsonfile.go

@@ -35,7 +35,7 @@ type LocalCache[T any] struct {
 }
 
 func (r *LocalCache[T]) Load(ctx context.Context, refresh func() (T, error)) (T, error) {
-	cached, err := r.loadCache()
+	cached, err := r.LoadCache()
 	if errors.Is(err, fs.ErrNotExist) {
 		return r.refreshCache(ctx, refresh, r.zero)
 	} else if err != nil {
@@ -96,7 +96,7 @@ func (r *LocalCache[T]) FileName() string {
 	return filepath.Join(r.dir, fmt.Sprintf("%s.json", r.name))
 }
 
-func (r *LocalCache[T]) loadCache() (*cached[T], error) {
+func (r *LocalCache[T]) LoadCache() (*cached[T], error) {
 	jsonFile := r.FileName()
 	raw, err := os.ReadFile(r.FileName())
 	if err != nil {

cmd/labs/project/fetcher.go

@@ -15,7 +15,7 @@ import (
 )
 
 type installable interface {
-	Install(ctx context.Context) error
+	Install(ctx context.Context, offlineInstall bool) error
 }
 
 type devInstallation struct {
@@ -23,7 +23,7 @@ type devInstallation struct {
 	*cobra.Command
 }
 
-func (d *devInstallation) Install(ctx context.Context) error {
+func (d *devInstallation) Install(ctx context.Context, offlineInstall bool) error {
 	if d.Installer == nil {
 		return nil
 	}
@@ -54,7 +54,7 @@ func (d *devInstallation) Install(ctx context.Context) error {
 	return d.Installer.runHook(d.Command)
 }
 
-func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
+func NewInstaller(cmd *cobra.Command, name string, offlineInstall bool) (installable, error) {
 	if name == "." {
 		wd, err := os.Getwd()
 		if err != nil {
@@ -75,14 +75,17 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
 		version = "latest"
 	}
 	f := &fetcher{name}
-	version, err := f.checkReleasedVersions(cmd, version)
+
+	version, err := f.checkReleasedVersions(cmd, version, offlineInstall)
 	if err != nil {
 		return nil, fmt.Errorf("version: %w", err)
 	}
-	prj, err := f.loadRemoteProjectDefinition(cmd, version)
+
+	prj, err := f.loadRemoteProjectDefinition(cmd, version, offlineInstall)
 	if err != nil {
 		return nil, fmt.Errorf("remote: %w", err)
 	}
+
 	return &installer{
 		Project: prj,
 		version: version,
@@ -92,11 +95,11 @@ func NewInstaller(cmd *cobra.Command, name string) (installable, error) {
 
 func NewUpgrader(cmd *cobra.Command, name string) (*installer, error) {
 	f := &fetcher{name}
-	version, err := f.checkReleasedVersions(cmd, "latest")
+	version, err := f.checkReleasedVersions(cmd, "latest", false)
 	if err != nil {
 		return nil, fmt.Errorf("version: %w", err)
 	}
-	prj, err := f.loadRemoteProjectDefinition(cmd, version)
+	prj, err := f.loadRemoteProjectDefinition(cmd, version, false)
 	if err != nil {
 		return nil, fmt.Errorf("remote: %w", err)
 	}
@@ -115,7 +118,7 @@ type fetcher struct {
 	name string
 }
 
-func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (string, error) {
+func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string, offlineInstall bool) (string, error) {
 	ctx := cmd.Context()
 	cacheDir, err := PathInLabs(ctx, f.name, "cache")
 	if err != nil {
@@ -123,7 +126,13 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
 	}
 	// `databricks labs isntall X` doesn't know which exact version to fetch, so first
 	// we fetch all versions and then pick the latest one dynamically.
-	versions, err := github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx)
+	var versions github.Versions
+	if offlineInstall {
+		versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).LoadCache(ctx)
+	} else {
+		versions, err = github.NewReleaseCache("databrickslabs", f.name, cacheDir).Load(ctx)
+	}
+
 	if err != nil {
 		return "", fmt.Errorf("versions: %w", err)
 	}
@@ -140,9 +149,21 @@ func (f *fetcher) checkReleasedVersions(cmd *cobra.Command, version string) (str
 	return version, nil
 }
 
-func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string) (*Project, error) {
+func (i *fetcher) loadRemoteProjectDefinition(cmd *cobra.Command, version string, offlineInstall bool) (*Project, error) {
 	ctx := cmd.Context()
-	raw, err := github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
+	var raw []byte
+	var err error
+	if !offlineInstall {
+		raw, err = github.ReadFileFromRef(ctx, "databrickslabs", i.name, version, "labs.yml")
+	} else {
+		libDir, file_err := PathInLabs(ctx, i.name, "lib")
+		if file_err != nil {
+			return nil, file_err
+		}
+		fileName := filepath.Join(libDir, "labs.yml")
+		raw, err = os.ReadFile(fileName)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("read labs.yml from GitHub: %w", err)
 	}

cmd/labs/project/installer.go

@@ -79,7 +79,7 @@ type installer struct {
 	cmd *cobra.Command
 }
 
-func (i *installer) Install(ctx context.Context) error {
+func (i *installer) Install(ctx context.Context, offlineInstall bool) error {
 	err := i.EnsureFoldersExist()
 	if err != nil {
 		return fmt.Errorf("folders: %w", err)
@@ -101,9 +101,11 @@ func (i *installer) Install(ctx context.Context) error {
 	} else if err != nil {
 		return fmt.Errorf("login: %w", err)
 	}
-	err = i.downloadLibrary(ctx)
+	if !offlineInstall {
-	if err != nil {
+		err = i.downloadLibrary(ctx)
-		return fmt.Errorf("lib: %w", err)
+		if err != nil {
+			return fmt.Errorf("lib: %w", err)
+		}
 	}
 	err = i.setupPythonVirtualEnvironment(ctx, w)
 	if err != nil {

cmd/labs/project/installer_test.go

@@ -240,6 +240,40 @@ func TestInstallerWorksForReleases(t *testing.T) {
 	r.RunAndExpectOutput("setting up important infrastructure")
 }
 
+func TestOfflineInstallerWorksForReleases(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/2.1/clusters/get" {
+			respondWithJSON(t, w, &compute.ClusterDetails{
+				State: compute.StateRunning,
+			})
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		t.FailNow()
+	}))
+	defer server.Close()
+
+	ctx := installerContext(t, server)
+	newHome := copyTestdata(t, "testdata/installed-in-home")
+	ctx = env.WithUserHomeDir(ctx, newHome)
+
+	ctx, stub := process.WithStub(ctx)
+	stub.WithStdoutFor(`python[\S]+ --version`, "Python 3.10.5")
+	// on Unix, we call `python3`, but on Windows it is `python.exe`
+	stub.WithStderrFor(`python[\S]+ -m venv .*/.databricks/labs/blueprint/state/venv`, "[mock venv create]")
+	stub.WithStderrFor(`python[\S]+ -m pip install --upgrade --upgrade-strategy eager .`, "[mock pip install]")
+	stub.WithStdoutFor(`python[\S]+ install.py`, "setting up important infrastructure")
+
+	// simulate the case of GitHub Actions
+	ctx = env.Set(ctx, "DATABRICKS_HOST", server.URL)
+	ctx = env.Set(ctx, "DATABRICKS_TOKEN", "...")
+	ctx = env.Set(ctx, "DATABRICKS_CLUSTER_ID", "installer-cluster")
+	ctx = env.Set(ctx, "DATABRICKS_WAREHOUSE_ID", "installer-warehouse")
+
+	r := testcli.NewRunner(t, ctx, "labs", "install", "blueprint", "--offline-install=true", "--debug")
+	r.RunAndExpectOutput("setting up important infrastructure")
+}
+
 func TestInstallerWorksForDevelopment(t *testing.T) {
 	defer func() {
 		if !t.Failed() {