Merge branch 'private-clone' into git-clone

internal/git_clone_test.go

@@ -0,0 +1,60 @@
+package internal
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/libs/git"
+	"github.com/stretchr/testify/assert"
+)
+
+// TODO: add assertion that the git tool is not called, maybe by voiding PATH
+// TODO: add assertion for error if git CLI is not found
+func TestGitClonePublicRepository(t *testing.T) {
+	// t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	tmpDir := t.TempDir()
+	ctx := context.Background()
+	var err error
+
+	// We unset PATH to ensure that git.Clone cannot rely on the git CLI
+	t.Setenv("PATH", "")
+
+	err = git.Clone(ctx, git.CloneOptions{
+		Provider:       "github",
+		Organization:   "databricks",
+		RepositoryName: "cli",
+		Reference:      "main",
+		TargetDir:      tmpDir,
+	})
+
+	assert.NoError(t, err)
+	assert.DirExists(t, filepath.Join(tmpDir, "cli-main"))
+
+	b, err := os.ReadFile(filepath.Join(tmpDir, "cli-main/NOTICE"))
+	assert.NoError(t, err)
+	assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
+}
+
+func TestAccGitClonePrivateRepository(t *testing.T) {
+	// t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
+
+	tmpDir := t.TempDir()
+	ctx := context.Background()
+
+	// This is a private repository only accessible to databricks employees
+	err := git.Clone(ctx, git.CloneOptions{
+		Provider:       "github",
+		Organization:   "databricks",
+		RepositoryName: "bundle-samples-internal",
+		Reference:      "main",
+		TargetDir:      tmpDir,
+	})
+	assert.NoError(t, err)
+
+	// assert examples from the private repository
+	assert.DirExists(t, filepath.Join(tmpDir, "bundle-samples-internal-main", "shark_sightings"))
+	assert.DirExists(t, filepath.Join(tmpDir, "bundle-samples-internal-main", "wikipedia_clickstream"))
+}

libs/git/clone.go

@@ -1,28 +1,75 @@
 package git
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/databricks/cli/libs/zip"
 )
 
-func githubZipUrl(org string, name string, ref string) string {
+var errNotFound = errors.New("not found")
-	return fmt.Sprintf(`https://github.com/%s/%s/archive/%s.zip`, org, name, ref)
+
+type RepositoryNotFoundError struct {
+	url string
 }
 
-func download(url string, dest string) error {
+func (err RepositoryNotFoundError) Error() string {
-	resp, err := http.Get(url)
+	return fmt.Sprintf("repository not found: %s", err.url)
+}
+
+func (err RepositoryNotFoundError) Is(other error) bool {
+	return other == errNotFound
+}
+
+type CloneOptions struct {
+	// Name of the organization or profile with the repository
+	Organization   string
+	RepositoryName string
+
+	// Git service provider. Eg: github, gitlab
+	Provider string
+
+	// Branch or tag name to clone
+	Reference string
+
+	// Path to clone into. The repository is cloned as ${RepositoryName}-${Reference}
+	// in this target directory.
+	TargetDir string
+}
+
+func (opts CloneOptions) repoUrl() string {
+	return fmt.Sprintf(`https://github.com/%s/%s`, opts.Organization, opts.RepositoryName)
+}
+
+func (opts CloneOptions) zipUrl() string {
+	return fmt.Sprintf(`%s/archive/%s.zip`, opts.repoUrl(), opts.Reference)
+}
+
+// TODO: test this holds true for alternate branches and tags
+func (opts CloneOptions) destination() string {
+	return filepath.Join(opts.TargetDir, opts.RepositoryName+"-"+opts.Reference)
+}
+
+func download(ctx context.Context, url string, dest string) error {
+	// Get request to download the ZIP archive
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(request)
 	if err != nil {
 		return err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusNotFound {
-		return fmt.Errorf("repository not found: %s", url)
+		return RepositoryNotFoundError{url}
 	}
 
 	if resp.StatusCode != http.StatusOK {
@@ -39,22 +86,45 @@ func download(url string, dest string) error {
 	return err
 }
 
-func Clone(org string, repoName string, targetDir string) error {
+// TODO: check stdin / stdout works properly with git clone and requesting an ID/password
-	zipDst := filepath.Join(targetDir, repoName+".zip")
+func clonePrivate(ctx context.Context, opts CloneOptions) error {
-	zipUrl := githubZipUrl(org, repoName, "main")
+	// TODO: test that the branch --branch flag works with tags
+	cmd := exec.CommandContext(ctx, "git", "clone", opts.repoUrl(), opts.destination(), "--branch", opts.Reference)
+	return cmd.Run()
+}
+
+func clonePublic(ctx context.Context, opts CloneOptions) error {
+	zipDst := filepath.Join(opts.TargetDir, opts.RepositoryName+".zip")
 
 	// Download public repository from github as a ZIP file
-	err := download(zipUrl, zipDst)
+	err := download(ctx, opts.zipUrl(), zipDst)
 	if err != nil {
 		return err
 	}
 
 	// Decompress the ZIP file
-	err = zip.Extract(zipDst, targetDir)
+	err = zip.Extract(zipDst, opts.TargetDir)
 	if err != nil {
 		return err
 	}
 
-	// Remove the ZIP file
+	// Remove the ZIP file post extraction
 	return os.Remove(zipDst)
 }
+
+func Clone(ctx context.Context, opts CloneOptions) error {
+	if opts.Provider != "github" {
+		return fmt.Errorf("git provider not supported: %s", opts.Provider)
+	}
+
+	// First we try to clone the repository as a public URL, as that does not
+	// require the git CLI
+	err := clonePublic(ctx, opts)
+
+	// If a public repository was not found, we defer to the git CLI
+	if errors.Is(err, errNotFound) {
+
+		return clonePrivate(ctx, opts)
+	}
+	return err
+}

libs/git/clone_test.go

@@ -1,22 +0,0 @@
-package git
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestGitClonePublicRepository(t *testing.T) {
-	tmpDir := t.TempDir()
-	var err error
-
-	err = Clone("databricks", "cli", tmpDir)
-	assert.NoError(t, err)
-	assert.DirExists(t, filepath.Join(tmpDir, "cli-main"))
-
-	b, err := os.ReadFile(filepath.Join(tmpDir, "cli-main/NOTICE"))
-	assert.NoError(t, err)
-	assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
-}