Merge branch 'private-clone' into git-clone

This commit is contained in:
Shreyas Goenka 2023-06-29 17:48:17 +02:00
commit fb6247bc16
No known key found for this signature in database
GPG Key ID: 92A07DF49CCB0622
3 changed files with 141 additions and 33 deletions

View File

@ -0,0 +1,60 @@
package internal
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/databricks/cli/libs/git"
"github.com/stretchr/testify/assert"
)
// TODO: add assertion that the git tool is not called, maybe by voiding PATH
// TODO: add assertion for error if git CLI is not found
func TestGitClonePublicRepository(t *testing.T) {
// t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
tmpDir := t.TempDir()
ctx := context.Background()
var err error
// We unset PATH to ensure that git.Clone cannot rely on the git CLI
t.Setenv("PATH", "")
err = git.Clone(ctx, git.CloneOptions{
Provider: "github",
Organization: "databricks",
RepositoryName: "cli",
Reference: "main",
TargetDir: tmpDir,
})
assert.NoError(t, err)
assert.DirExists(t, filepath.Join(tmpDir, "cli-main"))
b, err := os.ReadFile(filepath.Join(tmpDir, "cli-main/NOTICE"))
assert.NoError(t, err)
assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
}
func TestAccGitClonePrivateRepository(t *testing.T) {
// t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV"))
tmpDir := t.TempDir()
ctx := context.Background()
// This is a private repository only accessible to databricks employees
err := git.Clone(ctx, git.CloneOptions{
Provider: "github",
Organization: "databricks",
RepositoryName: "bundle-samples-internal",
Reference: "main",
TargetDir: tmpDir,
})
assert.NoError(t, err)
// assert examples from the private repository
assert.DirExists(t, filepath.Join(tmpDir, "bundle-samples-internal-main", "shark_sightings"))
assert.DirExists(t, filepath.Join(tmpDir, "bundle-samples-internal-main", "wikipedia_clickstream"))
}

View File

@ -1,28 +1,75 @@
package git package git
import ( import (
"context"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"github.com/databricks/cli/libs/zip" "github.com/databricks/cli/libs/zip"
) )
func githubZipUrl(org string, name string, ref string) string { var errNotFound = errors.New("not found")
return fmt.Sprintf(`https://github.com/%s/%s/archive/%s.zip`, org, name, ref)
type RepositoryNotFoundError struct {
url string
} }
func download(url string, dest string) error { func (err RepositoryNotFoundError) Error() string {
resp, err := http.Get(url) return fmt.Sprintf("repository not found: %s", err.url)
}
func (err RepositoryNotFoundError) Is(other error) bool {
return other == errNotFound
}
type CloneOptions struct {
// Name of the organization or profile with the repository
Organization string
RepositoryName string
// Git service provider. Eg: github, gitlab
Provider string
// Branch or tag name to clone
Reference string
// Path to clone into. The repository is cloned as ${RepositoryName}-${Reference}
// in this target directory.
TargetDir string
}
func (opts CloneOptions) repoUrl() string {
return fmt.Sprintf(`https://github.com/%s/%s`, opts.Organization, opts.RepositoryName)
}
func (opts CloneOptions) zipUrl() string {
return fmt.Sprintf(`%s/archive/%s.zip`, opts.repoUrl(), opts.Reference)
}
// TODO: test this holds true for alternate branches and tags
func (opts CloneOptions) destination() string {
return filepath.Join(opts.TargetDir, opts.RepositoryName+"-"+opts.Reference)
}
func download(ctx context.Context, url string, dest string) error {
// Get request to download the ZIP archive
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(request)
if err != nil { if err != nil {
return err return err
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound { if resp.StatusCode == http.StatusNotFound {
return fmt.Errorf("repository not found: %s", url) return RepositoryNotFoundError{url}
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
@ -39,22 +86,45 @@ func download(url string, dest string) error {
return err return err
} }
func Clone(org string, repoName string, targetDir string) error { // TODO: check stdin / stdout works properly with git clone and requesting an ID/password
zipDst := filepath.Join(targetDir, repoName+".zip") func clonePrivate(ctx context.Context, opts CloneOptions) error {
zipUrl := githubZipUrl(org, repoName, "main") // TODO: test that the branch --branch flag works with tags
cmd := exec.CommandContext(ctx, "git", "clone", opts.repoUrl(), opts.destination(), "--branch", opts.Reference)
return cmd.Run()
}
func clonePublic(ctx context.Context, opts CloneOptions) error {
zipDst := filepath.Join(opts.TargetDir, opts.RepositoryName+".zip")
// Download public repository from github as a ZIP file // Download public repository from github as a ZIP file
err := download(zipUrl, zipDst) err := download(ctx, opts.zipUrl(), zipDst)
if err != nil { if err != nil {
return err return err
} }
// Decompress the ZIP file // Decompress the ZIP file
err = zip.Extract(zipDst, targetDir) err = zip.Extract(zipDst, opts.TargetDir)
if err != nil { if err != nil {
return err return err
} }
// Remove the ZIP file // Remove the ZIP file post extraction
return os.Remove(zipDst) return os.Remove(zipDst)
} }
func Clone(ctx context.Context, opts CloneOptions) error {
if opts.Provider != "github" {
return fmt.Errorf("git provider not supported: %s", opts.Provider)
}
// First we try to clone the repository as a public URL, as that does not
// require the git CLI
err := clonePublic(ctx, opts)
// If a public repository was not found, we defer to the git CLI
if errors.Is(err, errNotFound) {
return clonePrivate(ctx, opts)
}
return err
}

View File

@ -1,22 +0,0 @@
package git
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGitClonePublicRepository(t *testing.T) {
tmpDir := t.TempDir()
var err error
err = Clone("databricks", "cli", tmpDir)
assert.NoError(t, err)
assert.DirExists(t, filepath.Join(tmpDir, "cli-main"))
b, err := os.ReadFile(filepath.Join(tmpDir, "cli-main/NOTICE"))
assert.NoError(t, err)
assert.Contains(t, string(b), "Copyright (2023) Databricks, Inc.")
}