Support Git worktrees for `sync` (#1831)

## Changes

This change allows the `sync` command to work from [git
worktrees](https://git-scm.com/docs/git-worktree).

## Tests

* Added unit tests for traversal of worktree related files.
* Manually confirmed that synchronization of files from a main checkout,
as well as a worktree, observed the same ignore rules (both locally
defined as well as from `$GIT_DIR/info/exclude`).

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

internal/bugbash/README.md

@@ -0,0 +1,13 @@
+# Bugbash
+
+The script in this directory can be used to conveniently exec into a shell
+where a CLI build for a specific branch is made available.
+
+## Usage
+
+This script prompts if you do NOT have at least Bash 5 installed,
+but works without command completion with earlier versions.
+
+```shell
+bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch
+```

internal/bugbash/exec.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Set the GitHub repository for the Databricks CLI.
+export GH_REPO="databricks/cli"
+
+# Synthesize the directory name for the snapshot build.
+function cli_snapshot_directory() {
+    dir="cli"
+
+    # Append OS
+    case "$(uname -s)" in
+    Linux)
+        dir="${dir}_linux"
+        ;;
+    Darwin)
+        dir="${dir}_darwin"
+        ;;
+    *)
+        echo "Unknown operating system: $os"
+        ;;
+    esac
+
+    # Append architecture
+    case "$(uname -m)" in
+    x86_64)
+        dir="${dir}_amd64_v1"
+        ;;
+    i386|i686)
+        dir="${dir}_386"
+        ;;
+    arm64|aarch64)
+        dir="${dir}_arm64"
+        ;;
+    armv7l|armv8l)
+        dir="${dir}_arm_6"
+        ;;
+    *)
+        echo "Unknown architecture: $arch"
+        ;;
+    esac
+
+    echo $dir
+}
+
+BRANCH=$1
+shift
+
+# Default to main branch if branch is not specified.
+if [ -z "$BRANCH" ]; then
+  BRANCH=main
+fi
+
+if [ -z "$BRANCH" ]; then
+  echo "Please specify which branch to bugbash..."
+  exit 1
+fi
+
+# Check if the "gh" command is available.
+if ! command -v gh &> /dev/null; then
+  echo "The GitHub CLI (gh) is required to download the snapshot build."
+  echo "Install and configure it with:"
+  echo ""
+  echo "  brew install gh"
+  echo "  gh auth login"
+  echo ""
+  exit 1
+fi
+
+echo "Looking for a snapshot build of the Databricks CLI on branch $BRANCH..."
+
+# Find last successful build on $BRANCH.
+last_successful_run_id=$(
+  gh run list -b "$BRANCH" -w release-snapshot --json 'databaseId,conclusion' |
+      jq 'limit(1; .[] | select(.conclusion == "success")) | .databaseId'
+)
+if [ -z "$last_successful_run_id" ]; then
+  echo "Unable to find last successful build of the release-snapshot workflow for branch $BRANCH."
+  exit 1
+fi
+
+# Determine artifact name with the right binaries for this runner.
+case "$(uname -s)" in
+Linux)
+    artifact="cli_linux_snapshot"
+    ;;
+Darwin)
+    artifact="cli_darwin_snapshot"
+    ;;
+esac
+
+# Create a temporary directory to download the artifact.
+dir=$(mktemp -d)
+
+# Download the artifact.
+echo "Downloading the snapshot build..."
+gh run download "$last_successful_run_id" -n "$artifact" -D "$dir/.bin"
+dir="$dir/.bin/$(cli_snapshot_directory)"
+if [ ! -d "$dir" ]; then
+    echo "Directory does not exist: $dir"
+    exit 1
+fi
+
+# Make CLI available on $PATH.
+chmod +x "$dir/databricks"
+export PATH="$dir:$PATH"
+
+# Set the prompt to indicate the bugbash environment and exec.
+export PS1="(bugbash $BRANCH) \[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ "
+
+# Display completion instructions.
+echo ""
+echo "=================================================================="
+
+if [[ ${BASH_VERSINFO[0]} -lt 5 ]]; then
+    echo -en "\033[31m"
+    echo "You have Bash version < 5 installed... completion won't work."
+    echo -en "\033[0m"
+    echo ""
+    echo "Install it with:"
+    echo ""
+    echo "  brew install bash bash-completion"
+    echo ""
+    echo "=================================================================="
+fi
+
+echo ""
+echo "To load completions in your current shell session:"
+echo ""
+echo "  source /opt/homebrew/etc/profile.d/bash_completion.sh"
+echo "  source <(databricks completion bash)"
+echo ""
+echo "=================================================================="
+echo ""
+
+# Exec into a new shell.
+# Note: don't use zsh because on macOS it _always_ overwrites PS1.
+exec /usr/bin/env bash

libs/git/repository.go

@@ -23,8 +23,21 @@ type Repository struct {
 	// directory where we process .gitignore files.
 	real bool
 
-	// root is the absolute path to the repository root.
+	// rootDir is the path to the root of the repository checkout.
-	root vfs.Path
+	// This can be either the main repository checkout or a worktree checkout.
+	// For more information about worktrees, see: https://git-scm.com/docs/git-worktree#_description.
+	rootDir vfs.Path
+
+	// gitDir is the equivalent of $GIT_DIR and points to the
+	// `.git` directory of a repository or a worktree directory.
+	// See https://git-scm.com/docs/git-worktree#_details for more information.
+	gitDir vfs.Path
+
+	// gitCommonDir is the equivalent of $GIT_COMMON_DIR and points to the
+	// `.git` directory of the main working tree (common between worktrees).
+	// This is equivalent to [gitDir] if this is the main working tree.
+	// See https://git-scm.com/docs/git-worktree#_details for more information.
+	gitCommonDir vfs.Path
 
 	// ignore contains a list of ignore patterns indexed by the
 	// path prefix relative to the repository root.
@@ -44,12 +57,11 @@ type Repository struct {
 
 // Root returns the absolute path to the repository root.
 func (r *Repository) Root() string {
-	return r.root.Native()
+	return r.rootDir.Native()
 }
 
 func (r *Repository) CurrentBranch() (string, error) {
-	// load .git/HEAD
+	ref, err := LoadReferenceFile(r.gitDir, "HEAD")
-	ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD"))
 	if err != nil {
 		return "", err
 	}
@@ -65,8 +77,7 @@ func (r *Repository) CurrentBranch() (string, error) {
 }
 
 func (r *Repository) LatestCommit() (string, error) {
-	// load .git/HEAD
+	ref, err := LoadReferenceFile(r.gitDir, "HEAD")
-	ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD"))
 	if err != nil {
 		return "", err
 	}
@@ -80,12 +91,12 @@ func (r *Repository) LatestCommit() (string, error) {
 		return ref.Content, nil
 	}
 
-	// read reference from .git/HEAD
+	// Read reference from $GIT_DIR/HEAD
 	branchHeadPath, err := ref.ResolvePath()
 	if err != nil {
 		return "", err
 	}
-	branchHeadRef, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, branchHeadPath))
+	branchHeadRef, err := LoadReferenceFile(r.gitCommonDir, branchHeadPath)
 	if err != nil {
 		return "", err
 	}
@@ -125,7 +136,7 @@ func (r *Repository) loadConfig() error {
 	if err != nil {
 		return fmt.Errorf("unable to load user specific gitconfig: %w", err)
 	}
-	err = config.loadFile(r.root, ".git/config")
+	err = config.loadFile(r.gitCommonDir, "config")
 	if err != nil {
 		return fmt.Errorf("unable to load repository specific gitconfig: %w", err)
 	}
@@ -133,12 +144,6 @@ func (r *Repository) loadConfig() error {
 	return nil
 }
 
-// newIgnoreFile constructs a new [ignoreRules] implementation backed by
-// a file using the specified path relative to the repository root.
-func (r *Repository) newIgnoreFile(relativeIgnoreFilePath string) ignoreRules {
-	return newIgnoreFile(r.root, relativeIgnoreFilePath)
-}
-
 // getIgnoreRules returns a slice of [ignoreRules] that apply
 // for the specified prefix. The prefix must be cleaned by the caller.
 // It lazily initializes an entry for the specified prefix if it
@@ -149,7 +154,7 @@ func (r *Repository) getIgnoreRules(prefix string) []ignoreRules {
 		return fs
 	}
 
-	r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(path.Join(prefix, gitIgnoreFileName)))
+	r.ignore[prefix] = append(r.ignore[prefix], newIgnoreFile(r.rootDir, path.Join(prefix, gitIgnoreFileName)))
 	return r.ignore[prefix]
 }
 
@@ -205,21 +210,30 @@ func (r *Repository) Ignore(relPath string) (bool, error) {
 
 func NewRepository(path vfs.Path) (*Repository, error) {
 	real := true
-	rootPath, err := vfs.FindLeafInTree(path, GitDirectoryName)
+	rootDir, err := vfs.FindLeafInTree(path, GitDirectoryName)
 	if err != nil {
 		if !errors.Is(err, fs.ErrNotExist) {
 			return nil, err
 		}
 		// Cannot find `.git` directory.
-		// Treat the specified path as a potential repository root.
+		// Treat the specified path as a potential repository root checkout.
 		real = false
-		rootPath = path
+		rootDir = path
+	}
+
+	// Derive $GIT_DIR and $GIT_COMMON_DIR paths if this is a real repository.
+	// If it isn't a real repository, they'll point to the (non-existent) `.git` directory.
+	gitDir, gitCommonDir, err := resolveGitDirs(rootDir)
+	if err != nil {
+		return nil, err
 	}
 
 	repo := &Repository{
-		real:   real,
+		real:         real,
-		root:   rootPath,
+		rootDir:      rootDir,
-		ignore: make(map[string][]ignoreRules),
+		gitDir:       gitDir,
+		gitCommonDir: gitCommonDir,
+		ignore:       make(map[string][]ignoreRules),
 	}
 
 	err = repo.loadConfig()
@@ -253,9 +267,9 @@ func NewRepository(path vfs.Path) (*Repository, error) {
 			".git",
 		}),
 		// Load repository-wide exclude file.
-		repo.newIgnoreFile(".git/info/exclude"),
+		newIgnoreFile(repo.gitCommonDir, "info/exclude"),
 		// Load root gitignore file.
-		repo.newIgnoreFile(".gitignore"),
+		newIgnoreFile(repo.rootDir, ".gitignore"),
 	}
 
 	return repo, nil

libs/git/view.go

@@ -80,7 +80,7 @@ func NewView(root vfs.Path) (*View, error) {
 
 	// Target path must be relative to the repository root path.
 	target := root.Native()
-	prefix := repo.root.Native()
+	prefix := repo.rootDir.Native()
 	if !strings.HasPrefix(target, prefix) {
 		return nil, fmt.Errorf("path %q is not within repository root %q", root.Native(), prefix)
 	}

libs/git/worktree.go

@@ -0,0 +1,123 @@
+package git
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/cli/libs/vfs"
+)
+
+func readLines(root vfs.Path, name string) ([]string, error) {
+	file, err := root.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+
+	var lines []string
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+
+	return lines, scanner.Err()
+}
+
+// readGitDir reads the value of the `.git` file in a worktree.
+func readGitDir(root vfs.Path) (string, error) {
+	lines, err := readLines(root, GitDirectoryName)
+	if err != nil {
+		return "", err
+	}
+
+	var gitDir string
+	for _, line := range lines {
+		parts := strings.SplitN(line, ": ", 2)
+		if len(parts) != 2 {
+			continue
+		}
+
+		if parts[0] == "gitdir" {
+			gitDir = strings.TrimSpace(parts[1])
+		}
+	}
+
+	if gitDir == "" {
+		return "", fmt.Errorf(`expected %q to contain a line with "gitdir: [...]"`, filepath.Join(root.Native(), GitDirectoryName))
+	}
+
+	return gitDir, nil
+}
+
+// readGitCommonDir reads the value of the `commondir` file in the `.git` directory of a worktree.
+// This file typically contains "../.." to point to $GIT_COMMON_DIR.
+func readGitCommonDir(gitDir vfs.Path) (string, error) {
+	lines, err := readLines(gitDir, "commondir")
+	if err != nil {
+		return "", err
+	}
+
+	if len(lines) == 0 {
+		return "", errors.New("file is empty")
+	}
+
+	return strings.TrimSpace(lines[0]), nil
+}
+
+// resolveGitDirs resolves the paths for $GIT_DIR and $GIT_COMMON_DIR.
+// The path argument is the root of the checkout where (supposedly) a `.git` file or directory exists.
+func resolveGitDirs(root vfs.Path) (vfs.Path, vfs.Path, error) {
+	fileInfo, err := root.Stat(GitDirectoryName)
+	if err != nil {
+		// If the `.git` file or directory does not exist, then this is not a git repository.
+		// Return paths that we know don't exist, so we do not need to perform nil checks in the caller.
+		if errors.Is(err, fs.ErrNotExist) {
+			gitDir := vfs.MustNew(filepath.Join(root.Native(), GitDirectoryName))
+			return gitDir, gitDir, nil
+		}
+		return nil, nil, err
+	}
+
+	// If the path is a directory, then it is the main working tree.
+	// Both $GIT_DIR and $GIT_COMMON_DIR point to the same directory.
+	if fileInfo.IsDir() {
+		gitDir := vfs.MustNew(filepath.Join(root.Native(), GitDirectoryName))
+		return gitDir, gitDir, nil
+	}
+
+	// If the path is not a directory, then it is a worktree.
+	// Read value for $GIT_DIR.
+	gitDirValue, err := readGitDir(root)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Resolve $GIT_DIR.
+	var gitDir vfs.Path
+	if filepath.IsAbs(gitDirValue) {
+		gitDir = vfs.MustNew(gitDirValue)
+	} else {
+		gitDir = vfs.MustNew(filepath.Join(root.Native(), gitDirValue))
+	}
+
+	// Read value for $GIT_COMMON_DIR.
+	gitCommonDirValue, err := readGitCommonDir(gitDir)
+	if err != nil {
+		return nil, nil, fmt.Errorf(`expected "commondir" file in worktree git folder at %q: %w`, gitDir.Native(), err)
+	}
+
+	// Resolve $GIT_COMMON_DIR.
+	var gitCommonDir vfs.Path
+	if filepath.IsAbs(gitCommonDirValue) {
+		gitCommonDir = vfs.MustNew(gitCommonDirValue)
+	} else {
+		gitCommonDir = vfs.MustNew(filepath.Join(gitDir.Native(), gitCommonDirValue))
+	}
+
+	return gitDir, gitCommonDir, nil
+}

libs/git/worktree_test.go

@@ -0,0 +1,108 @@
+package git
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/databricks/cli/libs/vfs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func setupWorktree(t *testing.T) string {
+	var err error
+
+	tmpDir := t.TempDir()
+
+	// Checkout path
+	err = os.MkdirAll(filepath.Join(tmpDir, "my_worktree"), os.ModePerm)
+	require.NoError(t, err)
+
+	// Main $GIT_COMMON_DIR
+	err = os.MkdirAll(filepath.Join(tmpDir, ".git"), os.ModePerm)
+	require.NoError(t, err)
+
+	// Worktree $GIT_DIR
+	err = os.MkdirAll(filepath.Join(tmpDir, ".git/worktrees/my_worktree"), os.ModePerm)
+	require.NoError(t, err)
+
+	return tmpDir
+}
+
+func writeGitDir(t *testing.T, dir, content string) {
+	err := os.WriteFile(filepath.Join(dir, "my_worktree/.git"), []byte(content), os.ModePerm)
+	require.NoError(t, err)
+}
+
+func writeGitCommonDir(t *testing.T, dir, content string) {
+	err := os.WriteFile(filepath.Join(dir, ".git/worktrees/my_worktree/commondir"), []byte(content), os.ModePerm)
+	require.NoError(t, err)
+}
+
+func verifyCorrectDirs(t *testing.T, dir string) {
+	gitDir, gitCommonDir, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree")))
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(dir, ".git/worktrees/my_worktree"), gitDir.Native())
+	assert.Equal(t, filepath.Join(dir, ".git"), gitCommonDir.Native())
+}
+
+func TestWorktreeResolveGitDir(t *testing.T) {
+	dir := setupWorktree(t)
+	writeGitCommonDir(t, dir, "../..")
+
+	t.Run("relative", func(t *testing.T) {
+		writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree"))
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("absolute", func(t *testing.T) {
+		writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", filepath.Join(dir, ".git/worktrees/my_worktree")))
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("additional spaces", func(t *testing.T) {
+		writeGitDir(t, dir, fmt.Sprintf("gitdir:    %s     \n\n\n", "../.git/worktrees/my_worktree"))
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		writeGitDir(t, dir, "")
+
+		_, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree")))
+		assert.ErrorContains(t, err, ` to contain a line with "gitdir: [...]"`)
+	})
+}
+
+func TestWorktreeResolveCommonDir(t *testing.T) {
+	dir := setupWorktree(t)
+	writeGitDir(t, dir, fmt.Sprintf("gitdir: %s", "../.git/worktrees/my_worktree"))
+
+	t.Run("relative", func(t *testing.T) {
+		writeGitCommonDir(t, dir, "../..")
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("absolute", func(t *testing.T) {
+		writeGitCommonDir(t, dir, filepath.Join(dir, ".git"))
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("additional spaces", func(t *testing.T) {
+		writeGitCommonDir(t, dir, "    ../..    \n\n\n")
+		verifyCorrectDirs(t, dir)
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		writeGitCommonDir(t, dir, "")
+
+		_, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree")))
+		assert.ErrorContains(t, err, `expected "commondir" file in worktree git folder at `)
+	})
+
+	t.Run("missing", func(t *testing.T) {
+		_, _, err := resolveGitDirs(vfs.MustNew(filepath.Join(dir, "my_worktree")))
+		assert.ErrorContains(t, err, `expected "commondir" file in worktree git folder at `)
+	})
+}