Add -inprocess mode for acceptance tests (#2184)

## Changes
- If you pass -inprocess flag to acceptance tests, they will run in the
same process as test itself. This enables debugging.
- If you set singleTest variable on top of acceptance_test.go, you'll
only run that test and with inprocess mode. This is intended for
debugging in VSCode.
- (minor) Converted KeepTmp to flag -keeptmp from env var KEEP_TMP for
consistency with other flags.

## Tests
- I verified that acceptance tests pass with -inprocess mode: `go test
-inprocess < /dev/null | cat`
- I verified that debugging in VSCode works: set a test name in
singleTest variable, set breakpoints inside CLI and click "debug test"
in VSCode.

acceptance/acceptance_test.go

@@ -3,6 +3,7 @@ package acceptance_test
 import (
 	"context"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -23,7 +24,22 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var KeepTmp = os.Getenv("KEEP_TMP") != ""
+var KeepTmp bool
+
+// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty"
+// Then install your breakpoints and click "debug test" near TestAccept in VSCODE.
+// example: var singleTest = "bundle/variables/empty"
+var singleTest = ""
+
+// If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs
+// CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py).
+// Also disables parallelism in tests.
+var InprocessMode bool
+
+func init() {
+	flag.BoolVar(&InprocessMode, "inprocess", singleTest != "", "Run CLI in the same process as test (for debugging)")
+	flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run")
+}
 
 const (
 	EntryPointScript = "script"
@@ -38,6 +54,23 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
+	testAccept(t, InprocessMode, "")
+}
+
+func TestInprocessMode(t *testing.T) {
+	if InprocessMode {
+		t.Skip("Already tested by TestAccept")
+	}
+	if runtime.GOOS == "windows" {
+		// -  catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
+		// +  catalogs                               A catalog is the first layer of Unity Catalog<6F>s three-level namespace.
+		t.Skip("Fails on CI on unicode characters")
+	}
+	require.NotZero(t, testAccept(t, true, "help"))
+}
+
+func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
+	repls := testdiff.ReplacementsContext{}
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
@@ -50,16 +83,22 @@ func TestAccept(t *testing.T) {
 		t.Logf("Writing coverage to %s", coverDir)
 	}
 
-	execPath := BuildCLI(t, cwd, coverDir)
+	execPath := ""
-	// $CLI is what test scripts are using
+
+	if InprocessMode {
+		cmdServer := StartCmdServer(t)
+		t.Setenv("CMD_SERVER_URL", cmdServer.URL)
+		execPath = filepath.Join(cwd, "bin", "callserver.py")
+	} else {
+		execPath = BuildCLI(t, cwd, coverDir)
+	}
+
 	t.Setenv("CLI", execPath)
+	repls.Set(execPath, "$CLI")
 
 	// Make helper scripts available
 	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
 
-	repls := testdiff.ReplacementsContext{}
-	repls.Set(execPath, "$CLI")
-
 	tempHomeDir := t.TempDir()
 	repls.Set(tempHomeDir, "$TMPHOME")
 	t.Logf("$TMPHOME=%v", tempHomeDir)
@@ -95,13 +134,25 @@ func TestAccept(t *testing.T) {
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
 
+	if singleTest != "" {
+		testDirs = slices.DeleteFunc(testDirs, func(n string) bool {
+			return n != singleTest
+		})
+		require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs)
+	}
+
 	for _, dir := range testDirs {
 		testName := strings.ReplaceAll(dir, "\\", "/")
 		t.Run(testName, func(t *testing.T) {
-			t.Parallel()
+			if !InprocessMode {
+				t.Parallel()
+			}
+
 			runTest(t, dir, coverDir, repls.Clone())
 		})
 	}
+
+	return len(testDirs)
 }
 
 func getTests(t *testing.T) []string {

acceptance/bin/callserver.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import urllib.request
+from urllib.parse import urlencode
+
+env = {}
+for key, value in os.environ.items():
+    if len(value) > 10_000:
+        sys.stderr.write(f"Dropping key={key} value len={len(value)}\n")
+        continue
+    env[key] = value
+
+q = {
+    "args": " ".join(sys.argv[1:]),
+    "cwd": os.getcwd(),
+    "env": json.dumps(env),
+}
+
+url = os.environ["CMD_SERVER_URL"] + "/?" + urlencode(q)
+if len(url) > 100_000:
+    sys.exit("url too large")
+
+resp = urllib.request.urlopen(url)
+assert resp.status == 200, (resp.status, resp.url, resp.headers)
+result = json.load(resp)
+sys.stderr.write(result["stderr"])
+sys.stdout.write(result["stdout"])
+exitcode = int(result["exitcode"])
+sys.exit(exitcode)

acceptance/cmd_server_test.go

@@ -0,0 +1,73 @@
+package acceptance_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/databricks/cli/internal/testcli"
+	"github.com/stretchr/testify/require"
+)
+
+func StartCmdServer(t *testing.T) *TestServer {
+	server := StartServer(t)
+	server.Handle("/", func(r *http.Request) (any, error) {
+		q := r.URL.Query()
+		args := strings.Split(q.Get("args"), " ")
+
+		var env map[string]string
+		require.NoError(t, json.Unmarshal([]byte(q.Get("env")), &env))
+
+		for key, val := range env {
+			defer Setenv(t, key, val)()
+		}
+
+		defer Chdir(t, q.Get("cwd"))()
+
+		c := testcli.NewRunner(t, r.Context(), args...)
+		c.Verbose = false
+		stdout, stderr, err := c.Run()
+		result := map[string]any{
+			"stdout": stdout.String(),
+			"stderr": stderr.String(),
+		}
+		exitcode := 0
+		if err != nil {
+			exitcode = 1
+		}
+		result["exitcode"] = exitcode
+		return result, nil
+	})
+	return server
+}
+
+// Chdir variant that is intended to be used with defer so that it can switch back before function ends.
+// This is unlike testutil.Chdir which switches back only when tests end.
+func Chdir(t *testing.T, cwd string) func() {
+	require.NotEmpty(t, cwd)
+	prevDir, err := os.Getwd()
+	require.NoError(t, err)
+	err = os.Chdir(cwd)
+	require.NoError(t, err)
+	return func() {
+		_ = os.Chdir(prevDir)
+	}
+}
+
+// Setenv variant that is intended to be used with defer so that it can switch back before function ends.
+// This is unlike t.Setenv which switches back only when tests end.
+func Setenv(t *testing.T, key, value string) func() {
+	prevVal, exists := os.LookupEnv(key)
+
+	require.NoError(t, os.Setenv(key, value))
+
+	return func() {
+		if exists {
+			_ = os.Setenv(key, prevVal)
+		} else {
+			_ = os.Unsetenv(key)
+		}
+	}
+}