From 3a32c63919d06408d37557b651b6d55e4fc71247 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 21 Jan 2025 22:21:12 +0100 Subject: [PATCH] Add -inprocess mode for acceptance tests (#2184) ## Changes - If you pass -inprocess flag to acceptance tests, they will run in the same process as test itself. This enables debugging. - If you set singleTest variable on top of acceptance_test.go, you'll only run that test and with inprocess mode. This is intended for debugging in VSCode. - (minor) Converted KeepTmp to flag -keeptmp from env var KEEP_TMP for consistency with other flags. ## Tests - I verified that acceptance tests pass with -inprocess mode: `go test -inprocess < /dev/null | cat` - I verified that debugging in VSCode works: set a test name in singleTest variable, set breakpoints inside CLI and click "debug test" in VSCode. --- acceptance/acceptance_test.go | 65 +++++++++++++++++++++++++++---- acceptance/bin/callserver.py | 31 +++++++++++++++ acceptance/cmd_server_test.go | 73 +++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) create mode 100755 acceptance/bin/callserver.py create mode 100644 acceptance/cmd_server_test.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 12fe6536f..cfcb0d29f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -3,6 +3,7 @@ package acceptance_test import ( "context" "errors" + "flag" "fmt" "io" "os" @@ -23,7 +24,22 @@ import ( "github.com/stretchr/testify/require" ) -var KeepTmp = os.Getenv("KEEP_TMP") != "" +var KeepTmp bool + +// In order to debug CLI running under acceptance test, set this to full subtest name, e.g. "bundle/variables/empty" +// Then install your breakpoints and click "debug test" near TestAccept in VSCODE. +// example: var singleTest = "bundle/variables/empty" +var singleTest = "" + +// If enabled, instead of compiling and running CLI externally, we'll start in-process server that accepts and runs +// CLI commands. The $CLI in test scripts is a helper that just forwards command-line arguments to this server (see bin/callserver.py). +// Also disables parallelism in tests. +var InprocessMode bool + +func init() { + flag.BoolVar(&InprocessMode, "inprocess", singleTest != "", "Run CLI in the same process as test (for debugging)") + flag.BoolVar(&KeepTmp, "keeptmp", false, "Do not delete TMP directory after run") +} const ( EntryPointScript = "script" @@ -38,6 +54,23 @@ var Scripts = map[string]bool{ } func TestAccept(t *testing.T) { + testAccept(t, InprocessMode, "") +} + +func TestInprocessMode(t *testing.T) { + if InprocessMode { + t.Skip("Already tested by TestAccept") + } + if runtime.GOOS == "windows" { + // - catalogs A catalog is the first layer of Unity Catalog’s three-level namespace. + // + catalogs A catalog is the first layer of Unity Catalog�s three-level namespace. + t.Skip("Fails on CI on unicode characters") + } + require.NotZero(t, testAccept(t, true, "help")) +} + +func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { + repls := testdiff.ReplacementsContext{} cwd, err := os.Getwd() require.NoError(t, err) @@ -50,16 +83,22 @@ func TestAccept(t *testing.T) { t.Logf("Writing coverage to %s", coverDir) } - execPath := BuildCLI(t, cwd, coverDir) - // $CLI is what test scripts are using + execPath := "" + + if InprocessMode { + cmdServer := StartCmdServer(t) + t.Setenv("CMD_SERVER_URL", cmdServer.URL) + execPath = filepath.Join(cwd, "bin", "callserver.py") + } else { + execPath = BuildCLI(t, cwd, coverDir) + } + t.Setenv("CLI", execPath) + repls.Set(execPath, "$CLI") // Make helper scripts available t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH"))) - repls := testdiff.ReplacementsContext{} - repls.Set(execPath, "$CLI") - tempHomeDir := t.TempDir() repls.Set(tempHomeDir, "$TMPHOME") t.Logf("$TMPHOME=%v", tempHomeDir) @@ -95,13 +134,25 @@ func TestAccept(t *testing.T) { testDirs := getTests(t) require.NotEmpty(t, testDirs) + if singleTest != "" { + testDirs = slices.DeleteFunc(testDirs, func(n string) bool { + return n != singleTest + }) + require.NotEmpty(t, testDirs, "singleTest=%#v did not match any tests\n%#v", singleTest, testDirs) + } + for _, dir := range testDirs { testName := strings.ReplaceAll(dir, "\\", "/") t.Run(testName, func(t *testing.T) { - t.Parallel() + if !InprocessMode { + t.Parallel() + } + runTest(t, dir, coverDir, repls.Clone()) }) } + + return len(testDirs) } func getTests(t *testing.T) []string { diff --git a/acceptance/bin/callserver.py b/acceptance/bin/callserver.py new file mode 100755 index 000000000..294ef8fdb --- /dev/null +++ b/acceptance/bin/callserver.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import urllib.request +from urllib.parse import urlencode + +env = {} +for key, value in os.environ.items(): + if len(value) > 10_000: + sys.stderr.write(f"Dropping key={key} value len={len(value)}\n") + continue + env[key] = value + +q = { + "args": " ".join(sys.argv[1:]), + "cwd": os.getcwd(), + "env": json.dumps(env), +} + +url = os.environ["CMD_SERVER_URL"] + "/?" + urlencode(q) +if len(url) > 100_000: + sys.exit("url too large") + +resp = urllib.request.urlopen(url) +assert resp.status == 200, (resp.status, resp.url, resp.headers) +result = json.load(resp) +sys.stderr.write(result["stderr"]) +sys.stdout.write(result["stdout"]) +exitcode = int(result["exitcode"]) +sys.exit(exitcode) diff --git a/acceptance/cmd_server_test.go b/acceptance/cmd_server_test.go new file mode 100644 index 000000000..28feec1bd --- /dev/null +++ b/acceptance/cmd_server_test.go @@ -0,0 +1,73 @@ +package acceptance_test + +import ( + "encoding/json" + "net/http" + "os" + "strings" + "testing" + + "github.com/databricks/cli/internal/testcli" + "github.com/stretchr/testify/require" +) + +func StartCmdServer(t *testing.T) *TestServer { + server := StartServer(t) + server.Handle("/", func(r *http.Request) (any, error) { + q := r.URL.Query() + args := strings.Split(q.Get("args"), " ") + + var env map[string]string + require.NoError(t, json.Unmarshal([]byte(q.Get("env")), &env)) + + for key, val := range env { + defer Setenv(t, key, val)() + } + + defer Chdir(t, q.Get("cwd"))() + + c := testcli.NewRunner(t, r.Context(), args...) + c.Verbose = false + stdout, stderr, err := c.Run() + result := map[string]any{ + "stdout": stdout.String(), + "stderr": stderr.String(), + } + exitcode := 0 + if err != nil { + exitcode = 1 + } + result["exitcode"] = exitcode + return result, nil + }) + return server +} + +// Chdir variant that is intended to be used with defer so that it can switch back before function ends. +// This is unlike testutil.Chdir which switches back only when tests end. +func Chdir(t *testing.T, cwd string) func() { + require.NotEmpty(t, cwd) + prevDir, err := os.Getwd() + require.NoError(t, err) + err = os.Chdir(cwd) + require.NoError(t, err) + return func() { + _ = os.Chdir(prevDir) + } +} + +// Setenv variant that is intended to be used with defer so that it can switch back before function ends. +// This is unlike t.Setenv which switches back only when tests end. +func Setenv(t *testing.T, key, value string) func() { + prevVal, exists := os.LookupEnv(key) + + require.NoError(t, os.Setenv(key, value)) + + return func() { + if exists { + _ = os.Setenv(key, prevVal) + } else { + _ = os.Unsetenv(key) + } + } +}