From a5e09ab28a3728252672bcaac3ffd4cdffac36f7 Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Tue, 14 Jan 2025 15:19:00 +0100
Subject: [PATCH] Coverage for acceptance tests (#2123)

## Changes

Add two new make commands:
- make acc-cover: runs acceptance tests and outputs
coverage-acceptance.txt
- make acc-showcover: show coverage-acceptance.txt locally in browser

Using the GOCOVERDIR functionality:
https://go.dev/blog/integration-test-coverage

This works, but there are a couple of issues encountered:
- GOCOVERDIR does not play well with regular "go test -cover". Once this
fixed, we can simplify the code and have 'make cover' output coverage
for everything at once. We can also probably get rid of CLI_GOCOVERDIR.
https://github.com/golang/go/issues/66225
- When running tests in parallel to the same directory there is rare
conflict on writing covmeta file. For this reason each tests writes
coverage to their own directory which is then merged together by 'make
acc-cover'.

<!-- Summary of your changes that are easy to understand --


## Tests
Manually running the new make commands.
---
 .gitignore                    |  1 +
 Makefile                      | 13 ++++++++++++-
 acceptance/acceptance_test.go | 30 ++++++++++++++++++++++++++----
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index edd1409ae..274c1341a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ dist/
 
 *.log
 coverage.txt
+coverage-acceptance.txt
 
 __pycache__
 *.pyc
diff --git a/Makefile b/Makefile
index 2c84d88ba..4e5067b5d 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,17 @@ cover:
 showcover:
 	go tool cover -html=coverage.txt
 
+acc-cover:
+	rm -fr ./acceptance/build/cover/
+	CLI_GOCOVERDIR=build/cover go test ./acceptance
+	rm -fr ./acceptance/build/cover-merged/
+	mkdir -p acceptance/build/cover-merged/
+	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
+	go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
+
+acc-showcover:
+	go tool cover -html=coverage-acceptance.txt
+
 build: vendor
 	go build -mod vendor
 
@@ -45,4 +56,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 63dd95f1f..4b5146503 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -40,7 +40,16 @@ func TestAccept(t *testing.T) {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
-	execPath := BuildCLI(t, cwd)
+	coverDir := os.Getenv("CLI_GOCOVERDIR")
+
+	if coverDir != "" {
+		require.NoError(t, os.MkdirAll(coverDir, os.ModePerm))
+		coverDir, err = filepath.Abs(coverDir)
+		require.NoError(t, err)
+		t.Logf("Writing coverage to %s", coverDir)
+	}
+
+	execPath := BuildCLI(t, cwd, coverDir)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
@@ -76,10 +85,11 @@ func TestAccept(t *testing.T) {
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
+
 	for _, dir := range testDirs {
 		t.Run(dir, func(t *testing.T) {
 			t.Parallel()
-			runTest(t, dir, repls)
+			runTest(t, dir, coverDir, repls)
 		})
 	}
 }
@@ -104,7 +114,7 @@ func getTests(t *testing.T) []string {
 	return testDirs
 }
 
-func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
+func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
 	var tmpDir string
 	var err error
 	if KeepTmp {
@@ -127,6 +137,15 @@ func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
 
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.Command(args[0], args[1:]...)
+	if coverDir != "" {
+		// Creating individual coverage directory for each test, because writing to the same one
+		// results in sporadic failures like this one (only if tests are running in parallel):
+		// +error: coverage meta-data emit failed: writing ... rename .../tmp.covmeta.b3f... .../covmeta.b3f2c...: no such file or directory
+		coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
+		err := os.MkdirAll(coverDir, os.ModePerm)
+		require.NoError(t, err)
+		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
+	}
 	cmd.Dir = tmpDir
 	outB, err := cmd.CombinedOutput()
 
@@ -226,7 +245,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd string) string {
+func BuildCLI(t *testing.T, cwd, coverDir string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
@@ -234,6 +253,9 @@ func BuildCLI(t *testing.T, cwd string) string {
 
 	start := time.Now()
 	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	if coverDir != "" {
+		args = append(args, "-cover")
+	}
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Dir = ".."
 	out, err := cmd.CombinedOutput()