acc: add EnvMatrix config setting (#2500)

## Changes
- New option EnvMatrix to run the same test on various env vars

## Why
For integration tests, I'd like to test different DBR versions.
Currently there are two options:
- manually copy paste test in multiple variants
- use a loop in script

both are not great, maintenance-wise and performance-wise.

With this PR you can easily run test with different env vars and all
those variants will run in parallel.

## Tests
New selftest: acceptance/selftest/envmatrix.

Ran it manually with -norepl to see environment variables passed.
Run it with additional "sleep 10" to see that total time is still 10
seconds.

acceptance/acceptance_test.go

@@ -211,7 +211,21 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 				t.Parallel()
 			}
 
-			runTest(t, dir, coverDir, repls.Clone(), config, configPath)
+			expanded := internal.ExpandEnvMatrix(config.EnvMatrix)
+
+			if len(expanded) == 1 && len(expanded[0]) == 0 {
+				runTest(t, dir, coverDir, repls.Clone(), config, configPath, expanded[0])
+			} else {
+				for _, envset := range expanded {
+					envname := strings.Join(envset, "/")
+					t.Run(envname, func(t *testing.T) {
+						if !InprocessMode {
+							t.Parallel()
+						}
+						runTest(t, dir, coverDir, repls.Clone(), config, configPath, envset)
+					})
+				}
+			}
 		})
 	}
 
@@ -286,7 +300,7 @@ func getSkipReason(config *internal.TestConfig, configPath string) string {
 	return ""
 }
 
-func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext, config internal.TestConfig, configPath string) {
+func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext, config internal.TestConfig, configPath string, customEnv []string) {
 	tailOutput := Tail
 	cloudEnv := os.Getenv("CLOUD_ENV")
 	isRunningOnCloud := cloudEnv != ""
@@ -434,6 +448,13 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	for _, keyvalue := range customEnv {
+		items := strings.Split(keyvalue, "=")
+		require.Len(t, items, 2)
+		cmd.Env = append(cmd.Env, keyvalue)
+		repls.Set(items[1], "["+items[0]+"]")
+	}
+
 	absDir, err := filepath.Abs(dir)
 	require.NoError(t, err)
 	cmd.Env = append(cmd.Env, "TESTDIR="+absDir)

acceptance/internal/config.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"sort"
 	"strings"
 	"testing"
 
@@ -64,6 +65,15 @@ type TestConfig struct {
 	Ignore []string
 
 	CompiledIgnoreObject *ignore.GitIgnore
+
+	// Environment variables matrix.
+	// For each key you can specify zero, one or more values.
+	// If you specify zero, the key is omitted, as if it was not defined at all.
+	// Otherwise, for each value, you will get a new test with that environment variable
+	// set to that value (and replacement configured to match the value).
+	// If there are multiple variables defined, all combinations of tests are created,
+	// similar to github actions matrix strategy.
+	EnvMatrix map[string][]string
 }
 
 type ServerStub struct {
@@ -149,3 +159,60 @@ func DoLoadConfig(t *testing.T, path string) TestConfig {
 
 	return config
 }
+
+// This function takes EnvMatrix and expands into a slice of environment configurations.
+// Each environment configuration is a slice of env vars in standard Golang format.
+// For example,
+//
+//	input: {"KEY": ["A", "B"], "OTHER": ["VALUE"]}
+//
+// output: [["KEY=A", "OTHER=VALUE"], ["KEY=B", "OTHER=VALUE"]]
+//
+// If any entries is an empty list, that variable is dropped from the matrix before processing.
+func ExpandEnvMatrix(matrix map[string][]string) [][]string {
+	result := [][]string{{}}
+
+	if len(matrix) == 0 {
+		return result
+	}
+
+	// Filter out keys with empty value slices
+	filteredMatrix := make(map[string][]string)
+	for key, values := range matrix {
+		if len(values) > 0 {
+			filteredMatrix[key] = values
+		}
+	}
+
+	if len(filteredMatrix) == 0 {
+		return result
+	}
+
+	keys := make([]string, 0, len(filteredMatrix))
+	for key := range filteredMatrix {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	// Build an expansion of all combinations.
+	// At each step we look at a given key and append each possible value to each
+	// possible result accumulated up to this point.
+
+	for _, key := range keys {
+		values := filteredMatrix[key]
+		var newResult [][]string
+
+		for _, env := range result {
+			for _, value := range values {
+				newEnv := make([]string, len(env)+1)
+				copy(newEnv, env)
+				newEnv[len(env)] = key + "=" + value
+				newResult = append(newResult, newEnv)
+			}
+		}
+
+		result = newResult
+	}
+
+	return result
+}

acceptance/internal/config_test.go

@@ -0,0 +1,101 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExpandEnvMatrix(t *testing.T) {
+	tests := []struct {
+		name     string
+		matrix   map[string][]string
+		expected [][]string
+	}{
+		{
+			name:     "empty matrix",
+			matrix:   map[string][]string{},
+			expected: [][]string{{}},
+		},
+		{
+			name: "single key with single value",
+			matrix: map[string][]string{
+				"KEY": {"VALUE"},
+			},
+			expected: [][]string{
+				{"KEY=VALUE"},
+			},
+		},
+		{
+			name: "single key with multiple values",
+			matrix: map[string][]string{
+				"KEY": {"A", "B"},
+			},
+			expected: [][]string{
+				{"KEY=A"},
+				{"KEY=B"},
+			},
+		},
+		{
+			name: "multiple keys with single values",
+			matrix: map[string][]string{
+				"KEY1": {"VALUE1"},
+				"KEY2": {"VALUE2"},
+			},
+			expected: [][]string{
+				{"KEY1=VALUE1", "KEY2=VALUE2"},
+			},
+		},
+		{
+			name: "multiple keys with multiple values",
+			matrix: map[string][]string{
+				"KEY1": {"A", "B"},
+				"KEY2": {"C", "D"},
+			},
+			expected: [][]string{
+				{"KEY1=A", "KEY2=C"},
+				{"KEY1=A", "KEY2=D"},
+				{"KEY1=B", "KEY2=C"},
+				{"KEY1=B", "KEY2=D"},
+			},
+		},
+		{
+			name: "keys with empty values are filtered out",
+			matrix: map[string][]string{
+				"KEY1": {"A", "B"},
+				"KEY2": {},
+				"KEY3": {"C"},
+			},
+			expected: [][]string{
+				{"KEY1=A", "KEY3=C"},
+				{"KEY1=B", "KEY3=C"},
+			},
+		},
+		{
+			name: "all keys with empty values",
+			matrix: map[string][]string{
+				"KEY1": {},
+				"KEY2": {},
+			},
+			expected: [][]string{{}},
+		},
+		{
+			name: "example from documentation",
+			matrix: map[string][]string{
+				"KEY":   {"A", "B"},
+				"OTHER": {"VALUE"},
+			},
+			expected: [][]string{
+				{"KEY=A", "OTHER=VALUE"},
+				{"KEY=B", "OTHER=VALUE"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := ExpandEnvMatrix(tt.matrix)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}

acceptance/selftest/envmatrix/output.txt

@@ -0,0 +1,2 @@
+FIRST=[FIRST]
+SECOND=[SECOND]

acceptance/selftest/envmatrix/script

@@ -0,0 +1,2 @@
+echo "FIRST=$FIRST"
+echo "SECOND=$SECOND"

acceptance/selftest/envmatrix/test.toml

@@ -0,0 +1,3 @@
+[EnvMatrix]
+FIRST = ["one", "two"]
+SECOND = ["variantA", "variantB"]