Migrate bundle/tests/undefined_resources_test.go to acceptance test (#2106)

Add sort_blocks.py helper to deal with non-determinism.

acceptance/acceptance_test.go

@@ -35,10 +35,16 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
-	execPath := BuildCLI(t)
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	execPath := BuildCLI(t, cwd)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
+	// Make helper scripts available
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
+
 	server := StartServer(t)
 	AddHandlers(server)
 	// Redirect API access to local server:
@@ -199,9 +205,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T) string {
+func BuildCLI(t *testing.T, cwd string) string {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"

acceptance/bin/sort_blocks.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
+
+This is to workaround non-determinism in the output.
+"""
+import sys
+
+blocks = []
+
+for line in sys.stdin:
+    if not line.strip():
+        if blocks and blocks[-1]:
+            blocks.append('')
+        continue
+    if not blocks:
+        blocks.append('')
+    blocks[-1] += line
+
+blocks.sort()
+print("\n".join(blocks))

acceptance/bundle/undefined_resources/output.txt

@@ -0,0 +1,19 @@
+Error: experiment undefined-experiment is not defined
+  at resources.experiments.undefined-experiment
+  in databricks.yml:11:26
+
+Error: job undefined-job is not defined
+  at resources.jobs.undefined-job
+  in databricks.yml:6:19
+
+Error: pipeline undefined-pipeline is not defined
+  at resources.pipelines.undefined-pipeline
+  in databricks.yml:14:24
+
+Found 3 errors
+
+Name: undefined-job
+Target: default
+
+
+Exit code: 1

acceptance/bundle/undefined_resources/script

@@ -0,0 +1,2 @@
+# We need sort_blocks.py because the order of diagnostics is currently randomized
+$CLI bundle validate 2>&1 | sort_blocks.py

bundle/tests/undefined_resources_test.go

@@ -1,50 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/validate"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUndefinedResourcesLoadWithError(t *testing.T) {
-	b := load(t, "./undefined_resources")
-	diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues())
-
-	assert.Len(t, diags, 3)
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "job undefined-job is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   6,
-			Column: 19,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.undefined-job")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "experiment undefined-experiment is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   11,
-			Column: 26,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.experiments.undefined-experiment")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "pipeline undefined-pipeline is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   14,
-			Column: 24,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.undefined-pipeline")},
-	})
-}