From a0455bcaefa488ca736b0baba0573da7dceeb30d Mon Sep 17 00:00:00 2001
From: Denis Bilenko <denis.bilenko@databricks.com>
Date: Thu, 9 Jan 2025 16:21:24 +0100
Subject: [PATCH] Migrate bundle/tests/undefined_resources_test.go to
 acceptance test (#2106)

Add sort_blocks.py helper to deal with non-determinism.
---
 acceptance/acceptance_test.go                 | 12 +++--
 acceptance/bin/sort_blocks.py                 | 21 ++++++++
 .../undefined_resources/databricks.yml        |  0
 .../bundle/undefined_resources/output.txt     | 19 +++++++
 acceptance/bundle/undefined_resources/script  |  2 +
 bundle/tests/undefined_resources_test.go      | 50 -------------------
 6 files changed, 50 insertions(+), 54 deletions(-)
 create mode 100755 acceptance/bin/sort_blocks.py
 rename {bundle/tests => acceptance/bundle}/undefined_resources/databricks.yml (100%)
 create mode 100644 acceptance/bundle/undefined_resources/output.txt
 create mode 100644 acceptance/bundle/undefined_resources/script
 delete mode 100644 bundle/tests/undefined_resources_test.go

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
index 759c0aeca..033f26dfb 100644
--- a/acceptance/acceptance_test.go
+++ b/acceptance/acceptance_test.go
@@ -35,10 +35,16 @@ var Scripts = map[string]bool{
 }
 
 func TestAccept(t *testing.T) {
-	execPath := BuildCLI(t)
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
+	execPath := BuildCLI(t, cwd)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
+	// Make helper scripts available
+	t.Setenv("PATH", fmt.Sprintf("%s%c%s", filepath.Join(cwd, "bin"), os.PathListSeparator, os.Getenv("PATH")))
+
 	server := StartServer(t)
 	AddHandlers(server)
 	// Redirect API access to local server:
@@ -199,9 +205,7 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T) string {
-	cwd, err := os.Getwd()
-	require.NoError(t, err)
+func BuildCLI(t *testing.T, cwd string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
 		execPath += ".exe"
diff --git a/acceptance/bin/sort_blocks.py b/acceptance/bin/sort_blocks.py
new file mode 100755
index 000000000..f50c6f50f
--- /dev/null
+++ b/acceptance/bin/sort_blocks.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Helper to sort blocks in text file. A block is a set of lines separated from others by empty line.
+
+This is to workaround non-determinism in the output.
+"""
+import sys
+
+blocks = []
+
+for line in sys.stdin:
+    if not line.strip():
+        if blocks and blocks[-1]:
+            blocks.append('')
+        continue
+    if not blocks:
+        blocks.append('')
+    blocks[-1] += line
+
+blocks.sort()
+print("\n".join(blocks))
diff --git a/bundle/tests/undefined_resources/databricks.yml b/acceptance/bundle/undefined_resources/databricks.yml
similarity index 100%
rename from bundle/tests/undefined_resources/databricks.yml
rename to acceptance/bundle/undefined_resources/databricks.yml
diff --git a/acceptance/bundle/undefined_resources/output.txt b/acceptance/bundle/undefined_resources/output.txt
new file mode 100644
index 000000000..29b51bc1a
--- /dev/null
+++ b/acceptance/bundle/undefined_resources/output.txt
@@ -0,0 +1,19 @@
+Error: experiment undefined-experiment is not defined
+  at resources.experiments.undefined-experiment
+  in databricks.yml:11:26
+
+Error: job undefined-job is not defined
+  at resources.jobs.undefined-job
+  in databricks.yml:6:19
+
+Error: pipeline undefined-pipeline is not defined
+  at resources.pipelines.undefined-pipeline
+  in databricks.yml:14:24
+
+Found 3 errors
+
+Name: undefined-job
+Target: default
+
+
+Exit code: 1
diff --git a/acceptance/bundle/undefined_resources/script b/acceptance/bundle/undefined_resources/script
new file mode 100644
index 000000000..10a3c485a
--- /dev/null
+++ b/acceptance/bundle/undefined_resources/script
@@ -0,0 +1,2 @@
+# We need sort_blocks.py because the order of diagnostics is currently randomized
+$CLI bundle validate 2>&1 | sort_blocks.py
diff --git a/bundle/tests/undefined_resources_test.go b/bundle/tests/undefined_resources_test.go
deleted file mode 100644
index 3dbacbc25..000000000
--- a/bundle/tests/undefined_resources_test.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package config_tests
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/databricks/cli/bundle"
-	"github.com/databricks/cli/bundle/config/validate"
-	"github.com/databricks/cli/libs/diag"
-	"github.com/databricks/cli/libs/dyn"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestUndefinedResourcesLoadWithError(t *testing.T) {
-	b := load(t, "./undefined_resources")
-	diags := bundle.Apply(context.Background(), b, validate.AllResourcesHaveValues())
-
-	assert.Len(t, diags, 3)
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "job undefined-job is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   6,
-			Column: 19,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.jobs.undefined-job")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "experiment undefined-experiment is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   11,
-			Column: 26,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.experiments.undefined-experiment")},
-	})
-	assert.Contains(t, diags, diag.Diagnostic{
-		Severity: diag.Error,
-		Summary:  "pipeline undefined-pipeline is not defined",
-		Locations: []dyn.Location{{
-			File:   filepath.FromSlash("undefined_resources/databricks.yml"),
-			Line:   14,
-			Column: 24,
-		}},
-		Paths: []dyn.Path{dyn.MustPathFromString("resources.pipelines.undefined-pipeline")},
-	})
-}