From d914a1b1e20843fd86415b9ca82772ab9aac1339 Mon Sep 17 00:00:00 2001
From: Andrew Nester <andrew.nester@databricks.com>
Date: Wed, 10 Apr 2024 11:55:02 +0200
Subject: [PATCH] Do not emit warning on YAML anchor blocks (#1354)

## Changes
In 0.217.0 we started to emit warning on unknown fields in YAML
configuration but wrongly considered YAML anchor blocks as unknown
field.

This PR fixes this by skipping normalising of YAML blocks.

## Tests
Added regression tests
---
 bundle/tests/loader.go                        | 15 +++++++++++---
 .../databricks.yml                            | 15 ++++++++++++++
 bundle/tests/yaml_anchors_test.go             | 12 +++++++++++
 libs/dyn/convert/normalize.go                 | 15 ++++++++------
 libs/dyn/convert/normalize_test.go            | 20 +++++++++++++++++++
 5 files changed, 68 insertions(+), 9 deletions(-)
 create mode 100644 bundle/tests/yaml_anchors_separate_block/databricks.yml

diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go
index e7cf18f7..8eddcf9a 100644
--- a/bundle/tests/loader.go
+++ b/bundle/tests/loader.go
@@ -7,6 +7,7 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config/mutator"
 	"github.com/databricks/cli/bundle/phases"
+	"github.com/databricks/cli/libs/diag"
 	"github.com/stretchr/testify/require"
 )
 
@@ -20,9 +21,18 @@ func load(t *testing.T, path string) *bundle.Bundle {
 }
 
 func loadTarget(t *testing.T, path, env string) *bundle.Bundle {
+	b, diags := loadTargetWithDiags(path, env)
+	require.NoError(t, diags.Error())
+	return b
+}
+
+func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) {
 	ctx := context.Background()
 	b, err := bundle.Load(ctx, path)
-	require.NoError(t, err)
+	if err != nil {
+		return nil, diag.FromErr(err)
+	}
+
 	diags := bundle.Apply(ctx, b, bundle.Seq(
 		phases.LoadNamedTarget(env),
 		mutator.RewriteSyncPaths(),
@@ -30,6 +40,5 @@ func loadTarget(t *testing.T, path, env string) *bundle.Bundle {
 		mutator.MergeJobTasks(),
 		mutator.MergePipelineClusters(),
 	))
-	require.NoError(t, diags.Error())
-	return b
+	return b, diags
 }
diff --git a/bundle/tests/yaml_anchors_separate_block/databricks.yml b/bundle/tests/yaml_anchors_separate_block/databricks.yml
new file mode 100644
index 00000000..447d5d0b
--- /dev/null
+++ b/bundle/tests/yaml_anchors_separate_block/databricks.yml
@@ -0,0 +1,15 @@
+bundle:
+  name: yaml_anchors_separate_block
+
+tags: &custom_tags
+  Tag1: "Value1"
+  Tag2: "Value2"
+  Tag3: "Value3"
+
+resources:
+  jobs:
+    my_job:
+      tasks:
+      - task_key: yaml_anchors_separate_block
+      tags:
+        <<: *custom_tags
diff --git a/bundle/tests/yaml_anchors_test.go b/bundle/tests/yaml_anchors_test.go
index 95cec30a..5c849705 100644
--- a/bundle/tests/yaml_anchors_test.go
+++ b/bundle/tests/yaml_anchors_test.go
@@ -19,6 +19,18 @@ func TestYAMLAnchors(t *testing.T) {
 	require.NotNil(t, t0)
 	require.NotNil(t, t1)
 
+	require.NotNil(t, t0.NewCluster)
+	require.NotNil(t, t1.NewCluster)
 	assert.Equal(t, "10.4.x-scala2.12", t0.NewCluster.SparkVersion)
 	assert.Equal(t, "10.4.x-scala2.12", t1.NewCluster.SparkVersion)
 }
+
+func TestYAMLAnchorsNoWarnings(t *testing.T) {
+	_, diags := loadTargetWithDiags("./yaml_anchors", "default")
+	assert.Empty(t, diags)
+}
+
+func TestYAMLAnchorsSeparateBlockNoWarnings(t *testing.T) {
+	_, diags := loadTargetWithDiags("./yaml_anchors_separate_block", "default")
+	assert.Empty(t, diags)
+}
diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go
index 296e2abb..b4bee977 100644
--- a/libs/dyn/convert/normalize.go
+++ b/libs/dyn/convert/normalize.go
@@ -89,14 +89,17 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen
 		for _, pair := range src.MustMap().Pairs() {
 			pk := pair.Key
 			pv := pair.Value
+
 			index, ok := info.Fields[pk.MustString()]
 			if !ok {
-				diags = diags.Append(diag.Diagnostic{
-					Severity: diag.Warning,
-					Summary:  fmt.Sprintf("unknown field: %s", pk.MustString()),
-					Location: pk.Location(),
-					Path:     path,
-				})
+				if !pv.IsAnchor() {
+					diags = diags.Append(diag.Diagnostic{
+						Severity: diag.Warning,
+						Summary:  fmt.Sprintf("unknown field: %s", pk.MustString()),
+						Location: pk.Location(),
+						Path:     path,
+					})
+				}
 				continue
 			}
 
diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go
index 133eaef8..1a0869a9 100644
--- a/libs/dyn/convert/normalize_test.go
+++ b/libs/dyn/convert/normalize_test.go
@@ -659,3 +659,23 @@ func TestNormalizeFloatError(t *testing.T) {
 		Path:     dyn.EmptyPath,
 	}, err[0])
 }
+
+func TestNormalizeAnchors(t *testing.T) {
+	type Tmp struct {
+		Foo string `json:"foo"`
+	}
+
+	var typ Tmp
+	vin := dyn.V(map[string]dyn.Value{
+		"foo":    dyn.V("bar"),
+		"anchor": dyn.V("anchor").MarkAnchor(),
+	})
+
+	vout, err := Normalize(typ, vin)
+	assert.Len(t, err, 0)
+
+	// The field that can be mapped to the struct field is retained.
+	assert.Equal(t, map[string]any{
+		"foo": "bar",
+	}, vout.AsAny())
+}