Improve resolution of complex variables within complex variables (#2157)

## Changes
- Remove ResolveVariableReferencesInComplexVariables - it blocked
complex-within-complex for no good reason.
- Repeat regular resolution twice, it helps with a couple test cases we
have.

There may be a case for running it 3 times or more in a loop, but there
is no test case for that, so this PR is simple incremental improvement.

## Tests
Existing acceptance tests. Previously all unit tests for complex
variables were converted to acceptance tests, to capture this change and
ensure nothing breaks.

acceptance/bundle/variables/complex-transitive/output.txt

@@ -1,3 +1,3 @@
 {
-  "spark.databricks.sql.initial.catalog.name": "${var.catalog}"
+  "spark.databricks.sql.initial.catalog.name": "hive_metastore"
 }

acceptance/bundle/variables/complex-within-complex/output.txt

@@ -2,17 +2,16 @@ Warning: unknown field: node_type_id
   at resources.jobs.job1.job_clusters[0]
   in databricks.yml:25:11
 
-Error: complex variables cannot contain references to another complex variables
-
 [
   {
     "job_cluster_key": "my_cluster",
-    "new_cluster": null
-  },
-  {
-    "job_cluster_key": "my_cluster",
-    "new_cluster": "${var.cluster}"
+    "new_cluster": {
+      "node_type_id": "Standard_DS3_v2",
+      "num_workers": 2,
+      "spark_conf": {
+        "spark.executor.cores": "2",
+        "spark.executor.memory": "4g"
+      }
+    }
   }
 ]
-
-Exit code: 1

bundle/config/mutator/resolve_variable_references.go

@@ -32,19 +32,6 @@ func ResolveVariableReferencesInLookup() bundle.Mutator {
 	}, pattern: dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("lookup")), lookupFn: lookupForVariables}
 }
 
-func ResolveVariableReferencesInComplexVariables() bundle.Mutator {
-	return &resolveVariableReferences{
-		prefixes: []string{
-			"bundle",
-			"workspace",
-			"variables",
-		},
-		pattern:  dyn.NewPattern(dyn.Key("variables"), dyn.AnyKey(), dyn.Key("value")),
-		lookupFn: lookupForComplexVariables,
-		skipFn:   skipResolvingInNonComplexVariables,
-	}
-}
-
 func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
 		if path.String() == "workspace.file_path" {
@@ -57,38 +44,6 @@ func lookup(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	return dyn.GetByPath(v, path)
 }
 
-func lookupForComplexVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
-	if path[0].Key() != "variables" {
-		return lookup(v, path, b)
-	}
-
-	varV, err := dyn.GetByPath(v, path[:len(path)-1])
-	if err != nil {
-		return dyn.InvalidValue, err
-	}
-
-	var vv variable.Variable
-	err = convert.ToTyped(&vv, varV)
-	if err != nil {
-		return dyn.InvalidValue, err
-	}
-
-	if vv.Type == variable.VariableTypeComplex {
-		return dyn.InvalidValue, errors.New("complex variables cannot contain references to another complex variables")
-	}
-
-	return lookup(v, path, b)
-}
-
-func skipResolvingInNonComplexVariables(v dyn.Value) bool {
-	switch v.Kind() {
-	case dyn.KindMap, dyn.KindSequence:
-		return false
-	default:
-		return true
-	}
-}
-
 func lookupForVariables(v dyn.Value, path dyn.Path, b *bundle.Bundle) (dyn.Value, error) {
 	if path[0].Key() != "variables" {
 		return lookup(v, path, b)

bundle/phases/initialize.go

@@ -61,7 +61,11 @@ func Initialize() bundle.Mutator {
 			pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseApplyMutators),
 			mutator.ResolveVariableReferencesInLookup(),
 			mutator.ResolveResourceReferences(),
-			mutator.ResolveVariableReferencesInComplexVariables(),
+			mutator.ResolveVariableReferences(
+				"bundle",
+				"workspace",
+				"variables",
+			),
 			mutator.ResolveVariableReferences(
 				"bundle",
 				"workspace",