Include acceptance tests in integration tests (#2242)

## Changes
- Include acceptance directory in integration tests. Acceptance tests
will not start local server if CLOUD_ENV is set, so they become
integration tests.
- Add dependency for vendor to integration, so that CLI can be build
there.
- Implement LocalOnly option in test.toml to opt out of running
acceptance tests as integration tests. Use it in certain tests that are
difficult or not necessary to fix when run as integration tests.
- Update terraform test to redact timings out.
- Clean up .workspace.current_user from outputs of the tests.

## Tests
Existing tests.

Makefile

@@ -51,12 +51,12 @@ schema:
 docs:
 	go run ./bundle/docsgen ./bundle/internal/schema ./bundle/docsgen
 
-INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./integration/..." -- -parallel 4 -timeout=2h
+INTEGRATION = gotestsum --format github-actions --rerun-fails --jsonfile output.json --packages "./acceptance ./integration/..." -- -parallel 4 -timeout=2h
 
-integration:
+integration: vendor
 	$(INTEGRATION)
 
-integration-short:
+integration-short: vendor
 	$(INTEGRATION) -short
 
 .PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover docs

acceptance/acceptance_test.go

@@ -209,6 +209,11 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		t.Skipf("Disabled via GOOS.%s setting in %s", runtime.GOOS, configPath)
 	}
 
+	cloudEnv := os.Getenv("CLOUD_ENV")
+	if config.LocalOnly && cloudEnv != "" {
+		t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv)
+	}
+
 	var tmpDir string
 	var err error
 	if KeepTmp {

acceptance/bundle/scripts/test.toml

@@ -0,0 +1 @@
+LocalOnly = true  # Deployment currently fails when run locally; once that is fixed, remove this setting

acceptance/bundle/templates/test.toml

@@ -0,0 +1,2 @@
+# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on.
+LocalOnly = true

acceptance/bundle/variables/env_overrides/test.toml

@@ -0,0 +1,2 @@
+# Cloud run fails with Error: failed to resolve cluster-policy: wrong-cluster-policy, err: Policy named 'wrong-cluster-policy' does not exist
+LocalOnly = true

acceptance/bundle/variables/git-branch/output.txt

@@ -6,7 +6,7 @@
     "git": {
       "actual_branch": "main",
       "branch": "",
-      "bundle_root_path": ".",
+      "bundle_root_path": "."
     },
     "name": "git",
     "target": "prod",
@@ -28,11 +28,6 @@
   },
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/files",
     "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/prod/resources",
     "root_path": "/Workspace/Users/$USERNAME/.bundle/git/prod",
@@ -56,7 +51,7 @@ Validation OK!
     "git": {
       "actual_branch": "main",
       "branch": "dev-branch",
-      "bundle_root_path": ".",
+      "bundle_root_path": "."
     },
     "name": "git",
     "target": "dev",
@@ -78,11 +73,6 @@ Validation OK!
   },
   "workspace": {
     "artifact_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/files",
     "resource_path": "/Workspace/Users/$USERNAME/.bundle/git/dev/resources",
     "root_path": "/Workspace/Users/$USERNAME/.bundle/git/dev",

acceptance/bundle/variables/git-branch/script

@@ -1,6 +1,6 @@
 git-repo-init
-trace $CLI bundle validate -o json | grep -v '"commit"'
+trace $CLI bundle validate -o json | jq 'del(.workspace.current_user, .bundle.git.commit)'
 trace $CLI bundle validate
-trace $CLI bundle validate -o json -t dev | grep -v '"commit"'
-trace $CLI bundle validate -t dev | grep -v '"commit"'
+trace $CLI bundle validate -o json -t dev | jq 'del(.workspace.current_user, .bundle.git.commit)'
+trace $CLI bundle validate -t dev
 rm -fr .git

acceptance/bundle/variables/prepend-workspace-var/output.txt

@@ -53,15 +53,10 @@
   },
   "workspace": {
     "artifact_path": "/Users/$USERNAME/path/to/root/artifacts",
-    "current_user": {
-      "id": "$USER.Id",
-      "short_name": "$USERNAME",
-      "userName": "$USERNAME"
-    },
     "file_path": "/Users/$USERNAME/path/to/root/files",
     "profile": "profile_name",
     "resource_path": "/Users/$USERNAME/path/to/root/resources",
     "root_path": "/Users/$USERNAME/path/to/root",
     "state_path": "/Users/$USERNAME/path/to/root/state"
   }
-}
+}

acceptance/bundle/variables/prepend-workspace-var/script

@@ -1,2 +1,2 @@
 echo /Workspace should be prepended on all paths, but it is not the case: #2181
-$CLI bundle validate -o json
+$CLI bundle validate -o json | jq 'del(.workspace.current_user)'

acceptance/bundle/variables/resolve-builtin/output.txt

@@ -1,10 +1,5 @@
 {
   "artifact_path": "TestResolveVariableReferences/bar/artifacts",
-  "current_user": {
-    "id": "$USER.Id",
-    "short_name": "$USERNAME",
-    "userName": "$USERNAME"
-  },
   "file_path": "TestResolveVariableReferences/bar/baz",
   "resource_path": "TestResolveVariableReferences/bar/resources",
   "root_path": "TestResolveVariableReferences/bar",

acceptance/bundle/variables/resolve-builtin/script

@@ -1 +1 @@
-$CLI bundle validate -o json | jq .workspace
+$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)'

acceptance/bundle/variables/resolve-builtin/test.toml

@@ -0,0 +1,2 @@
+# Cloud run fails with Error: Path (TestResolveVariableReferences/bar/baz) doesn't start with '/'
+LocalOnly = true

acceptance/bundle/variables/resolve-vars-in-root-path/output.txt

@@ -1,10 +1,5 @@
 {
   "artifact_path": "TestResolveVariableReferencesToBundleVariables/bar/artifacts",
-  "current_user": {
-    "id": "$USER.Id",
-    "short_name": "$USERNAME",
-    "userName": "$USERNAME"
-  },
   "file_path": "TestResolveVariableReferencesToBundleVariables/bar/files",
   "resource_path": "TestResolveVariableReferencesToBundleVariables/bar/resources",
   "root_path": "TestResolveVariableReferencesToBundleVariables/bar",

acceptance/bundle/variables/resolve-vars-in-root-path/script

@@ -1 +1 @@
-$CLI bundle validate -o json | jq .workspace
+$CLI bundle validate -o json | jq .workspace | jq 'del(.current_user)'

acceptance/bundle/variables/resolve-vars-in-root-path/test.toml

@@ -0,0 +1,2 @@
+# Cloud run fails with Error: Path (TestResolveVariableReferencesToBundleVariables/bar/files) doesn't start with '/'
+LocalOnly = true

acceptance/config_test.go

@@ -26,6 +26,9 @@ type TestConfig struct {
 	// If absent, default to true.
 	GOOS map[string]bool
 
+	// If true, do not run this test against cloud environment
+	LocalOnly bool
+
 	// List of additional replacements to apply on this test.
 	// Old is a regexp, New is a replacement expression.
 	Repls []testdiff.Replacement

acceptance/terraform/output.txt

@@ -37,7 +37,7 @@ commands will detect it and remind you to do so if necessary.
 
 >>> $TERRAFORM plan -no-color
 data.databricks_current_user.me: Reading...
-data.databricks_current_user.me: Read complete after 0s [id=$USER.Id]
+data.databricks_current_user.me: Read complete after (redacted) [id=$USER.Id]
 
 Changes to Outputs:
   + username = "$USERNAME"

acceptance/terraform/test.toml

@@ -0,0 +1,3 @@
+[[Repls]]
+Old = 'Read complete after [^\s]+'
+New = 'Read complete after (redacted)'

acceptance/workspace/jobs/create/test.toml

@@ -1,3 +1,4 @@
+LocalOnly = true  # request recording currently does not work with cloud environment
 RecordRequests = true
 
 [[Server]]