Merge remote-tracking branch 'origin' into pipeline-errors

bundle/schema/README.md

@@ -0,0 +1,26 @@
+### Overview
+
+`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
+descriptions for the json schema. Specifically
+1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
+2. `environments` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
+3. `bundle` : manually editted
+4. `include` : manually editted
+5. `workspace` : manually editted
+6. `artifacts` : manually editted
+
+These descriptions are rendered in the inline documentation in an IDE
+
+### SOP: Add schema descriptions for new fields in bundle config
+
+1. You can autogenerate empty descriptions for the new fields by running
+`bricks bundle schema --only-docs > ~/bricks/bundle/schema/docs/bundle_descriptions.json`
+2. Manually edit bundle_descriptions.json to add your descriptions
+3. Again run `bricks bundle schema --only-docs > ~/bricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `environments`
+4. push to repo
+
+
+### SOP: Update descriptions in resources from a newer openapi spec
+
+1. Run `bricks bundle schema --only-docs --openapi PATH_TO_SPEC > ~/bricks/bundle/schema/docs/bundle_descriptions.json`
+2. push to repo

bundle/schema/bundle_config_docs.yml

@@ -1,12 +0,0 @@
-documentation: Root of the bundle config
-children:
-  bundle:
-    documentation: |
-      Bundle contains details about this bundle, such as its name,
-      version of the spec (TODO), default cluster, default warehouse, etc.
-    children:
-      environment:
-        documentation: Environment is set by the mutator that selects the environment.
-
-  artifacts:
-    documentation: Artifacts contains a description of all code artifacts in this bundle.

bundle/schema/docs.go

@@ -2,37 +2,109 @@ package schema
 
 import (
 	_ "embed"
+	"encoding/json"
+	"fmt"
 	"os"
+	"reflect"
 
-	"gopkg.in/yaml.v3"
+	"github.com/databricks/bricks/bundle/config"
+	"github.com/databricks/databricks-sdk-go/openapi"
 )
 
+// A subset of Schema struct
 type Docs struct {
-	Documentation string          `json:"documentation"`
+	Description          string           `json:"description"`
-	Children      map[string]Docs `json:"children"`
+	Properties           map[string]*Docs `json:"properties,omitempty"`
+	Items                *Docs            `json:"items,omitempty"`
+	AdditionalProperties *Docs            `json:"additionalproperties,omitempty"`
 }
 
-func LoadDocs(path string) (*Docs, error) {
+//go:embed docs/bundle_descriptions.json
-	bytes, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	docs := Docs{}
-	err = yaml.Unmarshal(bytes, &docs)
-	if err != nil {
-		return nil, err
-	}
-	return &docs, nil
-}
-
-//go:embed bundle_config_docs.yml
 var bundleDocs []byte
 
-func GetBundleDocs() (*Docs, error) {
+func BundleDocs(openapiSpecPath string) (*Docs, error) {
-	docs := Docs{}
+	docs, err := initializeBundleDocs()
-	err := yaml.Unmarshal(bundleDocs, &docs)
 	if err != nil {
 		return nil, err
 	}
-	return &docs, nil
+	if openapiSpecPath != "" {
+		openapiSpec, err := os.ReadFile(openapiSpecPath)
+		if err != nil {
+			return nil, err
+		}
+		spec := &openapi.Specification{}
+		err = json.Unmarshal(openapiSpec, spec)
+		if err != nil {
+			return nil, err
+		}
+		openapiReader := &OpenapiReader{
+			OpenapiSpec: spec,
+			Memo:        make(map[string]*Schema),
+		}
+		resourcesDocs, err := openapiReader.ResourcesDocs()
+		if err != nil {
+			return nil, err
+		}
+		resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
+		if err != nil {
+			return nil, err
+		}
+		docs.Properties["resources"] = schemaToDocs(resourceSchema)
+	}
+	docs.refreshEnvironmentsDocs()
+	return docs, nil
+}
+
+func (docs *Docs) refreshEnvironmentsDocs() error {
+	environmentsDocs, ok := docs.Properties["environments"]
+	if !ok || environmentsDocs.AdditionalProperties == nil ||
+		environmentsDocs.AdditionalProperties.Properties == nil {
+		return fmt.Errorf("invalid environments descriptions")
+	}
+	environmentProperties := environmentsDocs.AdditionalProperties.Properties
+	propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
+	for _, p := range propertiesToCopy {
+		environmentProperties[p] = docs.Properties[p]
+	}
+	return nil
+}
+
+func initializeBundleDocs() (*Docs, error) {
+	// load embedded descriptions
+	embedded := Docs{}
+	err := json.Unmarshal(bundleDocs, &embedded)
+	if err != nil {
+		return nil, err
+	}
+	// generate schema with the embedded descriptions
+	schema, err := New(reflect.TypeOf(config.Root{}), &embedded)
+	if err != nil {
+		return nil, err
+	}
+	// converting the schema back to docs. This creates empty descriptions
+	// for any properties that were missing in the embedded descriptions
+	docs := schemaToDocs(schema)
+	return docs, nil
+}
+
+// *Docs are a subset of *Schema, this function selects that subset
+func schemaToDocs(schema *Schema) *Docs {
+	// terminate recursion if schema is nil
+	if schema == nil {
+		return nil
+	}
+	docs := &Docs{
+		Description: schema.Description,
+	}
+	if len(schema.Properties) > 0 {
+		docs.Properties = make(map[string]*Docs)
+	}
+	for k, v := range schema.Properties {
+		docs.Properties[k] = schemaToDocs(v)
+	}
+	docs.Items = schemaToDocs(schema.Items)
+	if additionalProperties, ok := schema.AdditionalProperties.(*Schema); ok {
+		docs.AdditionalProperties = schemaToDocs(additionalProperties)
+	}
+	return docs
 }

bundle/schema/docs_test.go

@@ -0,0 +1,61 @@
+package schema
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSchemaToDocs(t *testing.T) {
+	schema := &Schema{
+		Type:        "object",
+		Description: "root doc",
+		Properties: map[string]*Schema{
+			"foo": {Type: "number", Description: "foo doc"},
+			"bar": {Type: "string"},
+			"octave": {
+				Type:                 "object",
+				AdditionalProperties: &Schema{Type: "number"},
+				Description:          "octave docs",
+			},
+			"scales": {
+				Type:        "object",
+				Description: "scale docs",
+				Items:       &Schema{Type: "string"},
+			},
+		},
+	}
+	docs := schemaToDocs(schema)
+	docsJson, err := json.MarshalIndent(docs, "		", "	")
+	require.NoError(t, err)
+
+	expected :=
+		`{
+			"description": "root doc",
+			"properties": {
+				"bar": {
+					"description": ""
+				},
+				"foo": {
+					"description": "foo doc"
+				},
+				"octave": {
+					"description": "octave docs",
+					"additionalproperties": {
+						"description": ""
+					}
+				},
+				"scales": {
+					"description": "scale docs",
+					"items": {
+						"description": ""
+					}
+				}
+			}
+		}`
+	t.Log("[DEBUG] actual: ", string(docsJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(docsJson))
+}

bundle/schema/openapi.go

@@ -0,0 +1,214 @@
+package schema
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/databricks/databricks-sdk-go/openapi"
+)
+
+type OpenapiReader struct {
+	OpenapiSpec *openapi.Specification
+	Memo        map[string]*Schema
+}
+
+const SchemaPathPrefix = "#/components/schemas/"
+
+func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) {
+	schemaKey := strings.TrimPrefix(path, SchemaPathPrefix)
+
+	// return early if we already have a computed schema
+	memoSchema, ok := reader.Memo[schemaKey]
+	if ok {
+		return memoSchema, nil
+	}
+
+	// check path is present in openapi spec
+	openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey]
+	if !ok {
+		return nil, fmt.Errorf("schema with path %s not found in openapi spec", path)
+	}
+
+	// convert openapi schema to the native schema struct
+	bytes, err := json.Marshal(*openapiSchema)
+	if err != nil {
+		return nil, err
+	}
+	jsonSchema := &Schema{}
+	err = json.Unmarshal(bytes, jsonSchema)
+	if err != nil {
+		return nil, err
+	}
+
+	// A hack to convert a map[string]interface{} to *Schema
+	// We rely on the type of a AdditionalProperties in downstream functions
+	// to do reference interpolation
+	_, ok = jsonSchema.AdditionalProperties.(map[string]interface{})
+	if ok {
+		b, err := json.Marshal(jsonSchema.AdditionalProperties)
+		if err != nil {
+			return nil, err
+		}
+		additionalProperties := &Schema{}
+		err = json.Unmarshal(b, additionalProperties)
+		if err != nil {
+			return nil, err
+		}
+		jsonSchema.AdditionalProperties = additionalProperties
+	}
+
+	// store read schema into memo
+	reader.Memo[schemaKey] = jsonSchema
+
+	return jsonSchema, nil
+}
+
+// safe againt loops in refs
+func (reader *OpenapiReader) safeResolveRefs(root *Schema, seenRefs map[string]struct{}) (*Schema, error) {
+	if root.Reference == nil {
+		return reader.traverseSchema(root, seenRefs)
+	}
+	key := *root.Reference
+	_, ok := seenRefs[key]
+	if ok {
+		// self reference loops can be supported however the logic is non-trivial because
+		// cross refernce loops are not allowed (see: http://json-schema.org/understanding-json-schema/structuring.html#recursion)
+		return nil, fmt.Errorf("references loop detected")
+	}
+	ref := *root.Reference
+	description := root.Description
+	seenRefs[ref] = struct{}{}
+
+	// Mark reference nil, so we do not traverse this again. This is tracked
+	// in the memo
+	root.Reference = nil
+
+	// unroll one level of reference
+	selfRef, err := reader.readOpenapiSchema(ref)
+	if err != nil {
+		return nil, err
+	}
+	root = selfRef
+	root.Description = description
+
+	// traverse again to find new references
+	root, err = reader.traverseSchema(root, seenRefs)
+	if err != nil {
+		return nil, err
+	}
+	delete(seenRefs, ref)
+	return root, err
+}
+
+func (reader *OpenapiReader) traverseSchema(root *Schema, seenRefs map[string]struct{}) (*Schema, error) {
+	// case primitive (or invalid)
+	if root.Type != Object && root.Type != Array {
+		return root, nil
+	}
+	// only root references are resolved
+	if root.Reference != nil {
+		return reader.safeResolveRefs(root, seenRefs)
+	}
+	// case struct
+	if len(root.Properties) > 0 {
+		for k, v := range root.Properties {
+			childSchema, err := reader.safeResolveRefs(v, seenRefs)
+			if err != nil {
+				return nil, err
+			}
+			root.Properties[k] = childSchema
+		}
+	}
+	// case array
+	if root.Items != nil {
+		itemsSchema, err := reader.safeResolveRefs(root.Items, seenRefs)
+		if err != nil {
+			return nil, err
+		}
+		root.Items = itemsSchema
+	}
+	// case map
+	additionionalProperties, ok := root.AdditionalProperties.(*Schema)
+	if ok && additionionalProperties != nil {
+		valueSchema, err := reader.safeResolveRefs(additionionalProperties, seenRefs)
+		if err != nil {
+			return nil, err
+		}
+		root.AdditionalProperties = valueSchema
+	}
+	return root, nil
+}
+
+func (reader *OpenapiReader) readResolvedSchema(path string) (*Schema, error) {
+	root, err := reader.readOpenapiSchema(path)
+	if err != nil {
+		return nil, err
+	}
+	seenRefs := make(map[string]struct{})
+	seenRefs[path] = struct{}{}
+	root, err = reader.safeResolveRefs(root, seenRefs)
+	if err != nil {
+		trace := ""
+		count := 0
+		for k := range seenRefs {
+			if count == len(seenRefs)-1 {
+				trace += k
+				break
+			}
+			trace += k + " -> "
+			count++
+		}
+		return nil, fmt.Errorf("%s. schema ref trace: %s", err, trace)
+	}
+	return root, nil
+}
+
+func (reader *OpenapiReader) jobsDocs() (*Docs, error) {
+	jobSettingsSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "jobs.JobSettings")
+	if err != nil {
+		return nil, err
+	}
+	jobDocs := schemaToDocs(jobSettingsSchema)
+	// TODO: add description for id if needed.
+	// Tracked in https://github.com/databricks/bricks/issues/242
+	jobsDocs := &Docs{
+		Description:          "List of job definations",
+		AdditionalProperties: jobDocs,
+	}
+	return jobsDocs, nil
+}
+
+func (reader *OpenapiReader) pipelinesDocs() (*Docs, error) {
+	pipelineSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "pipelines.PipelineSpec")
+	if err != nil {
+		return nil, err
+	}
+	pipelineDocs := schemaToDocs(pipelineSpecSchema)
+	// TODO: Two fields in resources.Pipeline have the json tag id. Clarify the
+	// semantics and then add a description if needed. (https://github.com/databricks/bricks/issues/242)
+	pipelinesDocs := &Docs{
+		Description:          "List of pipeline definations",
+		AdditionalProperties: pipelineDocs,
+	}
+	return pipelinesDocs, nil
+}
+
+func (reader *OpenapiReader) ResourcesDocs() (*Docs, error) {
+	jobsDocs, err := reader.jobsDocs()
+	if err != nil {
+		return nil, err
+	}
+	pipelinesDocs, err := reader.pipelinesDocs()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Docs{
+		Description: "Specification of databricks resources to instantiate",
+		Properties: map[string]*Docs{
+			"jobs":      jobsDocs,
+			"pipelines": pipelinesDocs,
+		},
+	}, nil
+}

bundle/schema/openapi_test.go

@@ -0,0 +1,435 @@
+package schema
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/openapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadSchemaForObject(t *testing.T) {
+	specString := `
+	{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "number"
+				},
+				"fruits": {
+					"type": "object",
+					"description": "fruits that are cool",
+					"properties": {
+						"guava": {
+							"type": "string",
+							"description": "a guava for my schema"
+						},
+						"mango": {
+							"type": "object",
+							"description": "a mango for my schema",
+							"$ref": "#/components/schemas/mango"
+						}
+					}
+				},
+				"mango": {
+					"type": "object",
+					"properties": {
+						"foo": {
+							"$ref": "#/components/schemas/foo"
+						}
+					}
+				}
+			}
+		}
+	}
+	`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "fruits that are cool",
+			"properties": {
+				"guava": {
+					"type": "string",
+					"description": "a guava for my schema"
+				},
+				"mango": {
+					"type": "object",
+					"description": "a mango for my schema",
+					"properties": {
+						"foo": {
+							"type": "number"
+						}
+					}
+				}
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}
+
+func TestReadSchemaForArray(t *testing.T) {
+	specString := `
+	{
+		"components": {
+			"schemas": {
+				"fruits": {
+					"type": "object",
+					"description": "fruits that are cool",
+					"items": {
+						"description": "some papayas, because papayas are fruits too",
+						"$ref": "#/components/schemas/papaya"
+					}
+				},
+				"papaya": {
+					"type": "number"
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "fruits that are cool",
+			"items": {
+				"type": "number",
+				"description": "some papayas, because papayas are fruits too"
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}
+
+func TestReadSchemaForMap(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"fruits": {
+					"type": "object",
+					"description": "fruits that are meh",
+					"additionalProperties": {
+						"description": "watermelons. watermelons.",
+						"$ref": "#/components/schemas/watermelon"
+					}
+				},
+				"watermelon": {
+					"type": "number"
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "fruits that are meh",
+			"additionalProperties": {
+				"type": "number",
+				"description": "watermelons. watermelons."
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}
+
+func TestRootReferenceIsResolved(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "object",
+					"description": "this description is ignored",
+					"properties": {
+						"abc": {
+							"type": "string"
+						}
+					}
+				},
+				"fruits": {
+					"type": "object",
+					"description": "foo fighters fighting fruits",
+					"$ref": "#/components/schemas/foo"
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	schema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+	fruitsSchemaJson, err := json.MarshalIndent(schema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "foo fighters fighting fruits",
+			"properties": {
+				"abc": {
+					"type": "string"
+				}
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}
+
+func TestSelfReferenceLoopErrors(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "object",
+					"description": "this description is ignored",
+					"properties": {
+						"bar": {
+							"type": "object",
+							"$ref": "#/components/schemas/foo"
+						}
+					}
+				},
+				"fruits": {
+					"type": "object",
+					"description": "foo fighters fighting fruits",
+					"$ref": "#/components/schemas/foo"
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	_, err = reader.readResolvedSchema("#/components/schemas/fruits")
+	assert.ErrorContains(t, err, "references loop detected. schema ref trace: #/components/schemas/fruits -> #/components/schemas/foo")
+}
+
+func TestCrossReferenceLoopErrors(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "object",
+					"description": "this description is ignored",
+					"properties": {
+						"bar": {
+							"type": "object",
+							"$ref": "#/components/schemas/fruits"
+						}
+					}
+				},
+				"fruits": {
+					"type": "object",
+					"description": "foo fighters fighting fruits",
+					"$ref": "#/components/schemas/foo"
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	_, err = reader.readResolvedSchema("#/components/schemas/fruits")
+	assert.ErrorContains(t, err, "references loop detected. schema ref trace: #/components/schemas/fruits -> #/components/schemas/foo")
+}
+
+func TestReferenceResolutionForMapInObject(t *testing.T) {
+	specString := `
+	{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "number"
+				},
+				"fruits": {
+					"type": "object",
+					"description": "fruits that are cool",
+					"properties": {
+						"guava": {
+							"type": "string",
+							"description": "a guava for my schema"
+						},
+						"mangos": {
+							"type": "object",
+							"description": "multiple mangos",
+							"$ref": "#/components/schemas/mango"
+						}
+					}
+				},
+				"mango": {
+					"type": "object",
+					"additionalProperties": {
+						"description": "a single mango",
+						"$ref": "#/components/schemas/foo"
+					}
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "fruits that are cool",
+			"properties": {
+				"guava": {
+					"type": "string",
+					"description": "a guava for my schema"
+				},
+				"mangos": {
+					"type": "object",
+					"description": "multiple mangos",
+					"additionalProperties": {
+						"type": "number",
+						"description": "a single mango"
+					}
+				}
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}
+
+func TestReferenceResolutionForArrayInObject(t *testing.T) {
+	specString := `{
+		"components": {
+			"schemas": {
+				"foo": {
+					"type": "number"
+				},
+				"fruits": {
+					"type": "object",
+					"description": "fruits that are cool",
+					"properties": {
+						"guava": {
+							"type": "string",
+							"description": "a guava for my schema"
+						},
+						"mangos": {
+							"type": "object",
+							"description": "multiple mangos",
+							"$ref": "#/components/schemas/mango"
+						}
+					}
+				},
+				"mango": {
+					"type": "object",
+					"items": {
+						"description": "a single mango",
+						"$ref": "#/components/schemas/foo"
+					}
+				}
+			}
+		}
+	}`
+	spec := &openapi.Specification{}
+	reader := &OpenapiReader{
+		OpenapiSpec: spec,
+		Memo:        make(map[string]*Schema),
+	}
+	err := json.Unmarshal([]byte(specString), spec)
+	require.NoError(t, err)
+
+	fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
+	require.NoError(t, err)
+
+	fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, "		", "	")
+	require.NoError(t, err)
+
+	expected := `{
+			"type": "object",
+			"description": "fruits that are cool",
+			"properties": {
+				"guava": {
+					"type": "string",
+					"description": "a guava for my schema"
+				},
+				"mangos": {
+					"type": "object",
+					"description": "multiple mangos",
+					"items": {
+						"type": "number",
+						"description": "a single mango"
+					}
+				}
+			}
+		}`
+
+	t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
+	t.Log("[DEBUG] expected: ", expected)
+	assert.Equal(t, expected, string(fruitsSchemaJson))
+}

bundle/schema/schema.go

@@ -35,6 +35,9 @@ type Schema struct {
 	// Required properties for the object. Any fields missing the "omitempty"
 	// json tag will be included
 	Required []string `json:"required,omitempty"`
+
+	// URI to a json schema
+	Reference *string `json:"$ref,omitempty"`
 }
 
 // This function translates golang types into json schema. Here is the mapping
@@ -187,7 +190,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
 	schema := &Schema{Type: rootJavascriptType}
 
 	if docs != nil {
-		schema.Description = docs.Documentation
+		schema.Description = docs.Description
 	}
 
 	// case array/slice
@@ -197,7 +200,11 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
 		if err != nil {
 			return nil, err
 		}
-		elemProps, err := safeToSchema(elemGolangType, docs, "", tracker)
+		var childDocs *Docs
+		if docs != nil {
+			childDocs = docs.Items
+		}
+		elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker)
 		if err != nil {
 			return nil, err
 		}
@@ -215,7 +222,11 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
 		if golangType.Key().Kind() != reflect.String {
 			return nil, fmt.Errorf("only string keyed maps allowed")
 		}
-		schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), docs, "", tracker)
+		var childDocs *Docs
+		if docs != nil {
+			childDocs = docs.AdditionalProperties
+		}
+		schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker)
 		if err != nil {
 			return nil, err
 		}
@@ -240,8 +251,8 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
 			// get docs for the child if they exist
 			var childDocs *Docs
 			if docs != nil {
-				if val, ok := docs.Children[childName]; ok {
+				if val, ok := docs.Properties[childName]; ok {
-					childDocs = &val
+					childDocs = val
 				}
 			}
 

bundle/schema/schema_test.go

@@ -1062,60 +1062,31 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
 	assert.Equal(t, expectedSchema, string(jsonSchema))
 }
 
-func TestDocIngestionInSchema(t *testing.T) {
+func TestDocIngestionForObject(t *testing.T) {
 	docs := &Docs{
-		Documentation: "docs for root",
+		Description: "docs for root",
-		Children: map[string]Docs{
+		Properties: map[string]*Docs{
 			"my_struct": {
-				Documentation: "docs for my struct",
+				Description: "docs for my struct",
-			},
+				Properties: map[string]*Docs{
-			"my_val": {
+					"a": {
-				Documentation: "docs for my val",
+						Description: "docs for a",
-			},
-			"my_slice": {
-				Documentation: "docs for my slice",
-				Children: map[string]Docs{
-					"guava": {
-						Documentation: "docs for guava",
 					},
-					"pineapple": {
+					"c": {
-						Documentation: "docs for pineapple",
+						Description: "docs for c which does not exist on my_struct",
-					},
-				},
-			},
-			"my_map": {
-				Documentation: "docs for my map",
-				Children: map[string]Docs{
-					"apple": {
-						Documentation: "docs for apple",
-					},
-					"mango": {
-						Documentation: "docs for mango",
 					},
 				},
 			},
 		},
 	}
 
-	type Foo struct {
-		Apple int `json:"apple"`
-		Mango int `json:"mango"`
-	}
-
-	type Bar struct {
-		Guava     int `json:"guava"`
-		Pineapple int `json:"pineapple"`
-	}
-
 	type MyStruct struct {
 		A string `json:"a"`
+		B int    `json:"b"`
 	}
 
 	type Root struct {
-		MyStruct *MyStruct       `json:"my_struct"`
+		MyStruct *MyStruct `json:"my_struct"`
-		MyVal    int             `json:"my_val"`
-		MySlice  []Bar           `json:"my_slice"`
-		MyMap    map[string]*Foo `json:"my_map"`
 	}
 
 	elem := Root{}
@@ -1131,29 +1102,82 @@ func TestDocIngestionInSchema(t *testing.T) {
 			"type": "object",
 			"description": "docs for root",
 			"properties": {
-				"my_map": {
+				"my_struct": {
 					"type": "object",
-					"description": "docs for my map",
+					"description": "docs for my struct",
-					"additionalProperties": {
+					"properties": {
-						"type": "object",
+						"a": {
-						"description": "docs for my map",
+							"type": "string",
-						"properties": {
+							"description": "docs for a"
-							"apple": {
-								"type": "number",
-								"description": "docs for apple"
-							},
-							"mango": {
-								"type": "number",
-								"description": "docs for mango"
-							}
 						},
-						"additionalProperties": false,
+						"b": {
-						"required": [
+							"type": "number"
-							"apple",
+						}
-							"mango"
+					},
-						]
+					"additionalProperties": false,
-					}
+					"required": [
+						"a",
+						"b"
+					]
+				}
+			},
+			"additionalProperties": false,
+			"required": [
+				"my_struct"
+			]
+		}`
+
+	t.Log("[DEBUG] actual: ", string(jsonSchema))
+	t.Log("[DEBUG] expected: ", expectedSchema)
+
+	assert.Equal(t, expectedSchema, string(jsonSchema))
+}
+
+func TestDocIngestionForSlice(t *testing.T) {
+	docs := &Docs{
+		Description: "docs for root",
+		Properties: map[string]*Docs{
+			"my_slice": {
+				Description: "docs for my slice",
+				Items: &Docs{
+					Properties: map[string]*Docs{
+						"guava": {
+							Description: "docs for guava",
+						},
+						"pineapple": {
+							Description: "docs for pineapple",
+						},
+						"watermelon": {
+							Description: "docs for watermelon which does not exist in schema",
+						},
+					},
 				},
+			},
+		},
+	}
+
+	type Bar struct {
+		Guava     int `json:"guava"`
+		Pineapple int `json:"pineapple"`
+	}
+
+	type Root struct {
+		MySlice []Bar `json:"my_slice"`
+	}
+
+	elem := Root{}
+
+	schema, err := New(reflect.TypeOf(elem), docs)
+	require.NoError(t, err)
+
+	jsonSchema, err := json.MarshalIndent(schema, "		", "	")
+	assert.NoError(t, err)
+
+	expectedSchema :=
+		`{
+			"type": "object",
+			"description": "docs for root",
+			"properties": {
 				"my_slice": {
 					"type": "array",
 					"description": "docs for my slice",
@@ -1175,20 +1199,130 @@ func TestDocIngestionInSchema(t *testing.T) {
 							"pineapple"
 						]
 					}
-				},
+				}
-				"my_struct": {
+			},
-					"type": "object",
+			"additionalProperties": false,
-					"description": "docs for my struct",
+			"required": [
-					"properties": {
+				"my_slice"
-						"a": {
+			]
-							"type": "string"
+		}`
-						}
+
+	t.Log("[DEBUG] actual: ", string(jsonSchema))
+	t.Log("[DEBUG] expected: ", expectedSchema)
+
+	assert.Equal(t, expectedSchema, string(jsonSchema))
+}
+
+func TestDocIngestionForMap(t *testing.T) {
+	docs := &Docs{
+		Description: "docs for root",
+		Properties: map[string]*Docs{
+			"my_map": {
+				Description: "docs for my map",
+				AdditionalProperties: &Docs{
+					Properties: map[string]*Docs{
+						"apple": {
+							Description: "docs for apple",
+						},
+						"mango": {
+							Description: "docs for mango",
+						},
+						"watermelon": {
+							Description: "docs for watermelon which does not exist in schema",
+						},
+						"papaya": {
+							Description: "docs for papaya which does not exist in schema",
+						},
 					},
-					"additionalProperties": false,
-					"required": [
-						"a"
-					]
 				},
+			},
+		},
+	}
+
+	type Foo struct {
+		Apple int `json:"apple"`
+		Mango int `json:"mango"`
+	}
+
+	type Root struct {
+		MyMap map[string]*Foo `json:"my_map"`
+	}
+
+	elem := Root{}
+
+	schema, err := New(reflect.TypeOf(elem), docs)
+	require.NoError(t, err)
+
+	jsonSchema, err := json.MarshalIndent(schema, "		", "	")
+	assert.NoError(t, err)
+
+	expectedSchema :=
+		`{
+			"type": "object",
+			"description": "docs for root",
+			"properties": {
+				"my_map": {
+					"type": "object",
+					"description": "docs for my map",
+					"additionalProperties": {
+						"type": "object",
+						"properties": {
+							"apple": {
+								"type": "number",
+								"description": "docs for apple"
+							},
+							"mango": {
+								"type": "number",
+								"description": "docs for mango"
+							}
+						},
+						"additionalProperties": false,
+						"required": [
+							"apple",
+							"mango"
+						]
+					}
+				}
+			},
+			"additionalProperties": false,
+			"required": [
+				"my_map"
+			]
+		}`
+
+	t.Log("[DEBUG] actual: ", string(jsonSchema))
+	t.Log("[DEBUG] expected: ", expectedSchema)
+
+	assert.Equal(t, expectedSchema, string(jsonSchema))
+}
+
+func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
+	docs := &Docs{
+		Description: "docs for root",
+		Properties: map[string]*Docs{
+			"my_val": {
+				Description: "docs for my val",
+			},
+		},
+	}
+
+	type Root struct {
+		MyVal int `json:"my_val"`
+	}
+
+	elem := Root{}
+
+	schema, err := New(reflect.TypeOf(elem), docs)
+	require.NoError(t, err)
+
+	jsonSchema, err := json.MarshalIndent(schema, "		", "	")
+	assert.NoError(t, err)
+
+	expectedSchema :=
+		`{
+			"type": "object",
+			"description": "docs for root",
+			"properties": {
 				"my_val": {
 					"type": "number",
 					"description": "docs for my val"
@@ -1196,10 +1330,7 @@ func TestDocIngestionInSchema(t *testing.T) {
 			},
 			"additionalProperties": false,
 			"required": [
-				"my_struct",
+				"my_val"
-				"my_val",
-				"my_slice",
-				"my_map"
 			]
 		}`
 

cmd/bundle/launch.go

@@ -0,0 +1,37 @@
+package bundle
+
+import (
+	"fmt"
+
+	"github.com/databricks/bricks/cmd/root"
+	"github.com/spf13/cobra"
+)
+
+var launchCmd = &cobra.Command{
+	Use:   "launch",
+	Short: "Launches a notebook on development cluster",
+	Long:  `Reads a file and executes it on dev cluster`,
+	Args:  cobra.ExactArgs(1),
+
+	// We're not ready to expose this command until we specify its semantics.
+	Hidden: true,
+
+	PreRunE: root.MustConfigureBundle,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return fmt.Errorf("TODO")
+		// contents, err := os.ReadFile(args[0])
+		// if err != nil {
+		// 	return err
+		// }
+		// results := project.RunPythonOnDev(cmd.Context(), string(contents))
+		// if results.Failed() {
+		// 	return results.Err()
+		// }
+		// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
+		// return nil
+	},
+}
+
+func init() {
+	AddCommand(launchCmd)
+}

cmd/bundle/run.go

@@ -21,7 +21,6 @@ var runCmd = &cobra.Command{
 		b := bundle.Get(cmd.Context())
 		err := bundle.Apply(cmd.Context(), b, []bundle.Mutator{
 			phases.Initialize(),
-			terraform.Initialize(),
 			terraform.Load(),
 		})
 		if err != nil {

cmd/bundle/schema.go

@@ -14,7 +14,7 @@ var schemaCmd = &cobra.Command{
 	Short: "Generate JSON Schema for bundle configuration",
 
 	RunE: func(cmd *cobra.Command, args []string) error {
-		docs, err := schema.GetBundleDocs()
+		docs, err := schema.BundleDocs(openapi)
 		if err != nil {
 			return err
 		}
@@ -22,15 +22,26 @@ var schemaCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		jsonSchema, err := json.MarshalIndent(schema, "", "  ")
+		result, err := json.MarshalIndent(schema, "", "  ")
 		if err != nil {
 			return err
 		}
-		cmd.OutOrStdout().Write(jsonSchema)
+		if onlyDocs {
+			result, err = json.MarshalIndent(docs, "", "  ")
+			if err != nil {
+				return err
+			}
+		}
+		cmd.OutOrStdout().Write(result)
 		return nil
 	},
 }
 
+var openapi string
+var onlyDocs bool
+
 func init() {
 	AddCommand(schemaCmd)
+	schemaCmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
+	schemaCmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
 }

cmd/bundle/test.go

@@ -0,0 +1,32 @@
+package bundle
+
+import (
+	"fmt"
+
+	"github.com/databricks/bricks/cmd/root"
+	"github.com/spf13/cobra"
+)
+
+var testCmd = &cobra.Command{
+	Use:   "test",
+	Short: "run tests for the project",
+	Long:  `This is longer description of the command`,
+
+	// We're not ready to expose this command until we specify its semantics.
+	Hidden: true,
+
+	PreRunE: root.MustConfigureBundle,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return fmt.Errorf("TODO")
+		// results := project.RunPythonOnDev(cmd.Context(), `return 1`)
+		// if results.Failed() {
+		// 	return results.Err()
+		// }
+		// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
+		// return nil
+	},
+}
+
+func init() {
+	AddCommand(testCmd)
+}

cmd/launch/launch.go

@@ -1,36 +0,0 @@
-package launch
-
-import (
-	"fmt"
-	"log"
-	"os"
-
-	"github.com/databricks/bricks/cmd/root"
-	"github.com/databricks/bricks/project"
-	"github.com/spf13/cobra"
-)
-
-// launchCmd represents the launch command
-var launchCmd = &cobra.Command{
-	Use:   "launch",
-	Short: "Launches a notebook on development cluster",
-	Long:  `Reads a file and executes it on dev cluster`,
-	Args:  cobra.ExactArgs(1),
-	Run: func(cmd *cobra.Command, args []string) {
-		contents, err := os.ReadFile(args[0])
-		if err != nil {
-			log.Fatal(err)
-		}
-		results := project.RunPythonOnDev(cmd.Context(), string(contents))
-		if results.Failed() {
-			log.Fatal(results.Error())
-		}
-		fmt.Println(results.Text())
-	},
-}
-
-func init() {
-	// TODO: detect if we can skip registering
-	// launch command for DBSQL projects
-	root.RootCmd.AddCommand(launchCmd)
-}

cmd/test/test.go

@@ -1,37 +0,0 @@
-package test
-
-import (
-	"log"
-
-	"github.com/databricks/bricks/cmd/root"
-	"github.com/databricks/bricks/project"
-	"github.com/spf13/cobra"
-)
-
-// testCmd represents the test command
-var testCmd = &cobra.Command{
-	Use:   "test",
-	Short: "run tests for the project",
-	Long:  `This is longer description of the command`,
-	Run: func(cmd *cobra.Command, args []string) {
-		results := project.RunPythonOnDev(cmd.Context(), `return 1`)
-		if results.Failed() {
-			log.Fatal(results.Error())
-		}
-		log.Printf("Success: %s", results.Text())
-	},
-}
-
-func init() {
-	root.RootCmd.AddCommand(testCmd)
-
-	// Here you will define your flags and configuration settings.
-
-	// Cobra supports Persistent Flags which will work for this command
-	// and all subcommands, e.g.:
-	// testCmd.PersistentFlags().String("foo", "", "A help for foo")
-
-	// Cobra supports local flags which will only run when this command
-	// is called directly, e.g.:
-	// testCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-}

go.mod

@@ -22,17 +22,18 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/hashicorp/hc-install v0.5.0
-	github.com/hashicorp/terraform-exec v0.17.3
+	github.com/hashicorp/terraform-exec v0.18.1
-	github.com/hashicorp/terraform-json v0.15.0
+	github.com/hashicorp/terraform-json v0.16.0
 	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
 	golang.org/x/sync v0.1.0
 )
 
 require (
+	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/zclconf/go-cty v1.11.0 // indirect
+	github.com/zclconf/go-cty v1.13.0 // indirect
 	golang.org/x/crypto v0.5.0 // indirect
 )
 
@@ -61,5 +62,5 @@ require (
 	google.golang.org/grpc v1.53.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -15,7 +15,7 @@ github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -33,8 +33,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+<<<<<<< HEAD
 github.com/databricks/databricks-sdk-go v0.5.0 h1:u5tytS0+BHZXLS2cqg4gW+6LOGgfXOriBuT5bt+gTSk=
 github.com/databricks/databricks-sdk-go v0.5.0/go.mod h1:MTJHMjKBotE0IsyPbMcPvHb3JKknkRtOms/ET7F0rB8=
+=======
+github.com/databricks/databricks-sdk-go v0.4.1 h1:SX1Owt2XcY/NVt/LRbuO/0dtomEebfNa8yekQlQy4gM=
+github.com/databricks/databricks-sdk-go v0.4.1/go.mod h1:MOtUlDw9bj/1PuYmFkNSx+29Hu1clrob8vQQ6Fsji8A=
+>>>>>>> origin
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -63,11 +68,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -110,10 +113,10 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/hc-install v0.5.0 h1:D9bl4KayIYKEeJ4vUDe9L5huqxZXczKaykSRcmQ0xY0=
 github.com/hashicorp/hc-install v0.5.0/go.mod h1:JyzMfbzfSBSjoDCRPna1vi/24BEDxFaCPfdHtM5SCdo=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU=
+github.com/hashicorp/terraform-exec v0.18.1 h1:LAbfDvNQU1l0NOQlTuudjczVhHj061fNX5H8XZxHlH4=
-github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI=
+github.com/hashicorp/terraform-exec v0.18.1/go.mod h1:58wg4IeuAJ6LVsLUeD2DWZZoc/bYi6dzhLHzxM41980=
-github.com/hashicorp/terraform-json v0.15.0 h1:/gIyNtR6SFw6h5yzlbDbACyGvIhKtQi8mTsbkNd79lE=
+github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8c+85Albo3s=
-github.com/hashicorp/terraform-json v0.15.0/go.mod h1:+L1RNzjDU5leLFZkHTFTbJXaoqUC6TqXlFgDoOXrtvk=
+github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0=
 github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -135,7 +138,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
 github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
@@ -148,11 +150,9 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
-github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
@@ -165,7 +165,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
-github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
@@ -180,7 +179,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -190,19 +188,19 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+<<<<<<< HEAD
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
+=======
+>>>>>>> origin
 github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
 github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
+github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
-github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
-github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0=
-github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA=
-github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -226,14 +224,12 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
 golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
@@ -277,7 +273,6 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
@@ -293,18 +288,27 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+<<<<<<< HEAD
 google.golang.org/api v0.112.0 h1:iDmzvZ4C086R3+en4nSyIf07HlQKMOX1Xx2dmia/+KQ=
 google.golang.org/api v0.112.0/go.mod h1:737UfWHNsOq4F3REUTmb+GN9pugkgNLCayLTfoIKpPc=
+=======
+google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0=
+google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
+>>>>>>> origin
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+<<<<<<< HEAD
 google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos=
 google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
+=======
+google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds=
+google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
+>>>>>>> origin
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
@@ -326,7 +330,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

libs/sync/snapshot.go

@@ -165,13 +165,17 @@ func loadOrNewSnapshot(opts *SyncOptions) (*Snapshot, error) {
 }
 
 func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
-	currentFilenames := map[string]bool{}
 	lastModifiedTimes := s.LastUpdatedTimes
 	remoteToLocalNames := s.RemoteToLocalNames
 	localToRemoteNames := s.LocalToRemoteNames
+
+	// set of files currently present in the local file system and tracked by git
+	localFileSet := map[string]struct{}{}
+	for _, f := range all {
+		localFileSet[f.Relative] = struct{}{}
+	}
+
 	for _, f := range all {
-		// create set of current files to figure out if removals are needed
-		currentFilenames[f.Relative] = true
 		// get current modified timestamp
 		modified := f.Modified()
 		lastSeenModified, seen := lastModifiedTimes[f.Relative]
@@ -207,11 +211,13 @@ func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
 				change.delete = append(change.delete, oldRemoteName)
 				delete(remoteToLocalNames, oldRemoteName)
 			}
+
 			// We cannot allow two local files in the project to point to the same
 			// remote path
-			oldLocalName, ok := remoteToLocalNames[remoteName]
+			prevLocalName, ok := remoteToLocalNames[remoteName]
-			if ok && oldLocalName != f.Relative {
+			_, prevLocalFileExists := localFileSet[prevLocalName]
-				return change, fmt.Errorf("both %s and %s point to the same remote file location %s. Please remove one of them from your local project", oldLocalName, f.Relative, remoteName)
+			if ok && prevLocalName != f.Relative && prevLocalFileExists {
+				return change, fmt.Errorf("both %s and %s point to the same remote file location %s. Please remove one of them from your local project", prevLocalName, f.Relative, remoteName)
 			}
 			localToRemoteNames[f.Relative] = remoteName
 			remoteToLocalNames[remoteName] = f.Relative
@@ -220,7 +226,7 @@ func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
 	// figure out files in the snapshot.lastModifiedTimes, but not on local
 	// filesystem. These will be deleted
 	for localName := range lastModifiedTimes {
-		_, exists := currentFilenames[localName]
+		_, exists := localFileSet[localName]
 		if exists {
 			continue
 		}

libs/sync/snapshot_test.go

@@ -249,6 +249,43 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) {
 	assert.ErrorContains(t, err, "both foo and foo.py point to the same remote file location foo. Please remove one of them from your local project")
 }
 
+func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) {
+	// Create temp project dir
+	projectDir := t.TempDir()
+	fileSet, err := git.NewFileSet(projectDir)
+	require.NoError(t, err)
+	state := Snapshot{
+		LastUpdatedTimes:   make(map[string]time.Time),
+		LocalToRemoteNames: make(map[string]string),
+		RemoteToLocalNames: make(map[string]string),
+	}
+
+	// upload should work since they point to different destinations
+	pythonFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.py"))
+	defer pythonFoo.Close(t)
+	pythonFoo.Overwrite(t, "# Databricks notebook source\n")
+	files, err := fileSet.All()
+	assert.NoError(t, err)
+	change, err := state.diff(files)
+	assert.NoError(t, err)
+	assert.Len(t, change.delete, 0)
+	assert.Len(t, change.put, 1)
+	assert.Contains(t, change.put, "foo.py")
+
+	pythonFoo.Remove(t)
+	sqlFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.sql"))
+	defer sqlFoo.Close(t)
+	sqlFoo.Overwrite(t, "-- Databricks notebook source\n")
+	files, err = fileSet.All()
+	assert.NoError(t, err)
+	change, err = state.diff(files)
+	assert.NoError(t, err)
+	assert.Len(t, change.delete, 1)
+	assert.Len(t, change.put, 1)
+	assert.Contains(t, change.put, "foo.sql")
+	assert.Contains(t, change.delete, "foo")
+}
+
 func defaultOptions(t *testing.T) *SyncOptions {
 	return &SyncOptions{
 		Host:             "www.foobar.com",

libs/sync/sync.go

@@ -118,8 +118,6 @@ func (s *Sync) notifyComplete(ctx context.Context, d diff) {
 }
 
 func (s *Sync) RunOnce(ctx context.Context) error {
-	applyDiff := syncCallback(ctx, s)
-
 	// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
 	// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
 	all, err := s.fileSet.All()
@@ -139,7 +137,7 @@ func (s *Sync) RunOnce(ctx context.Context) error {
 		return nil
 	}
 
-	err = applyDiff(change)
+	err = s.applyDiff(ctx, change)
 	if err != nil {
 		return err
 	}

libs/sync/watchdog.go

@@ -6,53 +6,63 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-// See https://docs.databricks.com/resources/limits.html#limits-api-rate-limits for per api
+// Maximum number of concurrent requests during sync.
-// rate limits
 const MaxRequestsInFlight = 20
 
-func syncCallback(ctx context.Context, s *Sync) func(localDiff diff) error {
+// Perform a DELETE of the specified remote path.
-	return func(d diff) error {
+func (s *Sync) applyDelete(ctx context.Context, group *errgroup.Group, remoteName string) {
-		// Abstraction over wait groups which allows you to get the errors
+	// Return early if the context has already been cancelled.
-		// returned in goroutines
+	select {
-		var g errgroup.Group
+	case <-ctx.Done():
+		return
+	default:
+		// Proceed.
+	}
 
-		// Allow MaxRequestLimit maxiumum concurrent api calls
+	group.Go(func() error {
-		g.SetLimit(MaxRequestsInFlight)
+		s.notifyProgress(ctx, EventActionDelete, remoteName, 0.0)
-
+		err := s.repoFiles.DeleteFile(ctx, remoteName)
-		for _, remoteName := range d.delete {
+		if err != nil {
-			// Copy of remoteName created to make this safe for concurrent use.
-			// directly using remoteName can cause race conditions since the loop
-			// might iterate over to the next remoteName before the go routine function
-			// is evaluated
-			remoteNameCopy := remoteName
-			g.Go(func() error {
-				s.notifyProgress(ctx, EventActionDelete, remoteNameCopy, 0.0)
-				err := s.repoFiles.DeleteFile(ctx, remoteNameCopy)
-				if err != nil {
-					return err
-				}
-				s.notifyProgress(ctx, EventActionDelete, remoteNameCopy, 1.0)
-				return nil
-			})
-		}
-		for _, localRelativePath := range d.put {
-			// Copy of localName created to make this safe for concurrent use.
-			localRelativePathCopy := localRelativePath
-			g.Go(func() error {
-				s.notifyProgress(ctx, EventActionPut, localRelativePathCopy, 0.0)
-				err := s.repoFiles.PutFile(ctx, localRelativePathCopy)
-				if err != nil {
-					return err
-				}
-				s.notifyProgress(ctx, EventActionPut, localRelativePathCopy, 1.0)
-				return nil
-			})
-		}
-		// wait for goroutines to finish and return first non-nil error return
-		// if any
-		if err := g.Wait(); err != nil {
 			return err
 		}
+		s.notifyProgress(ctx, EventActionDelete, remoteName, 1.0)
 		return nil
-	}
+	})
+}
+
+// Perform a PUT of the specified local path.
+func (s *Sync) applyPut(ctx context.Context, group *errgroup.Group, localName string) {
+	// Return early if the context has already been cancelled.
+	select {
+	case <-ctx.Done():
+		return
+	default:
+		// Proceed.
+	}
+
+	group.Go(func() error {
+		s.notifyProgress(ctx, EventActionPut, localName, 0.0)
+		err := s.repoFiles.PutFile(ctx, localName)
+		if err != nil {
+			return err
+		}
+		s.notifyProgress(ctx, EventActionPut, localName, 1.0)
+		return nil
+	})
+}
+
+func (s *Sync) applyDiff(ctx context.Context, d diff) error {
+	group, ctx := errgroup.WithContext(ctx)
+	group.SetLimit(MaxRequestsInFlight)
+
+	for _, remoteName := range d.delete {
+		s.applyDelete(ctx, group, remoteName)
+	}
+
+	for _, localName := range d.put {
+		s.applyPut(ctx, group, localName)
+	}
+
+	// Wait for goroutines to finish and return first non-nil error return if any.
+	return group.Wait()
 }

main.go

@@ -9,10 +9,8 @@ import (
 	_ "github.com/databricks/bricks/cmd/configure"
 	_ "github.com/databricks/bricks/cmd/fs"
 	_ "github.com/databricks/bricks/cmd/init"
-	_ "github.com/databricks/bricks/cmd/launch"
 	"github.com/databricks/bricks/cmd/root"
 	_ "github.com/databricks/bricks/cmd/sync"
-	_ "github.com/databricks/bricks/cmd/test"
 	_ "github.com/databricks/bricks/cmd/version"
 )