mirror of https://github.com/databricks/cli.git
Merge remote-tracking branch 'origin' into pipeline-errors
This commit is contained in:
Shreyas Goenka
commit
59803f0552
|
|
@ -0,0 +1,26 @@
|
|||
### Overview
|
||||
|
||||
`docs/bundle_descriptions.json` contains both autogenerated as well as manually written
|
||||
descriptions for the json schema. Specifically
|
||||
1. `resources` : almost all descriptions are autogenerated from the OpenAPI spec
|
||||
2. `environments` : almost all descriptions are copied over from root level entities (eg: `bundle`, `artifacts`)
|
||||
3. `bundle` : manually editted
|
||||
4. `include` : manually editted
|
||||
5. `workspace` : manually editted
|
||||
6. `artifacts` : manually editted
|
||||
|
||||
These descriptions are rendered in the inline documentation in an IDE
|
||||
|
||||
### SOP: Add schema descriptions for new fields in bundle config
|
||||
|
||||
1. You can autogenerate empty descriptions for the new fields by running
|
||||
`bricks bundle schema --only-docs > ~/bricks/bundle/schema/docs/bundle_descriptions.json`
|
||||
2. Manually edit bundle_descriptions.json to add your descriptions
|
||||
3. Again run `bricks bundle schema --only-docs > ~/bricks/bundle/schema/docs/bundle_descriptions.json` to copy over any applicable descriptions to `environments`
|
||||
4. push to repo
|
||||
|
||||
|
||||
### SOP: Update descriptions in resources from a newer openapi spec
|
||||
|
||||
1. Run `bricks bundle schema --only-docs --openapi PATH_TO_SPEC > ~/bricks/bundle/schema/docs/bundle_descriptions.json`
|
||||
2. push to repo
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
documentation: Root of the bundle config
|
||||
children:
|
||||
bundle:
|
||||
documentation: |
|
||||
Bundle contains details about this bundle, such as its name,
|
||||
version of the spec (TODO), default cluster, default warehouse, etc.
|
||||
children:
|
||||
environment:
|
||||
documentation: Environment is set by the mutator that selects the environment.
|
||||
|
||||
artifacts:
|
||||
documentation: Artifacts contains a description of all code artifacts in this bundle.
|
||||
|
|
@ -2,37 +2,109 @@ package schema
|
|||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"github.com/databricks/bricks/bundle/config"
|
||||
"github.com/databricks/databricks-sdk-go/openapi"
|
||||
)
|
||||
|
||||
// A subset of Schema struct
|
||||
type Docs struct {
|
||||
Documentation string `json:"documentation"`
|
||||
Children map[string]Docs `json:"children"`
|
||||
Description string `json:"description"`
|
||||
Properties map[string]*Docs `json:"properties,omitempty"`
|
||||
Items *Docs `json:"items,omitempty"`
|
||||
AdditionalProperties *Docs `json:"additionalproperties,omitempty"`
|
||||
}
|
||||
|
||||
func LoadDocs(path string) (*Docs, error) {
|
||||
bytes, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docs := Docs{}
|
||||
err = yaml.Unmarshal(bytes, &docs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &docs, nil
|
||||
}
|
||||
|
||||
//go:embed bundle_config_docs.yml
|
||||
//go:embed docs/bundle_descriptions.json
|
||||
var bundleDocs []byte
|
||||
|
||||
func GetBundleDocs() (*Docs, error) {
|
||||
docs := Docs{}
|
||||
err := yaml.Unmarshal(bundleDocs, &docs)
|
||||
func BundleDocs(openapiSpecPath string) (*Docs, error) {
|
||||
docs, err := initializeBundleDocs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &docs, nil
|
||||
if openapiSpecPath != "" {
|
||||
openapiSpec, err := os.ReadFile(openapiSpecPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
spec := &openapi.Specification{}
|
||||
err = json.Unmarshal(openapiSpec, spec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
openapiReader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
resourcesDocs, err := openapiReader.ResourcesDocs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resourceSchema, err := New(reflect.TypeOf(config.Resources{}), resourcesDocs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docs.Properties["resources"] = schemaToDocs(resourceSchema)
|
||||
}
|
||||
docs.refreshEnvironmentsDocs()
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func (docs *Docs) refreshEnvironmentsDocs() error {
|
||||
environmentsDocs, ok := docs.Properties["environments"]
|
||||
if !ok || environmentsDocs.AdditionalProperties == nil ||
|
||||
environmentsDocs.AdditionalProperties.Properties == nil {
|
||||
return fmt.Errorf("invalid environments descriptions")
|
||||
}
|
||||
environmentProperties := environmentsDocs.AdditionalProperties.Properties
|
||||
propertiesToCopy := []string{"artifacts", "bundle", "resources", "workspace"}
|
||||
for _, p := range propertiesToCopy {
|
||||
environmentProperties[p] = docs.Properties[p]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func initializeBundleDocs() (*Docs, error) {
|
||||
// load embedded descriptions
|
||||
embedded := Docs{}
|
||||
err := json.Unmarshal(bundleDocs, &embedded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// generate schema with the embedded descriptions
|
||||
schema, err := New(reflect.TypeOf(config.Root{}), &embedded)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// converting the schema back to docs. This creates empty descriptions
|
||||
// for any properties that were missing in the embedded descriptions
|
||||
docs := schemaToDocs(schema)
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
// *Docs are a subset of *Schema, this function selects that subset
|
||||
func schemaToDocs(schema *Schema) *Docs {
|
||||
// terminate recursion if schema is nil
|
||||
if schema == nil {
|
||||
return nil
|
||||
}
|
||||
docs := &Docs{
|
||||
Description: schema.Description,
|
||||
}
|
||||
if len(schema.Properties) > 0 {
|
||||
docs.Properties = make(map[string]*Docs)
|
||||
}
|
||||
for k, v := range schema.Properties {
|
||||
docs.Properties[k] = schemaToDocs(v)
|
||||
}
|
||||
docs.Items = schemaToDocs(schema.Items)
|
||||
if additionalProperties, ok := schema.AdditionalProperties.(*Schema); ok {
|
||||
docs.AdditionalProperties = schemaToDocs(additionalProperties)
|
||||
}
|
||||
return docs
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,61 @@
|
|||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSchemaToDocs(t *testing.T) {
|
||||
schema := &Schema{
|
||||
Type: "object",
|
||||
Description: "root doc",
|
||||
Properties: map[string]*Schema{
|
||||
"foo": {Type: "number", Description: "foo doc"},
|
||||
"bar": {Type: "string"},
|
||||
"octave": {
|
||||
Type: "object",
|
||||
AdditionalProperties: &Schema{Type: "number"},
|
||||
Description: "octave docs",
|
||||
},
|
||||
"scales": {
|
||||
Type: "object",
|
||||
Description: "scale docs",
|
||||
Items: &Schema{Type: "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
docs := schemaToDocs(schema)
|
||||
docsJson, err := json.MarshalIndent(docs, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected :=
|
||||
`{
|
||||
"description": "root doc",
|
||||
"properties": {
|
||||
"bar": {
|
||||
"description": ""
|
||||
},
|
||||
"foo": {
|
||||
"description": "foo doc"
|
||||
},
|
||||
"octave": {
|
||||
"description": "octave docs",
|
||||
"additionalproperties": {
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"scales": {
|
||||
"description": "scale docs",
|
||||
"items": {
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
t.Log("[DEBUG] actual: ", string(docsJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(docsJson))
|
||||
}
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/databricks/databricks-sdk-go/openapi"
|
||||
)
|
||||
|
||||
type OpenapiReader struct {
|
||||
OpenapiSpec *openapi.Specification
|
||||
Memo map[string]*Schema
|
||||
}
|
||||
|
||||
const SchemaPathPrefix = "#/components/schemas/"
|
||||
|
||||
func (reader *OpenapiReader) readOpenapiSchema(path string) (*Schema, error) {
|
||||
schemaKey := strings.TrimPrefix(path, SchemaPathPrefix)
|
||||
|
||||
// return early if we already have a computed schema
|
||||
memoSchema, ok := reader.Memo[schemaKey]
|
||||
if ok {
|
||||
return memoSchema, nil
|
||||
}
|
||||
|
||||
// check path is present in openapi spec
|
||||
openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("schema with path %s not found in openapi spec", path)
|
||||
}
|
||||
|
||||
// convert openapi schema to the native schema struct
|
||||
bytes, err := json.Marshal(*openapiSchema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jsonSchema := &Schema{}
|
||||
err = json.Unmarshal(bytes, jsonSchema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// A hack to convert a map[string]interface{} to *Schema
|
||||
// We rely on the type of a AdditionalProperties in downstream functions
|
||||
// to do reference interpolation
|
||||
_, ok = jsonSchema.AdditionalProperties.(map[string]interface{})
|
||||
if ok {
|
||||
b, err := json.Marshal(jsonSchema.AdditionalProperties)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
additionalProperties := &Schema{}
|
||||
err = json.Unmarshal(b, additionalProperties)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jsonSchema.AdditionalProperties = additionalProperties
|
||||
}
|
||||
|
||||
// store read schema into memo
|
||||
reader.Memo[schemaKey] = jsonSchema
|
||||
|
||||
return jsonSchema, nil
|
||||
}
|
||||
|
||||
// safe againt loops in refs
|
||||
func (reader *OpenapiReader) safeResolveRefs(root *Schema, seenRefs map[string]struct{}) (*Schema, error) {
|
||||
if root.Reference == nil {
|
||||
return reader.traverseSchema(root, seenRefs)
|
||||
}
|
||||
key := *root.Reference
|
||||
_, ok := seenRefs[key]
|
||||
if ok {
|
||||
// self reference loops can be supported however the logic is non-trivial because
|
||||
// cross refernce loops are not allowed (see: http://json-schema.org/understanding-json-schema/structuring.html#recursion)
|
||||
return nil, fmt.Errorf("references loop detected")
|
||||
}
|
||||
ref := *root.Reference
|
||||
description := root.Description
|
||||
seenRefs[ref] = struct{}{}
|
||||
|
||||
// Mark reference nil, so we do not traverse this again. This is tracked
|
||||
// in the memo
|
||||
root.Reference = nil
|
||||
|
||||
// unroll one level of reference
|
||||
selfRef, err := reader.readOpenapiSchema(ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root = selfRef
|
||||
root.Description = description
|
||||
|
||||
// traverse again to find new references
|
||||
root, err = reader.traverseSchema(root, seenRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delete(seenRefs, ref)
|
||||
return root, err
|
||||
}
|
||||
|
||||
func (reader *OpenapiReader) traverseSchema(root *Schema, seenRefs map[string]struct{}) (*Schema, error) {
|
||||
// case primitive (or invalid)
|
||||
if root.Type != Object && root.Type != Array {
|
||||
return root, nil
|
||||
}
|
||||
// only root references are resolved
|
||||
if root.Reference != nil {
|
||||
return reader.safeResolveRefs(root, seenRefs)
|
||||
}
|
||||
// case struct
|
||||
if len(root.Properties) > 0 {
|
||||
for k, v := range root.Properties {
|
||||
childSchema, err := reader.safeResolveRefs(v, seenRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root.Properties[k] = childSchema
|
||||
}
|
||||
}
|
||||
// case array
|
||||
if root.Items != nil {
|
||||
itemsSchema, err := reader.safeResolveRefs(root.Items, seenRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root.Items = itemsSchema
|
||||
}
|
||||
// case map
|
||||
additionionalProperties, ok := root.AdditionalProperties.(*Schema)
|
||||
if ok && additionionalProperties != nil {
|
||||
valueSchema, err := reader.safeResolveRefs(additionionalProperties, seenRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root.AdditionalProperties = valueSchema
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
func (reader *OpenapiReader) readResolvedSchema(path string) (*Schema, error) {
|
||||
root, err := reader.readOpenapiSchema(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seenRefs := make(map[string]struct{})
|
||||
seenRefs[path] = struct{}{}
|
||||
root, err = reader.safeResolveRefs(root, seenRefs)
|
||||
if err != nil {
|
||||
trace := ""
|
||||
count := 0
|
||||
for k := range seenRefs {
|
||||
if count == len(seenRefs)-1 {
|
||||
trace += k
|
||||
break
|
||||
}
|
||||
trace += k + " -> "
|
||||
count++
|
||||
}
|
||||
return nil, fmt.Errorf("%s. schema ref trace: %s", err, trace)
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
func (reader *OpenapiReader) jobsDocs() (*Docs, error) {
|
||||
jobSettingsSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "jobs.JobSettings")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobDocs := schemaToDocs(jobSettingsSchema)
|
||||
// TODO: add description for id if needed.
|
||||
// Tracked in https://github.com/databricks/bricks/issues/242
|
||||
jobsDocs := &Docs{
|
||||
Description: "List of job definations",
|
||||
AdditionalProperties: jobDocs,
|
||||
}
|
||||
return jobsDocs, nil
|
||||
}
|
||||
|
||||
func (reader *OpenapiReader) pipelinesDocs() (*Docs, error) {
|
||||
pipelineSpecSchema, err := reader.readResolvedSchema(SchemaPathPrefix + "pipelines.PipelineSpec")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pipelineDocs := schemaToDocs(pipelineSpecSchema)
|
||||
// TODO: Two fields in resources.Pipeline have the json tag id. Clarify the
|
||||
// semantics and then add a description if needed. (https://github.com/databricks/bricks/issues/242)
|
||||
pipelinesDocs := &Docs{
|
||||
Description: "List of pipeline definations",
|
||||
AdditionalProperties: pipelineDocs,
|
||||
}
|
||||
return pipelinesDocs, nil
|
||||
}
|
||||
|
||||
func (reader *OpenapiReader) ResourcesDocs() (*Docs, error) {
|
||||
jobsDocs, err := reader.jobsDocs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pipelinesDocs, err := reader.pipelinesDocs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Docs{
|
||||
Description: "Specification of databricks resources to instantiate",
|
||||
Properties: map[string]*Docs{
|
||||
"jobs": jobsDocs,
|
||||
"pipelines": pipelinesDocs,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,435 @@
|
|||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/databricks/databricks-sdk-go/openapi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReadSchemaForObject(t *testing.T) {
|
||||
specString := `
|
||||
{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "number"
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mango": {
|
||||
"type": "object",
|
||||
"description": "a mango for my schema",
|
||||
"$ref": "#/components/schemas/mango"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mango": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"foo": {
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mango": {
|
||||
"type": "object",
|
||||
"description": "a mango for my schema",
|
||||
"properties": {
|
||||
"foo": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
||||
func TestReadSchemaForArray(t *testing.T) {
|
||||
specString := `
|
||||
{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"items": {
|
||||
"description": "some papayas, because papayas are fruits too",
|
||||
"$ref": "#/components/schemas/papaya"
|
||||
}
|
||||
},
|
||||
"papaya": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"items": {
|
||||
"type": "number",
|
||||
"description": "some papayas, because papayas are fruits too"
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
||||
func TestReadSchemaForMap(t *testing.T) {
|
||||
specString := `{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "fruits that are meh",
|
||||
"additionalProperties": {
|
||||
"description": "watermelons. watermelons.",
|
||||
"$ref": "#/components/schemas/watermelon"
|
||||
}
|
||||
},
|
||||
"watermelon": {
|
||||
"type": "number"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "fruits that are meh",
|
||||
"additionalProperties": {
|
||||
"type": "number",
|
||||
"description": "watermelons. watermelons."
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
||||
func TestRootReferenceIsResolved(t *testing.T) {
|
||||
specString := `{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "object",
|
||||
"description": "this description is ignored",
|
||||
"properties": {
|
||||
"abc": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "foo fighters fighting fruits",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
schema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
fruitsSchemaJson, err := json.MarshalIndent(schema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "foo fighters fighting fruits",
|
||||
"properties": {
|
||||
"abc": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
||||
func TestSelfReferenceLoopErrors(t *testing.T) {
|
||||
specString := `{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "object",
|
||||
"description": "this description is ignored",
|
||||
"properties": {
|
||||
"bar": {
|
||||
"type": "object",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "foo fighters fighting fruits",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
assert.ErrorContains(t, err, "references loop detected. schema ref trace: #/components/schemas/fruits -> #/components/schemas/foo")
|
||||
}
|
||||
|
||||
func TestCrossReferenceLoopErrors(t *testing.T) {
|
||||
specString := `{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "object",
|
||||
"description": "this description is ignored",
|
||||
"properties": {
|
||||
"bar": {
|
||||
"type": "object",
|
||||
"$ref": "#/components/schemas/fruits"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "foo fighters fighting fruits",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
assert.ErrorContains(t, err, "references loop detected. schema ref trace: #/components/schemas/fruits -> #/components/schemas/foo")
|
||||
}
|
||||
|
||||
func TestReferenceResolutionForMapInObject(t *testing.T) {
|
||||
specString := `
|
||||
{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "number"
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mangos": {
|
||||
"type": "object",
|
||||
"description": "multiple mangos",
|
||||
"$ref": "#/components/schemas/mango"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mango": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"description": "a single mango",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mangos": {
|
||||
"type": "object",
|
||||
"description": "multiple mangos",
|
||||
"additionalProperties": {
|
||||
"type": "number",
|
||||
"description": "a single mango"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
||||
func TestReferenceResolutionForArrayInObject(t *testing.T) {
|
||||
specString := `{
|
||||
"components": {
|
||||
"schemas": {
|
||||
"foo": {
|
||||
"type": "number"
|
||||
},
|
||||
"fruits": {
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mangos": {
|
||||
"type": "object",
|
||||
"description": "multiple mangos",
|
||||
"$ref": "#/components/schemas/mango"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mango": {
|
||||
"type": "object",
|
||||
"items": {
|
||||
"description": "a single mango",
|
||||
"$ref": "#/components/schemas/foo"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
spec := &openapi.Specification{}
|
||||
reader := &OpenapiReader{
|
||||
OpenapiSpec: spec,
|
||||
Memo: make(map[string]*Schema),
|
||||
}
|
||||
err := json.Unmarshal([]byte(specString), spec)
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
|
||||
require.NoError(t, err)
|
||||
|
||||
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := `{
|
||||
"type": "object",
|
||||
"description": "fruits that are cool",
|
||||
"properties": {
|
||||
"guava": {
|
||||
"type": "string",
|
||||
"description": "a guava for my schema"
|
||||
},
|
||||
"mangos": {
|
||||
"type": "object",
|
||||
"description": "multiple mangos",
|
||||
"items": {
|
||||
"type": "number",
|
||||
"description": "a single mango"
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
|
||||
t.Log("[DEBUG] expected: ", expected)
|
||||
assert.Equal(t, expected, string(fruitsSchemaJson))
|
||||
}
|
||||
|
|
@ -35,6 +35,9 @@ type Schema struct {
|
|||
// Required properties for the object. Any fields missing the "omitempty"
|
||||
// json tag will be included
|
||||
Required []string `json:"required,omitempty"`
|
||||
|
||||
// URI to a json schema
|
||||
Reference *string `json:"$ref,omitempty"`
|
||||
}
|
||||
|
||||
// This function translates golang types into json schema. Here is the mapping
|
||||
|
|
@ -187,7 +190,7 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
|
|||
schema := &Schema{Type: rootJavascriptType}
|
||||
|
||||
if docs != nil {
|
||||
schema.Description = docs.Documentation
|
||||
schema.Description = docs.Description
|
||||
}
|
||||
|
||||
// case array/slice
|
||||
|
|
@ -197,7 +200,11 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
elemProps, err := safeToSchema(elemGolangType, docs, "", tracker)
|
||||
var childDocs *Docs
|
||||
if docs != nil {
|
||||
childDocs = docs.Items
|
||||
}
|
||||
elemProps, err := safeToSchema(elemGolangType, childDocs, "", tracker)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -215,7 +222,11 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
|
|||
if golangType.Key().Kind() != reflect.String {
|
||||
return nil, fmt.Errorf("only string keyed maps allowed")
|
||||
}
|
||||
schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), docs, "", tracker)
|
||||
var childDocs *Docs
|
||||
if docs != nil {
|
||||
childDocs = docs.AdditionalProperties
|
||||
}
|
||||
schema.AdditionalProperties, err = safeToSchema(golangType.Elem(), childDocs, "", tracker)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -240,8 +251,8 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*Schema, e
|
|||
// get docs for the child if they exist
|
||||
var childDocs *Docs
|
||||
if docs != nil {
|
||||
if val, ok := docs.Children[childName]; ok {
|
||||
childDocs = &val
|
||||
if val, ok := docs.Properties[childName]; ok {
|
||||
childDocs = val
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1062,60 +1062,31 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
|
|||
assert.Equal(t, expectedSchema, string(jsonSchema))
|
||||
}
|
||||
|
||||
func TestDocIngestionInSchema(t *testing.T) {
|
||||
func TestDocIngestionForObject(t *testing.T) {
|
||||
docs := &Docs{
|
||||
Documentation: "docs for root",
|
||||
Children: map[string]Docs{
|
||||
Description: "docs for root",
|
||||
Properties: map[string]*Docs{
|
||||
"my_struct": {
|
||||
Documentation: "docs for my struct",
|
||||
},
|
||||
"my_val": {
|
||||
Documentation: "docs for my val",
|
||||
},
|
||||
"my_slice": {
|
||||
Documentation: "docs for my slice",
|
||||
Children: map[string]Docs{
|
||||
"guava": {
|
||||
Documentation: "docs for guava",
|
||||
Description: "docs for my struct",
|
||||
Properties: map[string]*Docs{
|
||||
"a": {
|
||||
Description: "docs for a",
|
||||
},
|
||||
"pineapple": {
|
||||
Documentation: "docs for pineapple",
|
||||
},
|
||||
},
|
||||
},
|
||||
"my_map": {
|
||||
Documentation: "docs for my map",
|
||||
Children: map[string]Docs{
|
||||
"apple": {
|
||||
Documentation: "docs for apple",
|
||||
},
|
||||
"mango": {
|
||||
Documentation: "docs for mango",
|
||||
"c": {
|
||||
Description: "docs for c which does not exist on my_struct",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type Foo struct {
|
||||
Apple int `json:"apple"`
|
||||
Mango int `json:"mango"`
|
||||
}
|
||||
|
||||
type Bar struct {
|
||||
Guava int `json:"guava"`
|
||||
Pineapple int `json:"pineapple"`
|
||||
}
|
||||
|
||||
type MyStruct struct {
|
||||
A string `json:"a"`
|
||||
B int `json:"b"`
|
||||
}
|
||||
|
||||
type Root struct {
|
||||
MyStruct *MyStruct `json:"my_struct"`
|
||||
MyVal int `json:"my_val"`
|
||||
MySlice []Bar `json:"my_slice"`
|
||||
MyMap map[string]*Foo `json:"my_map"`
|
||||
MyStruct *MyStruct `json:"my_struct"`
|
||||
}
|
||||
|
||||
elem := Root{}
|
||||
|
|
@ -1131,29 +1102,82 @@ func TestDocIngestionInSchema(t *testing.T) {
|
|||
"type": "object",
|
||||
"description": "docs for root",
|
||||
"properties": {
|
||||
"my_map": {
|
||||
"my_struct": {
|
||||
"type": "object",
|
||||
"description": "docs for my map",
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"description": "docs for my map",
|
||||
"properties": {
|
||||
"apple": {
|
||||
"type": "number",
|
||||
"description": "docs for apple"
|
||||
},
|
||||
"mango": {
|
||||
"type": "number",
|
||||
"description": "docs for mango"
|
||||
}
|
||||
"description": "docs for my struct",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "string",
|
||||
"description": "docs for a"
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"apple",
|
||||
"mango"
|
||||
]
|
||||
}
|
||||
"b": {
|
||||
"type": "number"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"a",
|
||||
"b"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"my_struct"
|
||||
]
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(jsonSchema))
|
||||
t.Log("[DEBUG] expected: ", expectedSchema)
|
||||
|
||||
assert.Equal(t, expectedSchema, string(jsonSchema))
|
||||
}
|
||||
|
||||
func TestDocIngestionForSlice(t *testing.T) {
|
||||
docs := &Docs{
|
||||
Description: "docs for root",
|
||||
Properties: map[string]*Docs{
|
||||
"my_slice": {
|
||||
Description: "docs for my slice",
|
||||
Items: &Docs{
|
||||
Properties: map[string]*Docs{
|
||||
"guava": {
|
||||
Description: "docs for guava",
|
||||
},
|
||||
"pineapple": {
|
||||
Description: "docs for pineapple",
|
||||
},
|
||||
"watermelon": {
|
||||
Description: "docs for watermelon which does not exist in schema",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type Bar struct {
|
||||
Guava int `json:"guava"`
|
||||
Pineapple int `json:"pineapple"`
|
||||
}
|
||||
|
||||
type Root struct {
|
||||
MySlice []Bar `json:"my_slice"`
|
||||
}
|
||||
|
||||
elem := Root{}
|
||||
|
||||
schema, err := New(reflect.TypeOf(elem), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
|
||||
assert.NoError(t, err)
|
||||
|
||||
expectedSchema :=
|
||||
`{
|
||||
"type": "object",
|
||||
"description": "docs for root",
|
||||
"properties": {
|
||||
"my_slice": {
|
||||
"type": "array",
|
||||
"description": "docs for my slice",
|
||||
|
|
@ -1175,20 +1199,130 @@ func TestDocIngestionInSchema(t *testing.T) {
|
|||
"pineapple"
|
||||
]
|
||||
}
|
||||
},
|
||||
"my_struct": {
|
||||
"type": "object",
|
||||
"description": "docs for my struct",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"my_slice"
|
||||
]
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(jsonSchema))
|
||||
t.Log("[DEBUG] expected: ", expectedSchema)
|
||||
|
||||
assert.Equal(t, expectedSchema, string(jsonSchema))
|
||||
}
|
||||
|
||||
func TestDocIngestionForMap(t *testing.T) {
|
||||
docs := &Docs{
|
||||
Description: "docs for root",
|
||||
Properties: map[string]*Docs{
|
||||
"my_map": {
|
||||
Description: "docs for my map",
|
||||
AdditionalProperties: &Docs{
|
||||
Properties: map[string]*Docs{
|
||||
"apple": {
|
||||
Description: "docs for apple",
|
||||
},
|
||||
"mango": {
|
||||
Description: "docs for mango",
|
||||
},
|
||||
"watermelon": {
|
||||
Description: "docs for watermelon which does not exist in schema",
|
||||
},
|
||||
"papaya": {
|
||||
Description: "docs for papaya which does not exist in schema",
|
||||
},
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"a"
|
||||
]
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type Foo struct {
|
||||
Apple int `json:"apple"`
|
||||
Mango int `json:"mango"`
|
||||
}
|
||||
|
||||
type Root struct {
|
||||
MyMap map[string]*Foo `json:"my_map"`
|
||||
}
|
||||
|
||||
elem := Root{}
|
||||
|
||||
schema, err := New(reflect.TypeOf(elem), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
|
||||
assert.NoError(t, err)
|
||||
|
||||
expectedSchema :=
|
||||
`{
|
||||
"type": "object",
|
||||
"description": "docs for root",
|
||||
"properties": {
|
||||
"my_map": {
|
||||
"type": "object",
|
||||
"description": "docs for my map",
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"apple": {
|
||||
"type": "number",
|
||||
"description": "docs for apple"
|
||||
},
|
||||
"mango": {
|
||||
"type": "number",
|
||||
"description": "docs for mango"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"apple",
|
||||
"mango"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"my_map"
|
||||
]
|
||||
}`
|
||||
|
||||
t.Log("[DEBUG] actual: ", string(jsonSchema))
|
||||
t.Log("[DEBUG] expected: ", expectedSchema)
|
||||
|
||||
assert.Equal(t, expectedSchema, string(jsonSchema))
|
||||
}
|
||||
|
||||
func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
|
||||
docs := &Docs{
|
||||
Description: "docs for root",
|
||||
Properties: map[string]*Docs{
|
||||
"my_val": {
|
||||
Description: "docs for my val",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
type Root struct {
|
||||
MyVal int `json:"my_val"`
|
||||
}
|
||||
|
||||
elem := Root{}
|
||||
|
||||
schema, err := New(reflect.TypeOf(elem), docs)
|
||||
require.NoError(t, err)
|
||||
|
||||
jsonSchema, err := json.MarshalIndent(schema, " ", " ")
|
||||
assert.NoError(t, err)
|
||||
|
||||
expectedSchema :=
|
||||
`{
|
||||
"type": "object",
|
||||
"description": "docs for root",
|
||||
"properties": {
|
||||
"my_val": {
|
||||
"type": "number",
|
||||
"description": "docs for my val"
|
||||
|
|
@ -1196,10 +1330,7 @@ func TestDocIngestionInSchema(t *testing.T) {
|
|||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"my_struct",
|
||||
"my_val",
|
||||
"my_slice",
|
||||
"my_map"
|
||||
"my_val"
|
||||
]
|
||||
}`
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
package bundle
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var launchCmd = &cobra.Command{
|
||||
Use: "launch",
|
||||
Short: "Launches a notebook on development cluster",
|
||||
Long: `Reads a file and executes it on dev cluster`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
|
||||
// We're not ready to expose this command until we specify its semantics.
|
||||
Hidden: true,
|
||||
|
||||
PreRunE: root.MustConfigureBundle,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return fmt.Errorf("TODO")
|
||||
// contents, err := os.ReadFile(args[0])
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// results := project.RunPythonOnDev(cmd.Context(), string(contents))
|
||||
// if results.Failed() {
|
||||
// return results.Err()
|
||||
// }
|
||||
// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
|
||||
// return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
AddCommand(launchCmd)
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ var runCmd = &cobra.Command{
|
|||
b := bundle.Get(cmd.Context())
|
||||
err := bundle.Apply(cmd.Context(), b, []bundle.Mutator{
|
||||
phases.Initialize(),
|
||||
terraform.Initialize(),
|
||||
terraform.Load(),
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ var schemaCmd = &cobra.Command{
|
|||
Short: "Generate JSON Schema for bundle configuration",
|
||||
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
docs, err := schema.GetBundleDocs()
|
||||
docs, err := schema.BundleDocs(openapi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -22,15 +22,26 @@ var schemaCmd = &cobra.Command{
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jsonSchema, err := json.MarshalIndent(schema, "", " ")
|
||||
result, err := json.MarshalIndent(schema, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cmd.OutOrStdout().Write(jsonSchema)
|
||||
if onlyDocs {
|
||||
result, err = json.MarshalIndent(docs, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
cmd.OutOrStdout().Write(result)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var openapi string
|
||||
var onlyDocs bool
|
||||
|
||||
func init() {
|
||||
AddCommand(schemaCmd)
|
||||
schemaCmd.Flags().StringVar(&openapi, "openapi", "", "path to a databricks openapi spec")
|
||||
schemaCmd.Flags().BoolVar(&onlyDocs, "only-docs", false, "only generate descriptions for the schema")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
package bundle
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var testCmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "run tests for the project",
|
||||
Long: `This is longer description of the command`,
|
||||
|
||||
// We're not ready to expose this command until we specify its semantics.
|
||||
Hidden: true,
|
||||
|
||||
PreRunE: root.MustConfigureBundle,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return fmt.Errorf("TODO")
|
||||
// results := project.RunPythonOnDev(cmd.Context(), `return 1`)
|
||||
// if results.Failed() {
|
||||
// return results.Err()
|
||||
// }
|
||||
// fmt.Fprintf(cmd.OutOrStdout(), "Success: %s", results.Text())
|
||||
// return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
AddCommand(testCmd)
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
package launch
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// launchCmd represents the launch command
|
||||
var launchCmd = &cobra.Command{
|
||||
Use: "launch",
|
||||
Short: "Launches a notebook on development cluster",
|
||||
Long: `Reads a file and executes it on dev cluster`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
contents, err := os.ReadFile(args[0])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
results := project.RunPythonOnDev(cmd.Context(), string(contents))
|
||||
if results.Failed() {
|
||||
log.Fatal(results.Error())
|
||||
}
|
||||
fmt.Println(results.Text())
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
// TODO: detect if we can skip registering
|
||||
// launch command for DBSQL projects
|
||||
root.RootCmd.AddCommand(launchCmd)
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
"github.com/databricks/bricks/project"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// testCmd represents the test command
|
||||
var testCmd = &cobra.Command{
|
||||
Use: "test",
|
||||
Short: "run tests for the project",
|
||||
Long: `This is longer description of the command`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
results := project.RunPythonOnDev(cmd.Context(), `return 1`)
|
||||
if results.Failed() {
|
||||
log.Fatal(results.Error())
|
||||
}
|
||||
log.Printf("Success: %s", results.Text())
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
root.RootCmd.AddCommand(testCmd)
|
||||
|
||||
// Here you will define your flags and configuration settings.
|
||||
|
||||
// Cobra supports Persistent Flags which will work for this command
|
||||
// and all subcommands, e.g.:
|
||||
// testCmd.PersistentFlags().String("foo", "", "A help for foo")
|
||||
|
||||
// Cobra supports local flags which will only run when this command
|
||||
// is called directly, e.g.:
|
||||
// testCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
|
||||
}
|
||||
9
go.mod
9
go.mod
|
|
@ -22,17 +22,18 @@ require (
|
|||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/go-version v1.6.0
|
||||
github.com/hashicorp/hc-install v0.5.0
|
||||
github.com/hashicorp/terraform-exec v0.17.3
|
||||
github.com/hashicorp/terraform-json v0.15.0
|
||||
github.com/hashicorp/terraform-exec v0.18.1
|
||||
github.com/hashicorp/terraform-json v0.16.0
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
|
||||
golang.org/x/sync v0.1.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/zclconf/go-cty v1.11.0 // indirect
|
||||
github.com/zclconf/go-cty v1.13.0 // indirect
|
||||
golang.org/x/crypto v0.5.0 // indirect
|
||||
)
|
||||
|
||||
|
|
@ -61,5 +62,5 @@ require (
|
|||
google.golang.org/grpc v1.53.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
47
go.sum
47
go.sum
|
|
@ -15,7 +15,7 @@ github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/
|
|||
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
||||
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
|
|
@ -33,8 +33,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
<<<<<<< HEAD
|
||||
github.com/databricks/databricks-sdk-go v0.5.0 h1:u5tytS0+BHZXLS2cqg4gW+6LOGgfXOriBuT5bt+gTSk=
|
||||
github.com/databricks/databricks-sdk-go v0.5.0/go.mod h1:MTJHMjKBotE0IsyPbMcPvHb3JKknkRtOms/ET7F0rB8=
|
||||
=======
|
||||
github.com/databricks/databricks-sdk-go v0.4.1 h1:SX1Owt2XcY/NVt/LRbuO/0dtomEebfNa8yekQlQy4gM=
|
||||
github.com/databricks/databricks-sdk-go v0.4.1/go.mod h1:MOtUlDw9bj/1PuYmFkNSx+29Hu1clrob8vQQ6Fsji8A=
|
||||
>>>>>>> origin
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
|
@ -63,11 +68,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
|
|||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
|
|
@ -110,10 +113,10 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
|
|||
github.com/hashicorp/hc-install v0.5.0 h1:D9bl4KayIYKEeJ4vUDe9L5huqxZXczKaykSRcmQ0xY0=
|
||||
github.com/hashicorp/hc-install v0.5.0/go.mod h1:JyzMfbzfSBSjoDCRPna1vi/24BEDxFaCPfdHtM5SCdo=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/terraform-exec v0.17.3 h1:MX14Kvnka/oWGmIkyuyvL6POx25ZmKrjlaclkx3eErU=
|
||||
github.com/hashicorp/terraform-exec v0.17.3/go.mod h1:+NELG0EqQekJzhvikkeQsOAZpsw0cv/03rbeQJqscAI=
|
||||
github.com/hashicorp/terraform-json v0.15.0 h1:/gIyNtR6SFw6h5yzlbDbACyGvIhKtQi8mTsbkNd79lE=
|
||||
github.com/hashicorp/terraform-json v0.15.0/go.mod h1:+L1RNzjDU5leLFZkHTFTbJXaoqUC6TqXlFgDoOXrtvk=
|
||||
github.com/hashicorp/terraform-exec v0.18.1 h1:LAbfDvNQU1l0NOQlTuudjczVhHj061fNX5H8XZxHlH4=
|
||||
github.com/hashicorp/terraform-exec v0.18.1/go.mod h1:58wg4IeuAJ6LVsLUeD2DWZZoc/bYi6dzhLHzxM41980=
|
||||
github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8c+85Albo3s=
|
||||
github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0=
|
||||
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
|
||||
|
|
@ -135,7 +138,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
|
||||
github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
|
||||
github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
|
||||
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
|
||||
|
|
@ -148,11 +150,9 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
|
|||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4=
|
||||
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
|
||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
|
|
@ -165,7 +165,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
|
|||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
|
|
@ -180,7 +179,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
|
|
@ -190,19 +188,19 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
|||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
<<<<<<< HEAD
|
||||
github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
||||
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
|
||||
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||
=======
|
||||
>>>>>>> origin
|
||||
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
|
||||
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
|
||||
github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
|
||||
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
|
||||
github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
|
||||
github.com/zclconf/go-cty v1.11.0 h1:726SxLdi2SDnjY+BStqB9J1hNp4+2WlzyXLuimibIe0=
|
||||
github.com/zclconf/go-cty v1.11.0/go.mod h1:s9IfD1LK5ccNMSWCVFCE2rJfHiZgi7JijgeWIMfhLvA=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
|
||||
github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0=
|
||||
github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
|
|
@ -226,14 +224,12 @@ golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
|||
golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
|
||||
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
|
||||
|
|
@ -277,7 +273,6 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
|
||||
|
|
@ -293,18 +288,27 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
<<<<<<< HEAD
|
||||
google.golang.org/api v0.112.0 h1:iDmzvZ4C086R3+en4nSyIf07HlQKMOX1Xx2dmia/+KQ=
|
||||
google.golang.org/api v0.112.0/go.mod h1:737UfWHNsOq4F3REUTmb+GN9pugkgNLCayLTfoIKpPc=
|
||||
=======
|
||||
google.golang.org/api v0.111.0 h1:bwKi+z2BsdwYFRKrqwutM+axAlYLz83gt5pDSXCJT+0=
|
||||
google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0=
|
||||
>>>>>>> origin
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
<<<<<<< HEAD
|
||||
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos=
|
||||
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA=
|
||||
=======
|
||||
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds=
|
||||
google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
>>>>>>> origin
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
|
|
@ -326,7 +330,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
|
|||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
|
|
|||
|
|
@ -165,13 +165,17 @@ func loadOrNewSnapshot(opts *SyncOptions) (*Snapshot, error) {
|
|||
}
|
||||
|
||||
func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
|
||||
currentFilenames := map[string]bool{}
|
||||
lastModifiedTimes := s.LastUpdatedTimes
|
||||
remoteToLocalNames := s.RemoteToLocalNames
|
||||
localToRemoteNames := s.LocalToRemoteNames
|
||||
|
||||
// set of files currently present in the local file system and tracked by git
|
||||
localFileSet := map[string]struct{}{}
|
||||
for _, f := range all {
|
||||
localFileSet[f.Relative] = struct{}{}
|
||||
}
|
||||
|
||||
for _, f := range all {
|
||||
// create set of current files to figure out if removals are needed
|
||||
currentFilenames[f.Relative] = true
|
||||
// get current modified timestamp
|
||||
modified := f.Modified()
|
||||
lastSeenModified, seen := lastModifiedTimes[f.Relative]
|
||||
|
|
@ -207,11 +211,13 @@ func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
|
|||
change.delete = append(change.delete, oldRemoteName)
|
||||
delete(remoteToLocalNames, oldRemoteName)
|
||||
}
|
||||
|
||||
// We cannot allow two local files in the project to point to the same
|
||||
// remote path
|
||||
oldLocalName, ok := remoteToLocalNames[remoteName]
|
||||
if ok && oldLocalName != f.Relative {
|
||||
return change, fmt.Errorf("both %s and %s point to the same remote file location %s. Please remove one of them from your local project", oldLocalName, f.Relative, remoteName)
|
||||
prevLocalName, ok := remoteToLocalNames[remoteName]
|
||||
_, prevLocalFileExists := localFileSet[prevLocalName]
|
||||
if ok && prevLocalName != f.Relative && prevLocalFileExists {
|
||||
return change, fmt.Errorf("both %s and %s point to the same remote file location %s. Please remove one of them from your local project", prevLocalName, f.Relative, remoteName)
|
||||
}
|
||||
localToRemoteNames[f.Relative] = remoteName
|
||||
remoteToLocalNames[remoteName] = f.Relative
|
||||
|
|
@ -220,7 +226,7 @@ func (s *Snapshot) diff(all []fileset.File) (change diff, err error) {
|
|||
// figure out files in the snapshot.lastModifiedTimes, but not on local
|
||||
// filesystem. These will be deleted
|
||||
for localName := range lastModifiedTimes {
|
||||
_, exists := currentFilenames[localName]
|
||||
_, exists := localFileSet[localName]
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
|
|
|
|||
|
|
@ -249,6 +249,43 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) {
|
|||
assert.ErrorContains(t, err, "both foo and foo.py point to the same remote file location foo. Please remove one of them from your local project")
|
||||
}
|
||||
|
||||
func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) {
|
||||
// Create temp project dir
|
||||
projectDir := t.TempDir()
|
||||
fileSet, err := git.NewFileSet(projectDir)
|
||||
require.NoError(t, err)
|
||||
state := Snapshot{
|
||||
LastUpdatedTimes: make(map[string]time.Time),
|
||||
LocalToRemoteNames: make(map[string]string),
|
||||
RemoteToLocalNames: make(map[string]string),
|
||||
}
|
||||
|
||||
// upload should work since they point to different destinations
|
||||
pythonFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.py"))
|
||||
defer pythonFoo.Close(t)
|
||||
pythonFoo.Overwrite(t, "# Databricks notebook source\n")
|
||||
files, err := fileSet.All()
|
||||
assert.NoError(t, err)
|
||||
change, err := state.diff(files)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, change.delete, 0)
|
||||
assert.Len(t, change.put, 1)
|
||||
assert.Contains(t, change.put, "foo.py")
|
||||
|
||||
pythonFoo.Remove(t)
|
||||
sqlFoo := testfile.CreateFile(t, filepath.Join(projectDir, "foo.sql"))
|
||||
defer sqlFoo.Close(t)
|
||||
sqlFoo.Overwrite(t, "-- Databricks notebook source\n")
|
||||
files, err = fileSet.All()
|
||||
assert.NoError(t, err)
|
||||
change, err = state.diff(files)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, change.delete, 1)
|
||||
assert.Len(t, change.put, 1)
|
||||
assert.Contains(t, change.put, "foo.sql")
|
||||
assert.Contains(t, change.delete, "foo")
|
||||
}
|
||||
|
||||
func defaultOptions(t *testing.T) *SyncOptions {
|
||||
return &SyncOptions{
|
||||
Host: "www.foobar.com",
|
||||
|
|
|
|||
|
|
@ -118,8 +118,6 @@ func (s *Sync) notifyComplete(ctx context.Context, d diff) {
|
|||
}
|
||||
|
||||
func (s *Sync) RunOnce(ctx context.Context) error {
|
||||
applyDiff := syncCallback(ctx, s)
|
||||
|
||||
// tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement
|
||||
// https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418
|
||||
all, err := s.fileSet.All()
|
||||
|
|
@ -139,7 +137,7 @@ func (s *Sync) RunOnce(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
err = applyDiff(change)
|
||||
err = s.applyDiff(ctx, change)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,53 +6,63 @@ import (
|
|||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// See https://docs.databricks.com/resources/limits.html#limits-api-rate-limits for per api
|
||||
// rate limits
|
||||
// Maximum number of concurrent requests during sync.
|
||||
const MaxRequestsInFlight = 20
|
||||
|
||||
func syncCallback(ctx context.Context, s *Sync) func(localDiff diff) error {
|
||||
return func(d diff) error {
|
||||
// Abstraction over wait groups which allows you to get the errors
|
||||
// returned in goroutines
|
||||
var g errgroup.Group
|
||||
// Perform a DELETE of the specified remote path.
|
||||
func (s *Sync) applyDelete(ctx context.Context, group *errgroup.Group, remoteName string) {
|
||||
// Return early if the context has already been cancelled.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
// Proceed.
|
||||
}
|
||||
|
||||
// Allow MaxRequestLimit maxiumum concurrent api calls
|
||||
g.SetLimit(MaxRequestsInFlight)
|
||||
|
||||
for _, remoteName := range d.delete {
|
||||
// Copy of remoteName created to make this safe for concurrent use.
|
||||
// directly using remoteName can cause race conditions since the loop
|
||||
// might iterate over to the next remoteName before the go routine function
|
||||
// is evaluated
|
||||
remoteNameCopy := remoteName
|
||||
g.Go(func() error {
|
||||
s.notifyProgress(ctx, EventActionDelete, remoteNameCopy, 0.0)
|
||||
err := s.repoFiles.DeleteFile(ctx, remoteNameCopy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.notifyProgress(ctx, EventActionDelete, remoteNameCopy, 1.0)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
for _, localRelativePath := range d.put {
|
||||
// Copy of localName created to make this safe for concurrent use.
|
||||
localRelativePathCopy := localRelativePath
|
||||
g.Go(func() error {
|
||||
s.notifyProgress(ctx, EventActionPut, localRelativePathCopy, 0.0)
|
||||
err := s.repoFiles.PutFile(ctx, localRelativePathCopy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.notifyProgress(ctx, EventActionPut, localRelativePathCopy, 1.0)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
// wait for goroutines to finish and return first non-nil error return
|
||||
// if any
|
||||
if err := g.Wait(); err != nil {
|
||||
group.Go(func() error {
|
||||
s.notifyProgress(ctx, EventActionDelete, remoteName, 0.0)
|
||||
err := s.repoFiles.DeleteFile(ctx, remoteName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.notifyProgress(ctx, EventActionDelete, remoteName, 1.0)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Perform a PUT of the specified local path.
|
||||
func (s *Sync) applyPut(ctx context.Context, group *errgroup.Group, localName string) {
|
||||
// Return early if the context has already been cancelled.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
// Proceed.
|
||||
}
|
||||
|
||||
group.Go(func() error {
|
||||
s.notifyProgress(ctx, EventActionPut, localName, 0.0)
|
||||
err := s.repoFiles.PutFile(ctx, localName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.notifyProgress(ctx, EventActionPut, localName, 1.0)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Sync) applyDiff(ctx context.Context, d diff) error {
|
||||
group, ctx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(MaxRequestsInFlight)
|
||||
|
||||
for _, remoteName := range d.delete {
|
||||
s.applyDelete(ctx, group, remoteName)
|
||||
}
|
||||
|
||||
for _, localName := range d.put {
|
||||
s.applyPut(ctx, group, localName)
|
||||
}
|
||||
|
||||
// Wait for goroutines to finish and return first non-nil error return if any.
|
||||
return group.Wait()
|
||||
}
|
||||
|
|
|
|||
2
main.go
2
main.go
|
|
@ -9,10 +9,8 @@ import (
|
|||
_ "github.com/databricks/bricks/cmd/configure"
|
||||
_ "github.com/databricks/bricks/cmd/fs"
|
||||
_ "github.com/databricks/bricks/cmd/init"
|
||||
_ "github.com/databricks/bricks/cmd/launch"
|
||||
"github.com/databricks/bricks/cmd/root"
|
||||
_ "github.com/databricks/bricks/cmd/sync"
|
||||
_ "github.com/databricks/bricks/cmd/test"
|
||||
_ "github.com/databricks/bricks/cmd/version"
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue