diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 0aa4b1028..8c62ac620 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -94684175b8bd65f8701f89729351f8069e8309c9 \ No newline at end of file +7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92 \ No newline at end of file diff --git a/.codegen/service.go.tmpl b/.codegen/service.go.tmpl index 6aabb02c9..ad482ebe6 100644 --- a/.codegen/service.go.tmpl +++ b/.codegen/service.go.tmpl @@ -39,6 +39,7 @@ import ( {{define "service"}} {{- $excludeMethods := list "put-secret" -}} +{{- $hideService := .IsPrivatePreview }} // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. @@ -57,7 +58,7 @@ func New() *cobra.Command { "package": "{{ .Package.Name }}", }, {{- end }} - {{- if .IsPrivatePreview }} + {{- if $hideService }} // This service is being previewed; hide from help output. Hidden: true, @@ -151,6 +152,7 @@ func new{{.PascalName}}() *cobra.Command { "provider-exchanges delete" "provider-exchanges delete-listing-from-exchange" "provider-exchanges list-exchanges-for-listing" + "provider-exchanges list-listings-for-exchange" -}} {{- $fullCommandName := (print $serviceName " " .KebabName) -}} {{- $noPrompt := or .IsCrudCreate (in $excludeFromPrompts $fullCommandName) }} @@ -189,7 +191,8 @@ func new{{.PascalName}}() *cobra.Command { {{- end -}} ` {{- end }} - {{- if .IsPrivatePreview }} + {{/* Don't hide commands if the service itself is already hidden. */}} + {{- if and (not $hideService) .IsPrivatePreview }} // This command is being previewed; hide from help output. cmd.Hidden = true diff --git a/.gitattributes b/.gitattributes index f9aa02d18..c11257e9e 100755 --- a/.gitattributes +++ b/.gitattributes @@ -37,6 +37,7 @@ cmd/workspace/clean-rooms/clean-rooms.go linguist-generated=true cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true cmd/workspace/clusters/clusters.go linguist-generated=true cmd/workspace/cmd.go linguist-generated=true +cmd/workspace/compliance-security-profile/compliance-security-profile.go linguist-generated=true cmd/workspace/connections/connections.go linguist-generated=true cmd/workspace/consumer-fulfillments/consumer-fulfillments.go linguist-generated=true cmd/workspace/consumer-installations/consumer-installations.go linguist-generated=true @@ -44,13 +45,12 @@ cmd/workspace/consumer-listings/consumer-listings.go linguist-generated=true cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go linguist-generated=true cmd/workspace/consumer-providers/consumer-providers.go linguist-generated=true cmd/workspace/credentials-manager/credentials-manager.go linguist-generated=true -cmd/workspace/csp-enablement/csp-enablement.go linguist-generated=true cmd/workspace/current-user/current-user.go linguist-generated=true cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true cmd/workspace/dashboards/dashboards.go linguist-generated=true cmd/workspace/data-sources/data-sources.go linguist-generated=true cmd/workspace/default-namespace/default-namespace.go linguist-generated=true -cmd/workspace/esm-enablement/esm-enablement.go linguist-generated=true +cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true cmd/workspace/experiments/experiments.go linguist-generated=true cmd/workspace/external-locations/external-locations.go linguist-generated=true cmd/workspace/functions/functions.go linguist-generated=true @@ -62,7 +62,6 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true cmd/workspace/jobs/jobs.go linguist-generated=true -cmd/workspace/lakehouse-monitors/lakehouse-monitors.go linguist-generated=true cmd/workspace/lakeview/lakeview.go linguist-generated=true cmd/workspace/libraries/libraries.go linguist-generated=true cmd/workspace/metastores/metastores.go linguist-generated=true @@ -81,6 +80,7 @@ cmd/workspace/provider-personalization-requests/provider-personalization-request cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go linguist-generated=true cmd/workspace/provider-providers/provider-providers.go linguist-generated=true cmd/workspace/providers/providers.go linguist-generated=true +cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true cmd/workspace/queries/queries.go linguist-generated=true cmd/workspace/query-history/query-history.go linguist-generated=true cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b74498ec..2fb35d479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,81 @@ # Version changelog +## 0.220.0 + +CLI: + * Add line about Docker installation to README.md ([#1363](https://github.com/databricks/cli/pull/1363)). + * Improve token refresh flow ([#1434](https://github.com/databricks/cli/pull/1434)). + +Bundles: + * Upgrade Terraform provider to v1.42.0 ([#1418](https://github.com/databricks/cli/pull/1418)). + * Upgrade Terraform provider to v1.43.0 ([#1429](https://github.com/databricks/cli/pull/1429)). + * Don't merge-in remote resources during deployments ([#1432](https://github.com/databricks/cli/pull/1432)). + * Remove dependency on `ConfigFilePath` from path translation mutator ([#1437](https://github.com/databricks/cli/pull/1437)). + * Add `merge.Override` transform ([#1428](https://github.com/databricks/cli/pull/1428)). + * Fixed panic when loading incorrectly defined jobs ([#1402](https://github.com/databricks/cli/pull/1402)). + * Add more tests for `merge.Override` ([#1439](https://github.com/databricks/cli/pull/1439)). + * Fixed seg fault when specifying environment key for tasks ([#1443](https://github.com/databricks/cli/pull/1443)). + * Fix conversion of zero valued scalar pointers to a dynamic value ([#1433](https://github.com/databricks/cli/pull/1433)). + +Internal: + * Don't hide commands of services that are already hidden ([#1438](https://github.com/databricks/cli/pull/1438)). + +API Changes: + * Renamed `lakehouse-monitors` command group to `quality-monitors`. + * Added `apps` command group. + * Renamed `csp-enablement` command group to `compliance-security-profile`. + * Renamed `esm-enablement` command group to `enhanced-security-monitoring`. + * Added `databricks vector-search-indexes scan-index` command. + +OpenAPI commit 7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92 (2024-05-21) + +Dependency updates: + * Bump golang.org/x/text from 0.14.0 to 0.15.0 ([#1419](https://github.com/databricks/cli/pull/1419)). + * Bump golang.org/x/oauth2 from 0.19.0 to 0.20.0 ([#1421](https://github.com/databricks/cli/pull/1421)). + * Bump golang.org/x/term from 0.19.0 to 0.20.0 ([#1422](https://github.com/databricks/cli/pull/1422)). + * Bump github.com/databricks/databricks-sdk-go from 0.39.0 to 0.40.1 ([#1431](https://github.com/databricks/cli/pull/1431)). + * Bump github.com/fatih/color from 1.16.0 to 1.17.0 ([#1441](https://github.com/databricks/cli/pull/1441)). + * Bump github.com/hashicorp/terraform-json from 0.21.0 to 0.22.1 ([#1440](https://github.com/databricks/cli/pull/1440)). + * Bump github.com/hashicorp/terraform-exec from 0.20.0 to 0.21.0 ([#1442](https://github.com/databricks/cli/pull/1442)). + * Update Go SDK to v0.41.0 ([#1445](https://github.com/databricks/cli/pull/1445)). + +## 0.219.0 + +Bundles: + * Don't fail while parsing outdated terraform state ([#1404](https://github.com/databricks/cli/pull/1404)). + * Annotate DLT pipelines when deployed using DABs ([#1410](https://github.com/databricks/cli/pull/1410)). + + +API Changes: + * Changed `databricks libraries cluster-status` command. New request type is compute.ClusterStatus. + * Changed `databricks libraries cluster-status` command to return . + * Added `databricks serving-endpoints get-open-api` command. + +OpenAPI commit 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55 (2024-04-23) +Dependency updates: + * Bump github.com/databricks/databricks-sdk-go from 0.38.0 to 0.39.0 ([#1405](https://github.com/databricks/cli/pull/1405)). + +## 0.218.1 + +This is a bugfix release. + +CLI: + * Pass `DATABRICKS_CONFIG_FILE` for `auth profiles` ([#1394](https://github.com/databricks/cli/pull/1394)). + +Bundles: + * Show a better error message for using wheel tasks with older DBR versions ([#1373](https://github.com/databricks/cli/pull/1373)). + * Allow variable references in non-string fields in the JSON schema ([#1398](https://github.com/databricks/cli/pull/1398)). + * Fix variable overrides in targets for non-string variables ([#1397](https://github.com/databricks/cli/pull/1397)). + * Fix bundle schema for variables ([#1396](https://github.com/databricks/cli/pull/1396)). + * Fix bundle documentation URL ([#1399](https://github.com/databricks/cli/pull/1399)). + +Internal: + * Removed autogenerated docs for the CLI commands ([#1392](https://github.com/databricks/cli/pull/1392)). + * Remove `JSON.parse` call from homebrew-tap action ([#1393](https://github.com/databricks/cli/pull/1393)). + * Ensure that Python dependencies are installed during upgrade ([#1390](https://github.com/databricks/cli/pull/1390)). + + + ## 0.218.0 This release marks the general availability of Databricks Asset Bundles. diff --git a/README.md b/README.md index 83051ccf7..5f3b78b79 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,18 @@ See https://github.com/databricks/cli/releases for releases and [the docs pages](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) for installation instructions. +------ +You can use the CLI via a Docker image by pulling the image from `ghcr.io`. You can find all available versions +at: https://github.com/databricks/cli/pkgs/container/cli. +``` +docker pull ghcr.io/databricks/cli:latest +``` + +Example of how to run the CLI using the Docker image. More documentation is available at https://docs.databricks.com/dev-tools/bundles/airgapped-environment.html. +``` +docker run -e DATABRICKS_HOST=$YOUR_HOST_URL -e DATABRICKS_TOKEN=$YOUR_TOKEN ghcr.io/databricks/cli:latest current-user me +``` + ## Authentication This CLI follows the Databricks Unified Authentication principles. diff --git a/bundle/artifacts/artifacts.go b/bundle/artifacts/artifacts.go index 101b598dd..470c329a1 100644 --- a/bundle/artifacts/artifacts.go +++ b/bundle/artifacts/artifacts.go @@ -150,6 +150,10 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u for i := range job.Environments { env := &job.Environments[i] + if env.Spec == nil { + continue + } + for j := range env.Spec.Dependencies { lib := env.Spec.Dependencies[j] if isArtifactMatchLibrary(f, lib, b) { diff --git a/bundle/bundle.go b/bundle/bundle.go index 977ca2247..1dc98656a 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -22,6 +22,7 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/tags" "github.com/databricks/cli/libs/terraform" + "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/hashicorp/terraform-exec/tfexec" @@ -208,7 +209,7 @@ func (b *Bundle) GitRepository() (*git.Repository, error) { return nil, fmt.Errorf("unable to locate repository root: %w", err) } - return git.NewRepository(rootPath) + return git.NewRepository(vfs.MustNew(rootPath)) } // AuthEnv returns a map with environment variables and their values diff --git a/bundle/config/mutator/default_queueing_test.go b/bundle/config/mutator/default_queueing_test.go index ea60daf7f..d3621663b 100644 --- a/bundle/config/mutator/default_queueing_test.go +++ b/bundle/config/mutator/default_queueing_test.go @@ -56,7 +56,11 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) { Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ - "job": {}, + "job": { + JobSettings: &jobs.JobSettings{ + Name: "job", + }, + }, }, }, }, @@ -77,7 +81,11 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) { Queue: &jobs.QueueSettings{Enabled: false}, }, }, - "job2": {}, + "job2": { + JobSettings: &jobs.JobSettings{ + Name: "job", + }, + }, "job3": { JobSettings: &jobs.JobSettings{ Queue: &jobs.QueueSettings{Enabled: true}, diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 7ce8476f1..d8b76f39e 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/vfs" ) type loadGitDetails struct{} @@ -22,7 +23,7 @@ func (m *loadGitDetails) Name() string { func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Load relevant git repository - repo, err := git.NewRepository(b.RootPath) + repo, err := git.NewRepository(vfs.MustNew(b.RootPath)) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/process_target_mode_test.go b/bundle/config/mutator/process_target_mode_test.go index 583efcfe5..cf8229bfe 100644 --- a/bundle/config/mutator/process_target_mode_test.go +++ b/bundle/config/mutator/process_target_mode_test.go @@ -97,6 +97,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle { RegisteredModels: map[string]*resources.RegisteredModel{ "registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}}, }, + QualityMonitors: map[string]*resources.QualityMonitor{ + "qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}}, + }, }, }, // Use AWS implementation for testing. @@ -145,6 +148,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Registered model 1 assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) + + // Quality Monitor 1 + assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) } func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) { @@ -200,6 +206,7 @@ func TestProcessTargetModeDefault(t *testing.T) { assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) + assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) } func TestProcessTargetModeProduction(t *testing.T) { @@ -240,6 +247,7 @@ func TestProcessTargetModeProduction(t *testing.T) { assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development) assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) + assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName) } func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) { diff --git a/bundle/config/mutator/run_as.go b/bundle/config/mutator/run_as.go index 26efaea4e..e4091399f 100644 --- a/bundle/config/mutator/run_as.go +++ b/bundle/config/mutator/run_as.go @@ -86,6 +86,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { ) } + // Monitors do not support run_as in the API. + if len(b.Config.Resources.QualityMonitors) > 0 { + return errUnsupportedResourceTypeForRunAs{ + resourceType: "quality_monitors", + resourceLocation: b.Config.GetLocation("resources.quality_monitors"), + currentUser: b.Config.Workspace.CurrentUser.UserName, + runAsUser: identity, + } + } + return nil } diff --git a/bundle/config/mutator/run_as_test.go b/bundle/config/mutator/run_as_test.go index b6f536564..498171582 100644 --- a/bundle/config/mutator/run_as_test.go +++ b/bundle/config/mutator/run_as_test.go @@ -37,6 +37,7 @@ func allResourceTypes(t *testing.T) []string { "model_serving_endpoints", "models", "pipelines", + "quality_monitors", "registered_models", }, resourceTypes, diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index bb88379e0..eae1fe2ab 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -53,8 +53,6 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di } // We should have had a value to set for the variable at this point. - // TODO: use cmdio to request values for unassigned variables if current - // terminal is a tty. Tracked in https://github.com/databricks/cli/issues/379 return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name) } diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 018fd79c6..18a09dfd6 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -213,3 +213,31 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos return diag.FromErr(err) } + +func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) { + var fallback = make(map[string]string) + var pattern = dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey()) + + // Previous behavior was to use a resource's location as the base path to resolve + // relative paths in its definition. With the introduction of [dyn.Value] throughout, + // we can use the location of the [dyn.Value] of the relative path itself. + // + // This is more flexible, as resources may have overrides that are not + // located in the same directory as the resource configuration file. + // + // To maintain backwards compatibility, we allow relative paths to be resolved using + // the original approach as fallback if the [dyn.Value] location cannot be resolved. + _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + key := p[2].Key() + dir, err := v.Location().Directory() + if err != nil { + return dyn.InvalidValue, fmt.Errorf("unable to determine directory for %s: %w", p, err) + } + fallback[key] = dir + return v, nil + }) + if err != nil { + return nil, err + } + return fallback, nil +} diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index d41660728..58b5e0fb0 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -55,21 +55,14 @@ func rewritePatterns(base dyn.Pattern) []jobRewritePattern { } func (m *translatePaths) applyJobTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) { - var fallback = make(map[string]string) + fallback, err := gatherFallbackPaths(v, "jobs") + if err != nil { + return dyn.InvalidValue, err + } + + // Do not translate job task paths if using Git source var ignore []string - var err error - for key, job := range b.Config.Resources.Jobs { - dir, err := job.ConfigFileDirectory() - if err != nil { - return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err) - } - - // If we cannot resolve the relative path using the [dyn.Value] location itself, - // use the job's location as fallback. This is necessary for backwards compatibility. - fallback[key] = dir - - // Do not translate job task paths if using git source if job.GitSource != nil { ignore = append(ignore, key) } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index caec4198e..5b2a2c346 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -8,18 +8,9 @@ import ( ) func (m *translatePaths) applyPipelineTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) { - var fallback = make(map[string]string) - var err error - - for key, pipeline := range b.Config.Resources.Pipelines { - dir, err := pipeline.ConfigFileDirectory() - if err != nil { - return dyn.InvalidValue, fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err) - } - - // If we cannot resolve the relative path using the [dyn.Value] location itself, - // use the pipeline's location as fallback. This is necessary for backwards compatibility. - fallback[key] = dir + fallback, err := gatherFallbackPaths(v, "pipelines") + if err != nil { + return dyn.InvalidValue, err } // Base pattern to match all libraries in all pipelines. diff --git a/bundle/config/paths/paths.go b/bundle/config/paths/paths.go index 68c32a48c..95977ee37 100644 --- a/bundle/config/paths/paths.go +++ b/bundle/config/paths/paths.go @@ -1,9 +1,6 @@ package paths import ( - "fmt" - "path/filepath" - "github.com/databricks/cli/libs/dyn" ) @@ -23,10 +20,3 @@ func (p *Paths) ConfigureConfigFilePath() { } p.ConfigFilePath = p.DynamicValue.Location().File } - -func (p *Paths) ConfigFileDirectory() (string, error) { - if p.ConfigFilePath == "" { - return "", fmt.Errorf("config file path not configured") - } - return filepath.Dir(p.ConfigFilePath), nil -} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 775280438..70030c664 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -18,6 +18,7 @@ type Resources struct { Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"` ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"` + QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"` } type UniqueResourceIdTracker struct { @@ -28,6 +29,7 @@ type UniqueResourceIdTracker struct { type ConfigResource interface { Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) TerraformResourceName() string + Validate() error json.Marshaler json.Unmarshaler @@ -132,9 +134,66 @@ func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker, tracker.Type[k] = "registered_model" tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath } + for k := range r.QualityMonitors { + if _, ok := tracker.Type[k]; ok { + return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)", + k, + tracker.Type[k], + tracker.ConfigPath[k], + "quality_monitor", + r.QualityMonitors[k].ConfigFilePath, + ) + } + tracker.Type[k] = "quality_monitor" + tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath + } return tracker, nil } +type resource struct { + resource ConfigResource + resource_type string + key string +} + +func (r *Resources) allResources() []resource { + all := make([]resource, 0) + for k, e := range r.Jobs { + all = append(all, resource{resource_type: "job", resource: e, key: k}) + } + for k, e := range r.Pipelines { + all = append(all, resource{resource_type: "pipeline", resource: e, key: k}) + } + for k, e := range r.Models { + all = append(all, resource{resource_type: "model", resource: e, key: k}) + } + for k, e := range r.Experiments { + all = append(all, resource{resource_type: "experiment", resource: e, key: k}) + } + for k, e := range r.ModelServingEndpoints { + all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k}) + } + for k, e := range r.RegisteredModels { + all = append(all, resource{resource_type: "registered model", resource: e, key: k}) + } + for k, e := range r.QualityMonitors { + all = append(all, resource{resource_type: "quality monitor", resource: e, key: k}) + } + return all +} + +func (r *Resources) VerifyAllResourcesDefined() error { + all := r.allResources() + for _, e := range all { + err := e.resource.Validate() + if err != nil { + return fmt.Errorf("%s %s is not defined", e.resource_type, e.key) + } + } + + return nil +} + // ConfigureConfigFilePath sets the specified path for all resources contained in this instance. // This property is used to correctly resolve paths relative to the path // of the configuration file they were defined in. @@ -157,6 +216,9 @@ func (r *Resources) ConfigureConfigFilePath() { for _, e := range r.RegisteredModels { e.ConfigureConfigFilePath() } + for _, e := range r.QualityMonitors { + e.ConfigureConfigFilePath() + } } func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) { diff --git a/bundle/config/resources/job.go b/bundle/config/resources/job.go index 45e9662d9..dde5d5663 100644 --- a/bundle/config/resources/job.go +++ b/bundle/config/resources/job.go @@ -2,6 +2,7 @@ package resources import ( "context" + "fmt" "strconv" "github.com/databricks/cli/bundle/config/paths" @@ -47,3 +48,11 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri func (j *Job) TerraformResourceName() string { return "databricks_job" } + +func (j *Job) Validate() error { + if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil { + return fmt.Errorf("job is not defined") + } + + return nil +} diff --git a/bundle/config/resources/mlflow_experiment.go b/bundle/config/resources/mlflow_experiment.go index 0f53096a0..7854ee7e8 100644 --- a/bundle/config/resources/mlflow_experiment.go +++ b/bundle/config/resources/mlflow_experiment.go @@ -1,7 +1,12 @@ package resources import ( + "context" + "fmt" + "github.com/databricks/cli/bundle/config/paths" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -23,3 +28,26 @@ func (s *MlflowExperiment) UnmarshalJSON(b []byte) error { func (s MlflowExperiment) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } + +func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Experiments.GetExperiment(ctx, ml.GetExperimentRequest{ + ExperimentId: id, + }) + if err != nil { + log.Debugf(ctx, "experiment %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *MlflowExperiment) TerraformResourceName() string { + return "databricks_mlflow_experiment" +} + +func (s *MlflowExperiment) Validate() error { + if s == nil || !s.DynamicValue.IsValid() { + return fmt.Errorf("experiment is not defined") + } + + return nil +} diff --git a/bundle/config/resources/mlflow_model.go b/bundle/config/resources/mlflow_model.go index 59893aa47..40da9f87d 100644 --- a/bundle/config/resources/mlflow_model.go +++ b/bundle/config/resources/mlflow_model.go @@ -1,7 +1,12 @@ package resources import ( + "context" + "fmt" + "github.com/databricks/cli/bundle/config/paths" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -23,3 +28,26 @@ func (s *MlflowModel) UnmarshalJSON(b []byte) error { func (s MlflowModel) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } + +func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.ModelRegistry.GetModel(ctx, ml.GetModelRequest{ + Name: id, + }) + if err != nil { + log.Debugf(ctx, "model %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *MlflowModel) TerraformResourceName() string { + return "databricks_mlflow_model" +} + +func (s *MlflowModel) Validate() error { + if s == nil || !s.DynamicValue.IsValid() { + return fmt.Errorf("model is not defined") + } + + return nil +} diff --git a/bundle/config/resources/model_serving_endpoint.go b/bundle/config/resources/model_serving_endpoint.go index d1d57bafc..503cfbbb7 100644 --- a/bundle/config/resources/model_serving_endpoint.go +++ b/bundle/config/resources/model_serving_endpoint.go @@ -1,7 +1,12 @@ package resources import ( + "context" + "fmt" + "github.com/databricks/cli/bundle/config/paths" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -33,3 +38,26 @@ func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error { func (s ModelServingEndpoint) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } + +func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.ServingEndpoints.Get(ctx, serving.GetServingEndpointRequest{ + Name: id, + }) + if err != nil { + log.Debugf(ctx, "serving endpoint %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *ModelServingEndpoint) TerraformResourceName() string { + return "databricks_model_serving" +} + +func (s *ModelServingEndpoint) Validate() error { + if s == nil || !s.DynamicValue.IsValid() { + return fmt.Errorf("serving endpoint is not defined") + } + + return nil +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index 2f9ff8d0d..7e914b909 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -2,6 +2,7 @@ package resources import ( "context" + "fmt" "github.com/databricks/cli/bundle/config/paths" "github.com/databricks/cli/libs/log" @@ -42,3 +43,11 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id func (p *Pipeline) TerraformResourceName() string { return "databricks_pipeline" } + +func (p *Pipeline) Validate() error { + if p == nil || !p.DynamicValue.IsValid() { + return fmt.Errorf("pipeline is not defined") + } + + return nil +} diff --git a/bundle/config/resources/quality_monitor.go b/bundle/config/resources/quality_monitor.go new file mode 100644 index 000000000..0d13e58fa --- /dev/null +++ b/bundle/config/resources/quality_monitor.go @@ -0,0 +1,60 @@ +package resources + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle/config/paths" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type QualityMonitor struct { + // Represents the Input Arguments for Terraform and will get + // converted to a HCL representation for CRUD + *catalog.CreateMonitor + + // This represents the id which is the full name of the monitor + // (catalog_name.schema_name.table_name) that can be used + // as a reference in other resources. This value is returned by terraform. + ID string `json:"id,omitempty" bundle:"readonly"` + + // Path to config file where the resource is defined. All bundle resources + // include this for interpolation purposes. + paths.Paths + + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` +} + +func (s *QualityMonitor) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s QualityMonitor) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.QualityMonitors.Get(ctx, catalog.GetQualityMonitorRequest{ + TableName: id, + }) + if err != nil { + log.Debugf(ctx, "quality monitor %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *QualityMonitor) TerraformResourceName() string { + return "databricks_quality_monitor" +} + +func (s *QualityMonitor) Validate() error { + if s == nil || !s.DynamicValue.IsValid() { + return fmt.Errorf("quality monitor is not defined") + } + + return nil +} diff --git a/bundle/config/resources/registered_model.go b/bundle/config/resources/registered_model.go index 7b4b70d1a..fba643c69 100644 --- a/bundle/config/resources/registered_model.go +++ b/bundle/config/resources/registered_model.go @@ -1,7 +1,12 @@ package resources import ( + "context" + "fmt" + "github.com/databricks/cli/bundle/config/paths" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -34,3 +39,26 @@ func (s *RegisteredModel) UnmarshalJSON(b []byte) error { func (s RegisteredModel) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } + +func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.RegisteredModels.Get(ctx, catalog.GetRegisteredModelRequest{ + FullName: id, + }) + if err != nil { + log.Debugf(ctx, "registered model %s does not exist", id) + return false, err + } + return true, nil +} + +func (s *RegisteredModel) TerraformResourceName() string { + return "databricks_registered_model" +} + +func (s *RegisteredModel) Validate() error { + if s == nil || !s.DynamicValue.IsValid() { + return fmt.Errorf("registered model is not defined") + } + + return nil +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 9c4104e4d..7415029b1 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -1,6 +1,8 @@ package config import ( + "encoding/json" + "reflect" "testing" "github.com/databricks/cli/bundle/config/paths" @@ -125,3 +127,57 @@ func TestVerifySafeMergeForRegisteredModels(t *testing.T) { err := r.VerifySafeMerge(&other) assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)") } + +// This test ensures that all resources have a custom marshaller and unmarshaller. +// This is required because DABs resources map to Databricks APIs, and they do so +// by embedding the corresponding Go SDK structs. +// +// Go SDK structs often implement custom marshalling and unmarshalling methods (based on the API specifics). +// If the Go SDK struct implements custom marshalling and unmarshalling and we do not +// for the resources at the top level, marshalling and unmarshalling operations will panic. +// Thus we will be overly cautious and ensure that all resources need a custom marshaller and unmarshaller. +// +// Why do we not assert this using an interface to assert MarshalJSON and UnmarshalJSON +// are implemented at the top level? +// If a method is implemented for an embedded struct, the top level struct will +// also have that method and satisfy the interface. This is why we cannot assert +// that the methods are implemented at the top level using an interface. +// +// Why don't we use reflection to assert that the methods are implemented at the +// top level? +// Same problem as above, the golang reflection package does not seem to provide +// a way to directly assert that MarshalJSON and UnmarshalJSON are implemented +// at the top level. +func TestCustomMarshallerIsImplemented(t *testing.T) { + r := Resources{} + rt := reflect.TypeOf(r) + + for i := 0; i < rt.NumField(); i++ { + field := rt.Field(i) + + // Fields in Resources are expected be of the form map[string]*resourceStruct + assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name) + kt := field.Type.Key() + assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name) + vt := field.Type.Elem() + assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name) + + // Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller + // This is because resourceStruct embeds a Go SDK struct that implements + // a custom marshaller. + // Eg: resource.Job implements MarshalJSON + v := reflect.Zero(vt.Elem()).Interface() + assert.NotPanics(t, func() { + json.Marshal(v) + }, "Resource %s does not have a custom marshaller", field.Name) + + // Unmarshalling a *resourceStruct will panic if the resource does not have a custom unmarshaller + // This is because resourceStruct embeds a Go SDK struct that implements + // a custom unmarshaller. + // Eg: *resource.Job implements UnmarshalJSON + v = reflect.New(vt.Elem()).Interface() + assert.NotPanics(t, func() { + json.Unmarshal([]byte("{}"), v) + }, "Resource %s does not have a custom unmarshaller", field.Name) + } +} diff --git a/bundle/config/root.go b/bundle/config/root.go index 17f2747ef..88197c2b8 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -138,6 +138,14 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error { // Assign the normalized configuration tree. r.value = nv + // At the moment the check has to be done as part of updateWithDynamicValue + // because otherwise ConfigureConfigFilePath will fail with a panic. + // In the future, we should move this check to a separate mutator in initialise phase. + err = r.Resources.VerifyAllResourcesDefined() + if err != nil { + return err + } + // Assign config file paths after converting to typed configuration. r.ConfigureConfigFilePath() return nil @@ -408,15 +416,19 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) { // For each variable, normalize its contents if it is a single string. return dyn.Map(target, "variables", dyn.Foreach(func(_ dyn.Path, variable dyn.Value) (dyn.Value, error) { - if variable.Kind() != dyn.KindString { + switch variable.Kind() { + + case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt: + // Rewrite the variable to a map with a single key called "default". + // This conforms to the variable type. Normalization back to the typed + // configuration will convert this to a string if necessary. + return dyn.NewValue(map[string]dyn.Value{ + "default": variable, + }, variable.Location()), nil + + default: return variable, nil } - - // Rewrite the variable to a map with a single key called "default". - // This conforms to the variable type. - return dyn.NewValue(map[string]dyn.Value{ - "default": variable, - }, variable.Location()), nil })) })) } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 58acf6ae4..832efede9 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" "golang.org/x/sync/errgroup" ) @@ -50,7 +51,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di index := i p := pattern errs.Go(func() error { - fs, err := fileset.NewGlobSet(rb.RootPath(), []string{p}) + fs, err := fileset.NewGlobSet(vfs.MustNew(rb.RootPath()), []string{p}) if err != nil { return err } diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index d78ab2d74..8d6efdae3 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/sync" + "github.com/databricks/cli/libs/vfs" ) func GetSync(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.Sync, error) { @@ -28,7 +29,7 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp } opts := &sync.SyncOptions{ - LocalPath: rb.RootPath(), + LocalPath: vfs.MustNew(rb.RootPath()), RemotePath: rb.Config().Workspace.FilePath, Include: includes, Exclude: rb.Config().Sync.Exclude, diff --git a/bundle/deploy/metadata/annotate_jobs.go b/bundle/deploy/metadata/annotate_jobs.go index 2b03a59b7..f42d46931 100644 --- a/bundle/deploy/metadata/annotate_jobs.go +++ b/bundle/deploy/metadata/annotate_jobs.go @@ -2,7 +2,6 @@ package metadata import ( "context" - "path" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -27,7 +26,7 @@ func (m *annotateJobs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti job.JobSettings.Deployment = &jobs.JobDeployment{ Kind: jobs.JobDeploymentKindBundle, - MetadataFilePath: path.Join(b.Config.Workspace.StatePath, MetadataFileName), + MetadataFilePath: metadataFilePath(b), } job.JobSettings.EditMode = jobs.JobEditModeUiLocked job.JobSettings.Format = jobs.FormatMultiTask diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go new file mode 100644 index 000000000..990f48907 --- /dev/null +++ b/bundle/deploy/metadata/annotate_pipelines.go @@ -0,0 +1,34 @@ +package metadata + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/pipelines" +) + +type annotatePipelines struct{} + +func AnnotatePipelines() bundle.Mutator { + return &annotatePipelines{} +} + +func (m *annotatePipelines) Name() string { + return "metadata.AnnotatePipelines" +} + +func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, pipeline := range b.Config.Resources.Pipelines { + if pipeline.PipelineSpec == nil { + continue + } + + pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{ + Kind: pipelines.DeploymentKindBundle, + MetadataFilePath: metadataFilePath(b), + } + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_pipelines_test.go b/bundle/deploy/metadata/annotate_pipelines_test.go new file mode 100644 index 000000000..448a022d0 --- /dev/null +++ b/bundle/deploy/metadata/annotate_pipelines_test.go @@ -0,0 +1,72 @@ +package metadata + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotatePipelinesMutator(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + StatePath: "/a/b/c", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline-1": { + PipelineSpec: &pipelines.PipelineSpec{ + Name: "My Pipeline One", + }, + }, + "my-pipeline-2": { + PipelineSpec: &pipelines.PipelineSpec{ + Name: "My Pipeline Two", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, AnnotatePipelines()) + require.NoError(t, diags.Error()) + + assert.Equal(t, + &pipelines.PipelineDeployment{ + Kind: pipelines.DeploymentKindBundle, + MetadataFilePath: "/a/b/c/metadata.json", + }, + b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment) + + assert.Equal(t, + &pipelines.PipelineDeployment{ + Kind: pipelines.DeploymentKindBundle, + MetadataFilePath: "/a/b/c/metadata.json", + }, + b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment) +} + +func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Workspace: config.Workspace{ + StatePath: "/a/b/c", + }, + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline-1": {}, + }, + }, + }, + } + + diags := bundle.Apply(context.Background(), b, AnnotatePipelines()) + require.NoError(t, diags.Error()) +} diff --git a/bundle/deploy/metadata/upload.go b/bundle/deploy/metadata/upload.go index a040a0ae8..ee87816de 100644 --- a/bundle/deploy/metadata/upload.go +++ b/bundle/deploy/metadata/upload.go @@ -4,13 +4,18 @@ import ( "bytes" "context" "encoding/json" + "path" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" ) -const MetadataFileName = "metadata.json" +const metadataFileName = "metadata.json" + +func metadataFilePath(b *bundle.Bundle) string { + return path.Join(b.Config.Workspace.StatePath, metadataFileName) +} type upload struct{} @@ -33,5 +38,5 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.FromErr(err) } - return diag.FromErr(f.Write(ctx, MetadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists)) + return diag.FromErr(f.Write(ctx, metadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists)) } diff --git a/bundle/deploy/state.go b/bundle/deploy/state.go index ffcadc9d6..ccff64fe7 100644 --- a/bundle/deploy/state.go +++ b/bundle/deploy/state.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" ) const DeploymentStateFileName = "deployment.json" @@ -112,12 +113,18 @@ func FromSlice(files []fileset.File) (Filelist, error) { func (f Filelist) ToSlice(basePath string) []fileset.File { var files []fileset.File + root := vfs.MustNew(basePath) for _, file := range f { - absPath := filepath.Join(basePath, file.LocalPath) + entry := newEntry(filepath.Join(basePath, file.LocalPath)) + + // Snapshots created with versions <= v0.220.0 use platform-specific + // paths (i.e. with backslashes). Files returned by [libs/fileset] always + // contain forward slashes after this version. Normalize before using. + relative := filepath.ToSlash(file.LocalPath) if file.IsNotebook { - files = append(files, fileset.NewNotebookFile(newEntry(absPath), absPath, file.LocalPath)) + files = append(files, fileset.NewNotebookFile(root, entry, relative)) } else { - files = append(files, fileset.NewSourceFile(newEntry(absPath), absPath, file.LocalPath)) + files = append(files, fileset.NewSourceFile(root, entry, relative)) } } return files diff --git a/bundle/deploy/state_test.go b/bundle/deploy/state_test.go index 15bdc96b4..efa051ab6 100644 --- a/bundle/deploy/state_test.go +++ b/bundle/deploy/state_test.go @@ -3,17 +3,17 @@ package deploy import ( "bytes" "encoding/json" - "path/filepath" "testing" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) func TestFromSlice(t *testing.T) { tmpDir := t.TempDir() - fileset := fileset.New(tmpDir) + fileset := fileset.New(vfs.MustNew(tmpDir)) testutil.Touch(t, tmpDir, "test1.py") testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") @@ -32,7 +32,7 @@ func TestFromSlice(t *testing.T) { func TestToSlice(t *testing.T) { tmpDir := t.TempDir() - fileset := fileset.New(tmpDir) + fileset := fileset.New(vfs.MustNew(tmpDir)) testutil.Touch(t, tmpDir, "test1.py") testutil.Touch(t, tmpDir, "test2.py") testutil.Touch(t, tmpDir, "test3.py") @@ -48,18 +48,11 @@ func TestToSlice(t *testing.T) { require.Len(t, s, 3) for _, file := range s { - require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Name()) - require.Contains(t, []string{ - filepath.Join(tmpDir, "test1.py"), - filepath.Join(tmpDir, "test2.py"), - filepath.Join(tmpDir, "test3.py"), - }, file.Absolute) - require.False(t, file.IsDir()) - require.NotZero(t, file.Type()) - info, err := file.Info() - require.NoError(t, err) - require.NotNil(t, info) - require.Equal(t, file.Name(), info.Name()) + require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Relative) + + // If the mtime is not zero we know we produced a valid fs.DirEntry. + ts := file.Modified() + require.NotZero(t, ts) } } diff --git a/bundle/deploy/check_running_resources.go b/bundle/deploy/terraform/check_running_resources.go similarity index 57% rename from bundle/deploy/check_running_resources.go rename to bundle/deploy/terraform/check_running_resources.go index 7f7a9bcac..737f773e5 100644 --- a/bundle/deploy/check_running_resources.go +++ b/bundle/deploy/terraform/check_running_resources.go @@ -1,4 +1,4 @@ -package deploy +package terraform import ( "context" @@ -10,7 +10,6 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" "golang.org/x/sync/errgroup" ) @@ -36,26 +35,16 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia return nil } - tf := b.Terraform - if tf == nil { - return diag.Errorf("terraform not initialized") - } - - err := tf.Init(ctx, tfexec.Upgrade(true)) - if err != nil { - return diag.Errorf("terraform init: %v", err) - } - - state, err := b.Terraform.Show(ctx) - if err != nil { + state, err := ParseResourcesState(ctx, b) + if err != nil && state == nil { return diag.FromErr(err) } - err = checkAnyResourceRunning(ctx, b.WorkspaceClient(), state) + w := b.WorkspaceClient() + err = checkAnyResourceRunning(ctx, w, state) if err != nil { - return diag.Errorf("deployment aborted, err: %v", err) + return diag.FromErr(err) } - return nil } @@ -63,53 +52,49 @@ func CheckRunningResource() *checkRunningResources { return &checkRunningResources{} } -func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *tfjson.State) error { - if state.Values == nil || state.Values.RootModule == nil { +func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *resourcesState) error { + if state == nil { return nil } errs, errCtx := errgroup.WithContext(ctx) - for _, resource := range state.Values.RootModule.Resources { - // Limit to resources. + for _, resource := range state.Resources { if resource.Mode != tfjson.ManagedResourceMode { continue } + for _, instance := range resource.Instances { + id := instance.Attributes.ID + if id == "" { + continue + } - value, ok := resource.AttributeValues["id"] - if !ok { - continue - } - id, ok := value.(string) - if !ok { - continue - } - - switch resource.Type { - case "databricks_job": - errs.Go(func() error { - isRunning, err := IsJobRunning(errCtx, w, id) - // If there's an error retrieving the job, we assume it's not running - if err != nil { - return err - } - if isRunning { - return &ErrResourceIsRunning{resourceType: "job", resourceId: id} - } - return nil - }) - case "databricks_pipeline": - errs.Go(func() error { - isRunning, err := IsPipelineRunning(errCtx, w, id) - // If there's an error retrieving the pipeline, we assume it's not running - if err != nil { + switch resource.Type { + case "databricks_job": + errs.Go(func() error { + isRunning, err := IsJobRunning(errCtx, w, id) + // If there's an error retrieving the job, we assume it's not running + if err != nil { + return err + } + if isRunning { + return &ErrResourceIsRunning{resourceType: "job", resourceId: id} + } return nil - } - if isRunning { - return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id} - } - return nil - }) + }) + case "databricks_pipeline": + errs.Go(func() error { + isRunning, err := IsPipelineRunning(errCtx, w, id) + // If there's an error retrieving the pipeline, we assume it's not running + if err != nil { + return nil + } + if isRunning { + return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id} + } + return nil + }) + } } } diff --git a/bundle/deploy/check_running_resources_test.go b/bundle/deploy/terraform/check_running_resources_test.go similarity index 68% rename from bundle/deploy/check_running_resources_test.go rename to bundle/deploy/terraform/check_running_resources_test.go index 7dc1fb865..a1bbbd37b 100644 --- a/bundle/deploy/check_running_resources_test.go +++ b/bundle/deploy/terraform/check_running_resources_test.go @@ -1,4 +1,4 @@ -package deploy +package terraform import ( "context" @@ -8,31 +8,26 @@ import ( "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" - tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) func TestIsAnyResourceRunningWithEmptyState(t *testing.T) { mock := mocks.NewMockWorkspaceClient(t) - state := &tfjson.State{} - err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, state) + err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &resourcesState{}) require.NoError(t, err) } func TestIsAnyResourceRunningWithJob(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) - state := &tfjson.State{ - Values: &tfjson.StateValues{ - RootModule: &tfjson.StateModule{ - Resources: []*tfjson.StateResource{ - { - Type: "databricks_job", - AttributeValues: map[string]interface{}{ - "id": "123", - }, - Mode: tfjson.ManagedResourceMode, - }, + resources := &resourcesState{ + Resources: []stateResource{ + { + Type: "databricks_job", + Mode: "managed", + Name: "job1", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "123"}}, }, }, }, @@ -46,7 +41,7 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) { {RunId: 1234}, }, nil).Once() - err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state) + err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources) require.ErrorContains(t, err, "job 123 is running") jobsApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{ @@ -54,23 +49,20 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) { ActiveOnly: true, }).Return([]jobs.BaseRun{}, nil).Once() - err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state) + err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources) require.NoError(t, err) } func TestIsAnyResourceRunningWithPipeline(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) - state := &tfjson.State{ - Values: &tfjson.StateValues{ - RootModule: &tfjson.StateModule{ - Resources: []*tfjson.StateResource{ - { - Type: "databricks_pipeline", - AttributeValues: map[string]interface{}{ - "id": "123", - }, - Mode: tfjson.ManagedResourceMode, - }, + resources := &resourcesState{ + Resources: []stateResource{ + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "pipeline1", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "123"}}, }, }, }, @@ -84,7 +76,7 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) { State: pipelines.PipelineStateRunning, }, nil).Once() - err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state) + err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources) require.ErrorContains(t, err, "pipeline 123 is running") pipelineApi.EXPECT().Get(mock.Anything, pipelines.GetPipelineRequest{ @@ -93,23 +85,20 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) { PipelineId: "123", State: pipelines.PipelineStateIdle, }, nil).Once() - err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state) + err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources) require.NoError(t, err) } func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) - state := &tfjson.State{ - Values: &tfjson.StateValues{ - RootModule: &tfjson.StateModule{ - Resources: []*tfjson.StateResource{ - { - Type: "databricks_pipeline", - AttributeValues: map[string]interface{}{ - "id": "123", - }, - Mode: tfjson.ManagedResourceMode, - }, + resources := &resourcesState{ + Resources: []stateResource{ + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "pipeline1", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "123"}}, }, }, }, @@ -120,6 +109,6 @@ func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) { PipelineId: "123", }).Return(nil, errors.New("API failure")).Once() - err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state) + err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources) require.NoError(t, err) } diff --git a/bundle/deploy/terraform/convert.go b/bundle/deploy/terraform/convert.go index 0ae6751d0..a6ec04d9a 100644 --- a/bundle/deploy/terraform/convert.go +++ b/bundle/deploy/terraform/convert.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "reflect" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" @@ -19,15 +18,6 @@ func conv(from any, to any) { json.Unmarshal(buf, &to) } -func convRemoteToLocal(remote any, local any) resources.ModifiedStatus { - var modifiedStatus resources.ModifiedStatus - if reflect.ValueOf(local).Elem().IsNil() { - modifiedStatus = resources.ModifiedStatusDeleted - } - conv(remote, local) - return modifiedStatus -} - func convPermissions(acl []resources.Permission) *schema.ResourcePermissions { if len(acl) == 0 { return nil @@ -232,6 +222,13 @@ func BundleToTerraform(config *config.Root) *schema.Root { } } + for k, src := range config.Resources.QualityMonitors { + noResources = false + var dst schema.ResourceQualityMonitor + conv(src, &dst) + tfroot.Resource.QualityMonitor[k] = &dst + } + // We explicitly set "resource" to nil to omit it from a JSON encoding. // This is required because the terraform CLI requires >= 1 resources defined // if the "resource" property is used in a .tf.json file. @@ -248,7 +245,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema tfroot.Provider = schema.NewProviders() // Convert each resource in the bundle to the equivalent Terraform representation. - resources, err := dyn.Get(root, "resources") + dynResources, err := dyn.Get(root, "resources") if err != nil { // If the resources key is missing, return an empty root. if dyn.IsNoSuchKeyError(err) { @@ -260,11 +257,20 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema tfroot.Resource = schema.NewResources() numResources := 0 - _, err = dyn.Walk(resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + _, err = dyn.Walk(dynResources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { if len(p) < 2 { return v, nil } + // Skip resources that have been deleted locally. + modifiedStatus, err := dyn.Get(v, "modified_status") + if err == nil { + modifiedStatusStr, ok := modifiedStatus.AsString() + if ok && modifiedStatusStr == resources.ModifiedStatusDeleted { + return v, dyn.ErrSkip + } + } + typ := p[0].Key() key := p[1].Key() @@ -275,7 +281,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema } // Convert resource to Terraform representation. - err := c.Convert(ctx, key, v, tfroot.Resource) + err = c.Convert(ctx, key, v, tfroot.Resource) if err != nil { return dyn.InvalidValue, err } @@ -299,76 +305,83 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema return tfroot, nil } -func TerraformToBundle(state *tfjson.State, config *config.Root) error { - if state.Values != nil && state.Values.RootModule != nil { - for _, resource := range state.Values.RootModule.Resources { - // Limit to resources. - if resource.Mode != tfjson.ManagedResourceMode { - continue - } - +func TerraformToBundle(state *resourcesState, config *config.Root) error { + for _, resource := range state.Resources { + if resource.Mode != tfjson.ManagedResourceMode { + continue + } + for _, instance := range resource.Instances { switch resource.Type { case "databricks_job": - var tmp schema.ResourceJob - conv(resource.AttributeValues, &tmp) if config.Resources.Jobs == nil { config.Resources.Jobs = make(map[string]*resources.Job) } cur := config.Resources.Jobs[resource.Name] - // TODO: make sure we can unmarshall tf state properly and don't swallow errors - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.Job{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.Jobs[resource.Name] = cur case "databricks_pipeline": - var tmp schema.ResourcePipeline - conv(resource.AttributeValues, &tmp) if config.Resources.Pipelines == nil { config.Resources.Pipelines = make(map[string]*resources.Pipeline) } cur := config.Resources.Pipelines[resource.Name] - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.Pipeline{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.Pipelines[resource.Name] = cur case "databricks_mlflow_model": - var tmp schema.ResourceMlflowModel - conv(resource.AttributeValues, &tmp) if config.Resources.Models == nil { config.Resources.Models = make(map[string]*resources.MlflowModel) } cur := config.Resources.Models[resource.Name] - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.MlflowModel{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.Models[resource.Name] = cur case "databricks_mlflow_experiment": - var tmp schema.ResourceMlflowExperiment - conv(resource.AttributeValues, &tmp) if config.Resources.Experiments == nil { config.Resources.Experiments = make(map[string]*resources.MlflowExperiment) } cur := config.Resources.Experiments[resource.Name] - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.MlflowExperiment{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.Experiments[resource.Name] = cur case "databricks_model_serving": - var tmp schema.ResourceModelServing - conv(resource.AttributeValues, &tmp) if config.Resources.ModelServingEndpoints == nil { config.Resources.ModelServingEndpoints = make(map[string]*resources.ModelServingEndpoint) } cur := config.Resources.ModelServingEndpoints[resource.Name] - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.ModelServingEndpoint{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.ModelServingEndpoints[resource.Name] = cur case "databricks_registered_model": - var tmp schema.ResourceRegisteredModel - conv(resource.AttributeValues, &tmp) if config.Resources.RegisteredModels == nil { config.Resources.RegisteredModels = make(map[string]*resources.RegisteredModel) } cur := config.Resources.RegisteredModels[resource.Name] - modifiedStatus := convRemoteToLocal(tmp, &cur) - cur.ModifiedStatus = modifiedStatus + if cur == nil { + cur = &resources.RegisteredModel{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID config.Resources.RegisteredModels[resource.Name] = cur + case "databricks_quality_monitor": + if config.Resources.QualityMonitors == nil { + config.Resources.QualityMonitors = make(map[string]*resources.QualityMonitor) + } + cur := config.Resources.QualityMonitors[resource.Name] + if cur == nil { + cur = &resources.QualityMonitor{ModifiedStatus: resources.ModifiedStatusDeleted} + } + cur.ID = instance.Attributes.ID + config.Resources.QualityMonitors[resource.Name] = cur case "databricks_permissions": case "databricks_grants": // Ignore; no need to pull these back into the configuration. @@ -408,6 +421,11 @@ func TerraformToBundle(state *tfjson.State, config *config.Root) error { src.ModifiedStatus = resources.ModifiedStatusCreated } } + for _, src := range config.Resources.QualityMonitors { + if src.ModifiedStatus == "" && src.ID == "" { + src.ModifiedStatus = resources.ModifiedStatusCreated + } + } return nil } diff --git a/bundle/deploy/terraform/convert_test.go b/bundle/deploy/terraform/convert_test.go index 986599a79..e1f73be28 100644 --- a/bundle/deploy/terraform/convert_test.go +++ b/bundle/deploy/terraform/convert_test.go @@ -17,7 +17,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" - tfjson "github.com/hashicorp/terraform-json" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -548,50 +547,94 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) { bundleToTerraformEquivalenceTest(t, &config) } +func TestBundleToTerraformDeletedResources(t *testing.T) { + var job1 = resources.Job{ + JobSettings: &jobs.JobSettings{}, + } + var job2 = resources.Job{ + ModifiedStatus: resources.ModifiedStatusDeleted, + JobSettings: &jobs.JobSettings{}, + } + var config = config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job1": &job1, + "my_job2": &job2, + }, + }, + } + + vin, err := convert.FromTyped(config, dyn.NilValue) + require.NoError(t, err) + out, err := BundleToTerraformWithDynValue(context.Background(), vin) + require.NoError(t, err) + + _, ok := out.Resource.Job["my_job1"] + assert.True(t, ok) + _, ok = out.Resource.Job["my_job2"] + assert.False(t, ok) +} + func TestTerraformToBundleEmptyLocalResources(t *testing.T) { var config = config.Root{ Resources: config.Resources{}, } - var tfState = tfjson.State{ - Values: &tfjson.StateValues{ - RootModule: &tfjson.StateModule{ - Resources: []*tfjson.StateResource{ - { - Type: "databricks_job", - Mode: "managed", - Name: "test_job", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_pipeline", - Mode: "managed", - Name: "test_pipeline", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_mlflow_model", - Mode: "managed", - Name: "test_mlflow_model", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_mlflow_experiment", - Mode: "managed", - Name: "test_mlflow_experiment", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_model_serving", - Mode: "managed", - Name: "test_model_serving", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_registered_model", - Mode: "managed", - Name: "test_registered_model", - AttributeValues: map[string]interface{}{"id": "1"}, - }, + var tfState = resourcesState{ + Resources: []stateResource{ + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_quality_monitor", + Mode: "managed", + Name: "test_monitor", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, }, }, }, @@ -617,6 +660,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.RegisteredModels["test_registered_model"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -665,10 +711,17 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + QualityMonitors: map[string]*resources.QualityMonitor{ + "test_monitor": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "test_monitor", + }, + }, + }, }, } - var tfState = tfjson.State{ - Values: nil, + var tfState = resourcesState{ + Resources: nil, } err := TerraformToBundle(&tfState, &config) assert.NoError(t, err) @@ -691,6 +744,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus) + assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -769,84 +825,132 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { }, }, }, + QualityMonitors: map[string]*resources.QualityMonitor{ + "test_monitor": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "test_monitor", + }, + }, + "test_monitor_new": { + CreateMonitor: &catalog.CreateMonitor{ + TableName: "test_monitor_new", + }, + }, + }, }, } - var tfState = tfjson.State{ - Values: &tfjson.StateValues{ - RootModule: &tfjson.StateModule{ - Resources: []*tfjson.StateResource{ - { - Type: "databricks_job", - Mode: "managed", - Name: "test_job", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_job", - Mode: "managed", - Name: "test_job_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, - { - Type: "databricks_pipeline", - Mode: "managed", - Name: "test_pipeline", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_pipeline", - Mode: "managed", - Name: "test_pipeline_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, - { - Type: "databricks_mlflow_model", - Mode: "managed", - Name: "test_mlflow_model", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_mlflow_model", - Mode: "managed", - Name: "test_mlflow_model_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, - { - Type: "databricks_mlflow_experiment", - Mode: "managed", - Name: "test_mlflow_experiment", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_mlflow_experiment", - Mode: "managed", - Name: "test_mlflow_experiment_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, - { - Type: "databricks_model_serving", - Mode: "managed", - Name: "test_model_serving", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_model_serving", - Mode: "managed", - Name: "test_model_serving_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, - { - Type: "databricks_registered_model", - Mode: "managed", - Name: "test_registered_model", - AttributeValues: map[string]interface{}{"id": "1"}, - }, - { - Type: "databricks_registered_model", - Mode: "managed", - Name: "test_registered_model_old", - AttributeValues: map[string]interface{}{"id": "2"}, - }, + var tfState = resourcesState{ + Resources: []stateResource{ + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_job", + Mode: "managed", + Name: "test_job_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_pipeline", + Mode: "managed", + Name: "test_pipeline_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_mlflow_model", + Mode: "managed", + Name: "test_mlflow_model_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_mlflow_experiment", + Mode: "managed", + Name: "test_mlflow_experiment_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_model_serving", + Mode: "managed", + Name: "test_model_serving_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "1"}}, + }, + }, + { + Type: "databricks_registered_model", + Mode: "managed", + Name: "test_registered_model_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "2"}}, + }, + }, + { + Type: "databricks_quality_monitor", + Mode: "managed", + Name: "test_monitor", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "test_monitor"}}, + }, + }, + { + Type: "databricks_quality_monitor", + Mode: "managed", + Name: "test_monitor_old", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "test_monitor_old"}}, }, }, }, @@ -896,6 +1000,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServingEndpoints["test_model_serving_new"].ModifiedStatus) + assert.Equal(t, "test_monitor", config.Resources.QualityMonitors["test_monitor"].ID) + assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ModifiedStatus) + assert.Equal(t, "test_monitor_old", config.Resources.QualityMonitors["test_monitor_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus) AssertFullResourceCoverage(t, &config) } diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index 358279a7a..608f1c795 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -54,6 +54,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D path = dyn.NewPath(dyn.Key("databricks_model_serving")).Append(path[2:]...) case dyn.Key("registered_models"): path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...) + case dyn.Key("quality_monitors"): + path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...) default: // Trigger "key not found" for unknown resource types. return dyn.GetByPath(root, path) diff --git a/bundle/deploy/terraform/load.go b/bundle/deploy/terraform/load.go index fa0cd5b4f..3fb76855e 100644 --- a/bundle/deploy/terraform/load.go +++ b/bundle/deploy/terraform/load.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/hashicorp/terraform-exec/tfexec" - tfjson "github.com/hashicorp/terraform-json" ) type loadMode int @@ -34,7 +33,7 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.Errorf("terraform init: %v", err) } - state, err := b.Terraform.Show(ctx) + state, err := ParseResourcesState(ctx, b) if err != nil { return diag.FromErr(err) } @@ -53,16 +52,13 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return nil } -func (l *load) validateState(state *tfjson.State) error { - if state.Values == nil { - if slices.Contains(l.modes, ErrorOnEmptyState) { - return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?") - } - return nil +func (l *load) validateState(state *resourcesState) error { + if state.Version != SupportedStateVersion { + return fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version) } - if state.Values.RootModule == nil { - return fmt.Errorf("malformed terraform state: RootModule not set") + if len(state.Resources) == 0 && slices.Contains(l.modes, ErrorOnEmptyState) { + return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?") } return nil diff --git a/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go b/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go new file mode 100644 index 000000000..341df7c22 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_quality_monitor.go @@ -0,0 +1,37 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +func convertQualityMonitorResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(schema.ResourceQualityMonitor{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "monitor normalization diagnostic: %s", diag.Summary) + } + return vout, nil +} + +type qualityMonitorConverter struct{} + +func (qualityMonitorConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + vout, err := convertQualityMonitorResource(ctx, vin) + if err != nil { + return err + } + + // Add the converted resource to the output. + out.QualityMonitor[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("quality_monitors", qualityMonitorConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go b/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go new file mode 100644 index 000000000..50bfce7a0 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go @@ -0,0 +1,46 @@ +package tfdyn + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertQualityMonitor(t *testing.T) { + var src = resources.QualityMonitor{ + CreateMonitor: &catalog.CreateMonitor{ + TableName: "test_table_name", + AssetsDir: "assets_dir", + OutputSchemaName: "output_schema_name", + InferenceLog: &catalog.MonitorInferenceLog{ + ModelIdCol: "model_id", + PredictionCol: "test_prediction_col", + ProblemType: "PROBLEM_TYPE_CLASSIFICATION", + }, + }, + } + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + ctx := context.Background() + out := schema.NewResources() + err = qualityMonitorConverter{}.Convert(ctx, "my_monitor", vin, out) + + require.NoError(t, err) + assert.Equal(t, map[string]any{ + "assets_dir": "assets_dir", + "output_schema_name": "output_schema_name", + "table_name": "test_table_name", + "inference_log": map[string]any{ + "model_id_col": "model_id", + "prediction_col": "test_prediction_col", + "problem_type": "PROBLEM_TYPE_CLASSIFICATION", + }, + }, out.QualityMonitor["my_monitor"]) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index a5978b397..1a8a83ac7 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -1,14 +1,46 @@ package terraform import ( + "context" "encoding/json" + "errors" "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + tfjson "github.com/hashicorp/terraform-json" ) -type state struct { +// Partial representation of the Terraform state file format. +// We are only interested global version and serial numbers, +// plus resource types, names, modes, and ids. +type resourcesState struct { + Version int `json:"version"` + Resources []stateResource `json:"resources"` +} + +const SupportedStateVersion = 4 + +type serialState struct { Serial int `json:"serial"` } +type stateResource struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []stateResourceInstance `json:"instances"` +} + +type stateResourceInstance struct { + Attributes stateInstanceAttributes `json:"attributes"` +} + +type stateInstanceAttributes struct { + ID string `json:"id"` +} + func IsLocalStateStale(local io.Reader, remote io.Reader) bool { localState, err := loadState(local) if err != nil { @@ -23,12 +55,12 @@ func IsLocalStateStale(local io.Reader, remote io.Reader) bool { return localState.Serial < remoteState.Serial } -func loadState(input io.Reader) (*state, error) { +func loadState(input io.Reader) (*serialState, error) { content, err := io.ReadAll(input) if err != nil { return nil, err } - var s state + var s serialState err = json.Unmarshal(content, &s) if err != nil { return nil, err @@ -36,3 +68,20 @@ func loadState(input io.Reader) (*state, error) { return &s, nil } + +func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) { + cacheDir, err := Dir(ctx, b) + if err != nil { + return nil, err + } + rawState, err := os.ReadFile(filepath.Join(cacheDir, TerraformStateFileName)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return &resourcesState{Version: SupportedStateVersion}, nil + } + return nil, err + } + var state resourcesState + err = json.Unmarshal(rawState, &state) + return &state, err +} diff --git a/bundle/deploy/terraform/util_test.go b/bundle/deploy/terraform/util_test.go index 4f2cf2918..8949ebca8 100644 --- a/bundle/deploy/terraform/util_test.go +++ b/bundle/deploy/terraform/util_test.go @@ -1,11 +1,16 @@ package terraform import ( + "context" "fmt" + "os" + "path/filepath" "strings" "testing" "testing/iotest" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/stretchr/testify/assert" ) @@ -38,3 +43,97 @@ func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) { remote := iotest.ErrReader(fmt.Errorf("Random error")) assert.False(t, IsLocalStateStale(local, remote)) } + +func TestParseResourcesStateWithNoFile(t *testing.T) { + b := &bundle.Bundle{ + RootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + Terraform: &config.Terraform{ + ExecPath: "terraform", + }, + }, + }, + } + state, err := ParseResourcesState(context.Background(), b) + assert.NoError(t, err) + assert.Equal(t, &resourcesState{Version: SupportedStateVersion}, state) +} + +func TestParseResourcesStateWithExistingStateFile(t *testing.T) { + ctx := context.Background() + b := &bundle.Bundle{ + RootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{ + Target: "whatever", + Terraform: &config.Terraform{ + ExecPath: "terraform", + }, + }, + }, + } + cacheDir, err := Dir(ctx, b) + assert.NoError(t, err) + data := []byte(`{ + "version": 4, + "unknown_field": "hello", + "resources": [ + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "test_pipeline", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "catalog": null, + "channel": "CURRENT", + "cluster": [], + "random_field": "random_value", + "configuration": { + "bundle.sourcePath": "/Workspace//Users/user/.bundle/test/dev/files/src" + }, + "continuous": false, + "development": true, + "edition": "ADVANCED", + "filters": [], + "id": "123", + "library": [], + "name": "test_pipeline", + "notification": [], + "photon": false, + "serverless": false, + "storage": "dbfs:/123456", + "target": "test_dev", + "timeouts": null, + "url": "https://test.com" + }, + "sensitive_attributes": [] + } + ] + } + ] + }`) + err = os.WriteFile(filepath.Join(cacheDir, TerraformStateFileName), data, os.ModePerm) + assert.NoError(t, err) + state, err := ParseResourcesState(ctx, b) + assert.NoError(t, err) + expected := &resourcesState{ + Version: 4, + Resources: []stateResource{ + { + Mode: "managed", + Type: "databricks_pipeline", + Name: "test_pipeline", + Instances: []stateResourceInstance{ + {Attributes: stateInstanceAttributes{ID: "123"}}, + }, + }, + }, + } + assert.Equal(t, expected, state) +} diff --git a/bundle/internal/tf/codegen/schema/version.go b/bundle/internal/tf/codegen/schema/version.go index 4fb4bf2c5..f55b6c4f0 100644 --- a/bundle/internal/tf/codegen/schema/version.go +++ b/bundle/internal/tf/codegen/schema/version.go @@ -1,3 +1,3 @@ package schema -const ProviderVersion = "1.40.0" +const ProviderVersion = "1.46.0" diff --git a/bundle/internal/tf/schema/data_source_catalog.go b/bundle/internal/tf/schema/data_source_catalog.go new file mode 100644 index 000000000..6f9237cfa --- /dev/null +++ b/bundle/internal/tf/schema/data_source_catalog.go @@ -0,0 +1,46 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag struct { + InheritedFromName string `json:"inherited_from_name,omitempty"` + InheritedFromType string `json:"inherited_from_type,omitempty"` + Value string `json:"value"` +} + +type DataSourceCatalogCatalogInfoProvisioningInfo struct { + State string `json:"state,omitempty"` +} + +type DataSourceCatalogCatalogInfo struct { + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogType string `json:"catalog_type,omitempty"` + Comment string `json:"comment,omitempty"` + ConnectionName string `json:"connection_name,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"` + FullName string `json:"full_name,omitempty"` + IsolationMode string `json:"isolation_mode,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Options map[string]string `json:"options,omitempty"` + Owner string `json:"owner,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + ProviderName string `json:"provider_name,omitempty"` + SecurableKind string `json:"securable_kind,omitempty"` + SecurableType string `json:"securable_type,omitempty"` + ShareName string `json:"share_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + EffectivePredictiveOptimizationFlag *DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"` + ProvisioningInfo *DataSourceCatalogCatalogInfoProvisioningInfo `json:"provisioning_info,omitempty"` +} + +type DataSourceCatalog struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + CatalogInfo *DataSourceCatalogCatalogInfo `json:"catalog_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_job.go b/bundle/internal/tf/schema/data_source_job.go index dbd29f4ba..d517bbe0f 100644 --- a/bundle/internal/tf/schema/data_source_job.go +++ b/bundle/internal/tf/schema/data_source_job.go @@ -55,9 +55,9 @@ type DataSourceJobJobSettingsSettingsGitSource struct { } type DataSourceJobJobSettingsSettingsHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsHealth struct { @@ -222,7 +222,7 @@ type DataSourceJobJobSettingsSettingsJobClusterNewCluster struct { } type DataSourceJobJobSettingsSettingsJobCluster struct { - JobClusterKey string `json:"job_cluster_key,omitempty"` + JobClusterKey string `json:"job_cluster_key"` NewCluster *DataSourceJobJobSettingsSettingsJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -243,12 +243,13 @@ type DataSourceJobJobSettingsSettingsLibraryPypi struct { } type DataSourceJobJobSettingsSettingsLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *DataSourceJobJobSettingsSettingsLibraryCran `json:"cran,omitempty"` - Maven *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"` - Pypi *DataSourceJobJobSettingsSettingsLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceJobJobSettingsSettingsLibraryCran `json:"cran,omitempty"` + Maven *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceJobJobSettingsSettingsLibraryPypi `json:"pypi,omitempty"` } type DataSourceJobJobSettingsSettingsNewClusterAutoscale struct { @@ -532,9 +533,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskEmailNotifications struc } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealth struct { @@ -558,12 +559,13 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi struct { } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"` - Maven *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"` - Pypi *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"` + Maven *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"` } type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskNewClusterAutoscale struct { @@ -803,7 +805,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -842,7 +844,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` @@ -870,9 +872,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTask struct { } type DataSourceJobJobSettingsSettingsTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type DataSourceJobJobSettingsSettingsTaskHealth struct { @@ -896,12 +898,13 @@ type DataSourceJobJobSettingsSettingsTaskLibraryPypi struct { } type DataSourceJobJobSettingsSettingsTaskLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *DataSourceJobJobSettingsSettingsTaskLibraryCran `json:"cran,omitempty"` - Maven *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"` - Pypi *DataSourceJobJobSettingsSettingsTaskLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *DataSourceJobJobSettingsSettingsTaskLibraryCran `json:"cran,omitempty"` + Maven *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"` + Pypi *DataSourceJobJobSettingsSettingsTaskLibraryPypi `json:"pypi,omitempty"` } type DataSourceJobJobSettingsSettingsTaskNewClusterAutoscale struct { @@ -1141,7 +1144,7 @@ type DataSourceJobJobSettingsSettingsTaskSqlTaskQuery struct { type DataSourceJobJobSettingsSettingsTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *DataSourceJobJobSettingsSettingsTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *DataSourceJobJobSettingsSettingsTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *DataSourceJobJobSettingsSettingsTaskSqlTaskFile `json:"file,omitempty"` @@ -1180,7 +1183,7 @@ type DataSourceJobJobSettingsSettingsTask struct { MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` + TaskKey string `json:"task_key"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` ConditionTask *DataSourceJobJobSettingsSettingsTaskConditionTask `json:"condition_task,omitempty"` DbtTask *DataSourceJobJobSettingsSettingsTaskDbtTask `json:"dbt_task,omitempty"` diff --git a/bundle/internal/tf/schema/data_source_mlflow_experiment.go b/bundle/internal/tf/schema/data_source_mlflow_experiment.go new file mode 100644 index 000000000..979130c5f --- /dev/null +++ b/bundle/internal/tf/schema/data_source_mlflow_experiment.go @@ -0,0 +1,19 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceMlflowExperimentTags struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +type DataSourceMlflowExperiment struct { + ArtifactLocation string `json:"artifact_location,omitempty"` + CreationTime int `json:"creation_time,omitempty"` + ExperimentId string `json:"experiment_id,omitempty"` + Id string `json:"id,omitempty"` + LastUpdateTime int `json:"last_update_time,omitempty"` + LifecycleStage string `json:"lifecycle_stage,omitempty"` + Name string `json:"name,omitempty"` + Tags []DataSourceMlflowExperimentTags `json:"tags,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_source_table.go b/bundle/internal/tf/schema/data_source_table.go new file mode 100644 index 000000000..f59959696 --- /dev/null +++ b/bundle/internal/tf/schema/data_source_table.go @@ -0,0 +1,127 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type DataSourceTableTableInfoColumnsMask struct { + FunctionName string `json:"function_name,omitempty"` + UsingColumnNames []string `json:"using_column_names,omitempty"` +} + +type DataSourceTableTableInfoColumns struct { + Comment string `json:"comment,omitempty"` + Name string `json:"name,omitempty"` + Nullable bool `json:"nullable,omitempty"` + PartitionIndex int `json:"partition_index,omitempty"` + Position int `json:"position,omitempty"` + TypeIntervalType string `json:"type_interval_type,omitempty"` + TypeJson string `json:"type_json,omitempty"` + TypeName string `json:"type_name,omitempty"` + TypePrecision int `json:"type_precision,omitempty"` + TypeScale int `json:"type_scale,omitempty"` + TypeText string `json:"type_text,omitempty"` + Mask *DataSourceTableTableInfoColumnsMask `json:"mask,omitempty"` +} + +type DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs struct { + DeltaRuntimeProperties map[string]string `json:"delta_runtime_properties"` +} + +type DataSourceTableTableInfoEffectivePredictiveOptimizationFlag struct { + InheritedFromName string `json:"inherited_from_name,omitempty"` + InheritedFromType string `json:"inherited_from_type,omitempty"` + Value string `json:"value"` +} + +type DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails struct { + Algorithm string `json:"algorithm,omitempty"` + AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"` +} + +type DataSourceTableTableInfoEncryptionDetails struct { + SseEncryptionDetails *DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"` +} + +type DataSourceTableTableInfoRowFilter struct { + FunctionName string `json:"function_name"` + InputColumnNames []string `json:"input_column_names"` +} + +type DataSourceTableTableInfoTableConstraintsForeignKeyConstraint struct { + ChildColumns []string `json:"child_columns"` + Name string `json:"name"` + ParentColumns []string `json:"parent_columns"` + ParentTable string `json:"parent_table"` +} + +type DataSourceTableTableInfoTableConstraintsNamedTableConstraint struct { + Name string `json:"name"` +} + +type DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint struct { + ChildColumns []string `json:"child_columns"` + Name string `json:"name"` +} + +type DataSourceTableTableInfoTableConstraints struct { + ForeignKeyConstraint *DataSourceTableTableInfoTableConstraintsForeignKeyConstraint `json:"foreign_key_constraint,omitempty"` + NamedTableConstraint *DataSourceTableTableInfoTableConstraintsNamedTableConstraint `json:"named_table_constraint,omitempty"` + PrimaryKeyConstraint *DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint `json:"primary_key_constraint,omitempty"` +} + +type DataSourceTableTableInfoViewDependenciesDependenciesFunction struct { + FunctionFullName string `json:"function_full_name"` +} + +type DataSourceTableTableInfoViewDependenciesDependenciesTable struct { + TableFullName string `json:"table_full_name"` +} + +type DataSourceTableTableInfoViewDependenciesDependencies struct { + Function *DataSourceTableTableInfoViewDependenciesDependenciesFunction `json:"function,omitempty"` + Table *DataSourceTableTableInfoViewDependenciesDependenciesTable `json:"table,omitempty"` +} + +type DataSourceTableTableInfoViewDependencies struct { + Dependencies []DataSourceTableTableInfoViewDependenciesDependencies `json:"dependencies,omitempty"` +} + +type DataSourceTableTableInfo struct { + AccessPoint string `json:"access_point,omitempty"` + BrowseOnly bool `json:"browse_only,omitempty"` + CatalogName string `json:"catalog_name,omitempty"` + Comment string `json:"comment,omitempty"` + CreatedAt int `json:"created_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + DataAccessConfigurationId string `json:"data_access_configuration_id,omitempty"` + DataSourceFormat string `json:"data_source_format,omitempty"` + DeletedAt int `json:"deleted_at,omitempty"` + EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"` + FullName string `json:"full_name,omitempty"` + MetastoreId string `json:"metastore_id,omitempty"` + Name string `json:"name,omitempty"` + Owner string `json:"owner,omitempty"` + PipelineId string `json:"pipeline_id,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + SchemaName string `json:"schema_name,omitempty"` + SqlPath string `json:"sql_path,omitempty"` + StorageCredentialName string `json:"storage_credential_name,omitempty"` + StorageLocation string `json:"storage_location,omitempty"` + TableId string `json:"table_id,omitempty"` + TableType string `json:"table_type,omitempty"` + UpdatedAt int `json:"updated_at,omitempty"` + UpdatedBy string `json:"updated_by,omitempty"` + ViewDefinition string `json:"view_definition,omitempty"` + Columns []DataSourceTableTableInfoColumns `json:"columns,omitempty"` + DeltaRuntimePropertiesKvpairs *DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs `json:"delta_runtime_properties_kvpairs,omitempty"` + EffectivePredictiveOptimizationFlag *DataSourceTableTableInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"` + EncryptionDetails *DataSourceTableTableInfoEncryptionDetails `json:"encryption_details,omitempty"` + RowFilter *DataSourceTableTableInfoRowFilter `json:"row_filter,omitempty"` + TableConstraints []DataSourceTableTableInfoTableConstraints `json:"table_constraints,omitempty"` + ViewDependencies *DataSourceTableTableInfoViewDependencies `json:"view_dependencies,omitempty"` +} + +type DataSourceTable struct { + Id string `json:"id,omitempty"` + Name string `json:"name"` + TableInfo *DataSourceTableTableInfo `json:"table_info,omitempty"` +} diff --git a/bundle/internal/tf/schema/data_sources.go b/bundle/internal/tf/schema/data_sources.go index 2e02c4388..c32483db0 100644 --- a/bundle/internal/tf/schema/data_sources.go +++ b/bundle/internal/tf/schema/data_sources.go @@ -7,6 +7,7 @@ type DataSources struct { AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"` AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"` AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"` + Catalog map[string]any `json:"databricks_catalog,omitempty"` Catalogs map[string]any `json:"databricks_catalogs,omitempty"` Cluster map[string]any `json:"databricks_cluster,omitempty"` ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` @@ -26,6 +27,7 @@ type DataSources struct { Jobs map[string]any `json:"databricks_jobs,omitempty"` Metastore map[string]any `json:"databricks_metastore,omitempty"` Metastores map[string]any `json:"databricks_metastores,omitempty"` + MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` @@ -43,6 +45,7 @@ type DataSources struct { SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"` StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"` + Table map[string]any `json:"databricks_table,omitempty"` Tables map[string]any `json:"databricks_tables,omitempty"` User map[string]any `json:"databricks_user,omitempty"` Views map[string]any `json:"databricks_views,omitempty"` @@ -56,6 +59,7 @@ func NewDataSources() *DataSources { AwsBucketPolicy: make(map[string]any), AwsCrossaccountPolicy: make(map[string]any), AwsUnityCatalogPolicy: make(map[string]any), + Catalog: make(map[string]any), Catalogs: make(map[string]any), Cluster: make(map[string]any), ClusterPolicy: make(map[string]any), @@ -75,6 +79,7 @@ func NewDataSources() *DataSources { Jobs: make(map[string]any), Metastore: make(map[string]any), Metastores: make(map[string]any), + MlflowExperiment: make(map[string]any), MlflowModel: make(map[string]any), MwsCredentials: make(map[string]any), MwsWorkspaces: make(map[string]any), @@ -92,6 +97,7 @@ func NewDataSources() *DataSources { SqlWarehouses: make(map[string]any), StorageCredential: make(map[string]any), StorageCredentials: make(map[string]any), + Table: make(map[string]any), Tables: make(map[string]any), User: make(map[string]any), Views: make(map[string]any), diff --git a/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go new file mode 100644 index 000000000..e95639de8 --- /dev/null +++ b/bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go @@ -0,0 +1,39 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct { + ForcedForComplianceMode bool `json:"forced_for_compliance_mode,omitempty"` + UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"` + UnavailableForNonEnterpriseTier bool `json:"unavailable_for_non_enterprise_tier,omitempty"` +} + +type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct { + Hours int `json:"hours,omitempty"` + Minutes int `json:"minutes,omitempty"` +} + +type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct { + DayOfWeek string `json:"day_of_week,omitempty"` + Frequency string `json:"frequency,omitempty"` + WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"` +} + +type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow struct { + WeekDayBasedSchedule *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule `json:"week_day_based_schedule,omitempty"` +} + +type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct { + CanToggle bool `json:"can_toggle,omitempty"` + Enabled bool `json:"enabled,omitempty"` + RestartEvenIfNoUpdatesAvailable bool `json:"restart_even_if_no_updates_available,omitempty"` + EnablementDetails *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"` + MaintenanceWindow *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"` +} + +type ResourceAutomaticClusterUpdateWorkspaceSetting struct { + Etag string `json:"etag,omitempty"` + Id string `json:"id,omitempty"` + SettingName string `json:"setting_name,omitempty"` + AutomaticClusterUpdateWorkspace *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace `json:"automatic_cluster_update_workspace,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_cluster.go b/bundle/internal/tf/schema/resource_cluster.go index 6f866ba87..e4106d049 100644 --- a/bundle/internal/tf/schema/resource_cluster.go +++ b/bundle/internal/tf/schema/resource_cluster.go @@ -32,10 +32,6 @@ type ResourceClusterAzureAttributes struct { LogAnalyticsInfo *ResourceClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } -type ResourceClusterCloneFrom struct { - SourceClusterId string `json:"source_cluster_id"` -} - type ResourceClusterClusterLogConfDbfs struct { Destination string `json:"destination"` } @@ -146,12 +142,13 @@ type ResourceClusterLibraryPypi struct { } type ResourceClusterLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceClusterLibraryCran `json:"cran,omitempty"` - Maven *ResourceClusterLibraryMaven `json:"maven,omitempty"` - Pypi *ResourceClusterLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceClusterLibraryCran `json:"cran,omitempty"` + Maven *ResourceClusterLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceClusterLibraryPypi `json:"pypi,omitempty"` } type ResourceClusterWorkloadTypeClients struct { @@ -168,7 +165,6 @@ type ResourceCluster struct { AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` - ClusterSource string `json:"cluster_source,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` DataSecurityMode string `json:"data_security_mode,omitempty"` DefaultTags map[string]string `json:"default_tags,omitempty"` @@ -194,7 +190,6 @@ type ResourceCluster struct { Autoscale *ResourceClusterAutoscale `json:"autoscale,omitempty"` AwsAttributes *ResourceClusterAwsAttributes `json:"aws_attributes,omitempty"` AzureAttributes *ResourceClusterAzureAttributes `json:"azure_attributes,omitempty"` - CloneFrom *ResourceClusterCloneFrom `json:"clone_from,omitempty"` ClusterLogConf *ResourceClusterClusterLogConf `json:"cluster_log_conf,omitempty"` ClusterMountInfo []ResourceClusterClusterMountInfo `json:"cluster_mount_info,omitempty"` DockerImage *ResourceClusterDockerImage `json:"docker_image,omitempty"` diff --git a/bundle/internal/tf/schema/resource_cluster_policy.go b/bundle/internal/tf/schema/resource_cluster_policy.go index 637fe6455..d8111fef2 100644 --- a/bundle/internal/tf/schema/resource_cluster_policy.go +++ b/bundle/internal/tf/schema/resource_cluster_policy.go @@ -19,12 +19,13 @@ type ResourceClusterPolicyLibrariesPypi struct { } type ResourceClusterPolicyLibraries struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceClusterPolicyLibrariesCran `json:"cran,omitempty"` - Maven *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"` - Pypi *ResourceClusterPolicyLibrariesPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceClusterPolicyLibrariesCran `json:"cran,omitempty"` + Maven *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"` + Pypi *ResourceClusterPolicyLibrariesPypi `json:"pypi,omitempty"` } type ResourceClusterPolicy struct { diff --git a/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go new file mode 100644 index 000000000..50815f753 --- /dev/null +++ b/bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go @@ -0,0 +1,15 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct { + ComplianceStandards []string `json:"compliance_standards,omitempty"` + IsEnabled bool `json:"is_enabled,omitempty"` +} + +type ResourceComplianceSecurityProfileWorkspaceSetting struct { + Etag string `json:"etag,omitempty"` + Id string `json:"id,omitempty"` + SettingName string `json:"setting_name,omitempty"` + ComplianceSecurityProfileWorkspace *ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace `json:"compliance_security_profile_workspace,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go new file mode 100644 index 000000000..2f552402a --- /dev/null +++ b/bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go @@ -0,0 +1,14 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct { + IsEnabled bool `json:"is_enabled,omitempty"` +} + +type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct { + Etag string `json:"etag,omitempty"` + Id string `json:"id,omitempty"` + SettingName string `json:"setting_name,omitempty"` + EnhancedSecurityMonitoringWorkspace *ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace `json:"enhanced_security_monitoring_workspace,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_job.go b/bundle/internal/tf/schema/resource_job.go index 2431262c1..0950073e2 100644 --- a/bundle/internal/tf/schema/resource_job.go +++ b/bundle/internal/tf/schema/resource_job.go @@ -39,6 +39,10 @@ type ResourceJobEnvironment struct { Spec *ResourceJobEnvironmentSpec `json:"spec,omitempty"` } +type ResourceJobGitSourceGitSnapshot struct { + UsedCommit string `json:"used_commit,omitempty"` +} + type ResourceJobGitSourceJobSource struct { DirtyState string `json:"dirty_state,omitempty"` ImportFromGitBranch string `json:"import_from_git_branch"` @@ -46,18 +50,19 @@ type ResourceJobGitSourceJobSource struct { } type ResourceJobGitSource struct { - Branch string `json:"branch,omitempty"` - Commit string `json:"commit,omitempty"` - Provider string `json:"provider,omitempty"` - Tag string `json:"tag,omitempty"` - Url string `json:"url"` - JobSource *ResourceJobGitSourceJobSource `json:"job_source,omitempty"` + Branch string `json:"branch,omitempty"` + Commit string `json:"commit,omitempty"` + Provider string `json:"provider,omitempty"` + Tag string `json:"tag,omitempty"` + Url string `json:"url"` + GitSnapshot *ResourceJobGitSourceGitSnapshot `json:"git_snapshot,omitempty"` + JobSource *ResourceJobGitSourceJobSource `json:"job_source,omitempty"` } type ResourceJobHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobHealth struct { @@ -72,7 +77,9 @@ type ResourceJobJobClusterNewClusterAutoscale struct { type ResourceJobJobClusterNewClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -80,10 +87,16 @@ type ResourceJobJobClusterNewClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourceJobJobClusterNewClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourceJobJobClusterNewClusterClusterLogConfDbfs struct { @@ -179,6 +192,32 @@ type ResourceJobJobClusterNewClusterInitScripts struct { Workspace *ResourceJobJobClusterNewClusterInitScriptsWorkspace `json:"workspace,omitempty"` } +type ResourceJobJobClusterNewClusterLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobJobClusterNewClusterLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobJobClusterNewClusterLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobJobClusterNewClusterLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobJobClusterNewClusterLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobJobClusterNewClusterLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobJobClusterNewClusterLibraryPypi `json:"pypi,omitempty"` +} + type ResourceJobJobClusterNewClusterWorkloadTypeClients struct { Jobs bool `json:"jobs,omitempty"` Notebooks bool `json:"notebooks,omitempty"` @@ -190,7 +229,6 @@ type ResourceJobJobClusterNewClusterWorkloadType struct { type ResourceJobJobClusterNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -218,11 +256,12 @@ type ResourceJobJobClusterNewCluster struct { DockerImage *ResourceJobJobClusterNewClusterDockerImage `json:"docker_image,omitempty"` GcpAttributes *ResourceJobJobClusterNewClusterGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []ResourceJobJobClusterNewClusterInitScripts `json:"init_scripts,omitempty"` + Library []ResourceJobJobClusterNewClusterLibrary `json:"library,omitempty"` WorkloadType *ResourceJobJobClusterNewClusterWorkloadType `json:"workload_type,omitempty"` } type ResourceJobJobCluster struct { - JobClusterKey string `json:"job_cluster_key,omitempty"` + JobClusterKey string `json:"job_cluster_key"` NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"` } @@ -243,12 +282,13 @@ type ResourceJobLibraryPypi struct { } type ResourceJobLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceJobLibraryCran `json:"cran,omitempty"` - Maven *ResourceJobLibraryMaven `json:"maven,omitempty"` - Pypi *ResourceJobLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobLibraryPypi `json:"pypi,omitempty"` } type ResourceJobNewClusterAutoscale struct { @@ -259,7 +299,9 @@ type ResourceJobNewClusterAutoscale struct { type ResourceJobNewClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -267,10 +309,16 @@ type ResourceJobNewClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourceJobNewClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourceJobNewClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourceJobNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourceJobNewClusterClusterLogConfDbfs struct { @@ -366,6 +414,32 @@ type ResourceJobNewClusterInitScripts struct { Workspace *ResourceJobNewClusterInitScriptsWorkspace `json:"workspace,omitempty"` } +type ResourceJobNewClusterLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobNewClusterLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobNewClusterLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobNewClusterLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobNewClusterLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobNewClusterLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobNewClusterLibraryPypi `json:"pypi,omitempty"` +} + type ResourceJobNewClusterWorkloadTypeClients struct { Jobs bool `json:"jobs,omitempty"` Notebooks bool `json:"notebooks,omitempty"` @@ -377,7 +451,6 @@ type ResourceJobNewClusterWorkloadType struct { type ResourceJobNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -405,6 +478,7 @@ type ResourceJobNewCluster struct { DockerImage *ResourceJobNewClusterDockerImage `json:"docker_image,omitempty"` GcpAttributes *ResourceJobNewClusterGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []ResourceJobNewClusterInitScripts `json:"init_scripts,omitempty"` + Library []ResourceJobNewClusterLibrary `json:"library,omitempty"` WorkloadType *ResourceJobNewClusterWorkloadType `json:"workload_type,omitempty"` } @@ -532,9 +606,9 @@ type ResourceJobTaskForEachTaskTaskEmailNotifications struct { } type ResourceJobTaskForEachTaskTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobTaskForEachTaskTaskHealth struct { @@ -558,12 +632,13 @@ type ResourceJobTaskForEachTaskTaskLibraryPypi struct { } type ResourceJobTaskForEachTaskTaskLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceJobTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"` - Maven *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"` - Pypi *ResourceJobTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"` } type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct { @@ -574,7 +649,9 @@ type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct { type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -582,10 +659,16 @@ type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfDbfs struct { @@ -681,6 +764,32 @@ type ResourceJobTaskForEachTaskTaskNewClusterInitScripts struct { Workspace *ResourceJobTaskForEachTaskTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"` } +type ResourceJobTaskForEachTaskTaskNewClusterLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskForEachTaskTaskNewClusterLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobTaskForEachTaskTaskNewClusterLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi `json:"pypi,omitempty"` +} + type ResourceJobTaskForEachTaskTaskNewClusterWorkloadTypeClients struct { Jobs bool `json:"jobs,omitempty"` Notebooks bool `json:"notebooks,omitempty"` @@ -692,7 +801,6 @@ type ResourceJobTaskForEachTaskTaskNewClusterWorkloadType struct { type ResourceJobTaskForEachTaskTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -704,7 +812,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct { IdempotencyToken string `json:"idempotency_token,omitempty"` InstancePoolId string `json:"instance_pool_id,omitempty"` NodeTypeId string `json:"node_type_id,omitempty"` - NumWorkers int `json:"num_workers"` + NumWorkers int `json:"num_workers,omitempty"` PolicyId string `json:"policy_id,omitempty"` RuntimeEngine string `json:"runtime_engine,omitempty"` SingleUserName string `json:"single_user_name,omitempty"` @@ -720,6 +828,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct { DockerImage *ResourceJobTaskForEachTaskTaskNewClusterDockerImage `json:"docker_image,omitempty"` GcpAttributes *ResourceJobTaskForEachTaskTaskNewClusterGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []ResourceJobTaskForEachTaskTaskNewClusterInitScripts `json:"init_scripts,omitempty"` + Library []ResourceJobTaskForEachTaskTaskNewClusterLibrary `json:"library,omitempty"` WorkloadType *ResourceJobTaskForEachTaskTaskNewClusterWorkloadType `json:"workload_type,omitempty"` } @@ -748,9 +857,21 @@ type ResourceJobTaskForEachTaskTaskPythonWheelTask struct { Parameters []string `json:"parameters,omitempty"` } +type ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams struct { + FullRefresh bool `json:"full_refresh,omitempty"` +} + type ResourceJobTaskForEachTaskTaskRunJobTask struct { - JobId int `json:"job_id"` - JobParameters map[string]string `json:"job_parameters,omitempty"` + DbtCommands []string `json:"dbt_commands,omitempty"` + JarParams []string `json:"jar_params,omitempty"` + JobId int `json:"job_id"` + JobParameters map[string]string `json:"job_parameters,omitempty"` + NotebookParams map[string]string `json:"notebook_params,omitempty"` + PythonNamedParams map[string]string `json:"python_named_params,omitempty"` + PythonParams []string `json:"python_params,omitempty"` + SparkSubmitParams []string `json:"spark_submit_params,omitempty"` + SqlParams map[string]string `json:"sql_params,omitempty"` + PipelineParams *ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"` } type ResourceJobTaskForEachTaskTaskSparkJarTask struct { @@ -803,7 +924,7 @@ type ResourceJobTaskForEachTaskTaskSqlTaskQuery struct { type ResourceJobTaskForEachTaskTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *ResourceJobTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"` @@ -834,33 +955,34 @@ type ResourceJobTaskForEachTaskTaskWebhookNotifications struct { } type ResourceJobTaskForEachTaskTask struct { - Description string `json:"description,omitempty"` - EnvironmentKey string `json:"environment_key,omitempty"` - ExistingClusterId string `json:"existing_cluster_id,omitempty"` - JobClusterKey string `json:"job_cluster_key,omitempty"` - MaxRetries int `json:"max_retries,omitempty"` - MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` - RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` - RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` - TimeoutSeconds int `json:"timeout_seconds,omitempty"` - ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` - DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` - DependsOn []ResourceJobTaskForEachTaskTaskDependsOn `json:"depends_on,omitempty"` - EmailNotifications *ResourceJobTaskForEachTaskTaskEmailNotifications `json:"email_notifications,omitempty"` - Health *ResourceJobTaskForEachTaskTaskHealth `json:"health,omitempty"` - Library []ResourceJobTaskForEachTaskTaskLibrary `json:"library,omitempty"` - NewCluster *ResourceJobTaskForEachTaskTaskNewCluster `json:"new_cluster,omitempty"` - NotebookTask *ResourceJobTaskForEachTaskTaskNotebookTask `json:"notebook_task,omitempty"` - NotificationSettings *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"` - PipelineTask *ResourceJobTaskForEachTaskTaskPipelineTask `json:"pipeline_task,omitempty"` - PythonWheelTask *ResourceJobTaskForEachTaskTaskPythonWheelTask `json:"python_wheel_task,omitempty"` - RunJobTask *ResourceJobTaskForEachTaskTaskRunJobTask `json:"run_job_task,omitempty"` - SparkJarTask *ResourceJobTaskForEachTaskTaskSparkJarTask `json:"spark_jar_task,omitempty"` - SparkPythonTask *ResourceJobTaskForEachTaskTaskSparkPythonTask `json:"spark_python_task,omitempty"` - SparkSubmitTask *ResourceJobTaskForEachTaskTaskSparkSubmitTask `json:"spark_submit_task,omitempty"` - SqlTask *ResourceJobTaskForEachTaskTaskSqlTask `json:"sql_task,omitempty"` - WebhookNotifications *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"` + Description string `json:"description,omitempty"` + DisableAutoOptimization bool `json:"disable_auto_optimization,omitempty"` + EnvironmentKey string `json:"environment_key,omitempty"` + ExistingClusterId string `json:"existing_cluster_id,omitempty"` + JobClusterKey string `json:"job_cluster_key,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` + RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` + RunIf string `json:"run_if,omitempty"` + TaskKey string `json:"task_key"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"` + DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"` + DependsOn []ResourceJobTaskForEachTaskTaskDependsOn `json:"depends_on,omitempty"` + EmailNotifications *ResourceJobTaskForEachTaskTaskEmailNotifications `json:"email_notifications,omitempty"` + Health *ResourceJobTaskForEachTaskTaskHealth `json:"health,omitempty"` + Library []ResourceJobTaskForEachTaskTaskLibrary `json:"library,omitempty"` + NewCluster *ResourceJobTaskForEachTaskTaskNewCluster `json:"new_cluster,omitempty"` + NotebookTask *ResourceJobTaskForEachTaskTaskNotebookTask `json:"notebook_task,omitempty"` + NotificationSettings *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"` + PipelineTask *ResourceJobTaskForEachTaskTaskPipelineTask `json:"pipeline_task,omitempty"` + PythonWheelTask *ResourceJobTaskForEachTaskTaskPythonWheelTask `json:"python_wheel_task,omitempty"` + RunJobTask *ResourceJobTaskForEachTaskTaskRunJobTask `json:"run_job_task,omitempty"` + SparkJarTask *ResourceJobTaskForEachTaskTaskSparkJarTask `json:"spark_jar_task,omitempty"` + SparkPythonTask *ResourceJobTaskForEachTaskTaskSparkPythonTask `json:"spark_python_task,omitempty"` + SparkSubmitTask *ResourceJobTaskForEachTaskTaskSparkSubmitTask `json:"spark_submit_task,omitempty"` + SqlTask *ResourceJobTaskForEachTaskTaskSqlTask `json:"sql_task,omitempty"` + WebhookNotifications *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"` } type ResourceJobTaskForEachTask struct { @@ -870,9 +992,9 @@ type ResourceJobTaskForEachTask struct { } type ResourceJobTaskHealthRules struct { - Metric string `json:"metric,omitempty"` - Op string `json:"op,omitempty"` - Value int `json:"value,omitempty"` + Metric string `json:"metric"` + Op string `json:"op"` + Value int `json:"value"` } type ResourceJobTaskHealth struct { @@ -896,12 +1018,13 @@ type ResourceJobTaskLibraryPypi struct { } type ResourceJobTaskLibrary struct { - Egg string `json:"egg,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceJobTaskLibraryCran `json:"cran,omitempty"` - Maven *ResourceJobTaskLibraryMaven `json:"maven,omitempty"` - Pypi *ResourceJobTaskLibraryPypi `json:"pypi,omitempty"` + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobTaskLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobTaskLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobTaskLibraryPypi `json:"pypi,omitempty"` } type ResourceJobTaskNewClusterAutoscale struct { @@ -912,7 +1035,9 @@ type ResourceJobTaskNewClusterAutoscale struct { type ResourceJobTaskNewClusterAwsAttributes struct { Availability string `json:"availability,omitempty"` EbsVolumeCount int `json:"ebs_volume_count,omitempty"` + EbsVolumeIops int `json:"ebs_volume_iops,omitempty"` EbsVolumeSize int `json:"ebs_volume_size,omitempty"` + EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"` EbsVolumeType string `json:"ebs_volume_type,omitempty"` FirstOnDemand int `json:"first_on_demand,omitempty"` InstanceProfileArn string `json:"instance_profile_arn,omitempty"` @@ -920,10 +1045,16 @@ type ResourceJobTaskNewClusterAwsAttributes struct { ZoneId string `json:"zone_id,omitempty"` } +type ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo struct { + LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"` + LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"` +} + type ResourceJobTaskNewClusterAzureAttributes struct { - Availability string `json:"availability,omitempty"` - FirstOnDemand int `json:"first_on_demand,omitempty"` - SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + Availability string `json:"availability,omitempty"` + FirstOnDemand int `json:"first_on_demand,omitempty"` + SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"` + LogAnalyticsInfo *ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"` } type ResourceJobTaskNewClusterClusterLogConfDbfs struct { @@ -1019,6 +1150,32 @@ type ResourceJobTaskNewClusterInitScripts struct { Workspace *ResourceJobTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"` } +type ResourceJobTaskNewClusterLibraryCran struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskNewClusterLibraryMaven struct { + Coordinates string `json:"coordinates"` + Exclusions []string `json:"exclusions,omitempty"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskNewClusterLibraryPypi struct { + Package string `json:"package"` + Repo string `json:"repo,omitempty"` +} + +type ResourceJobTaskNewClusterLibrary struct { + Egg string `json:"egg,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceJobTaskNewClusterLibraryCran `json:"cran,omitempty"` + Maven *ResourceJobTaskNewClusterLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceJobTaskNewClusterLibraryPypi `json:"pypi,omitempty"` +} + type ResourceJobTaskNewClusterWorkloadTypeClients struct { Jobs bool `json:"jobs,omitempty"` Notebooks bool `json:"notebooks,omitempty"` @@ -1030,7 +1187,6 @@ type ResourceJobTaskNewClusterWorkloadType struct { type ResourceJobTaskNewCluster struct { ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"` - AutoterminationMinutes int `json:"autotermination_minutes,omitempty"` ClusterId string `json:"cluster_id,omitempty"` ClusterName string `json:"cluster_name,omitempty"` CustomTags map[string]string `json:"custom_tags,omitempty"` @@ -1058,6 +1214,7 @@ type ResourceJobTaskNewCluster struct { DockerImage *ResourceJobTaskNewClusterDockerImage `json:"docker_image,omitempty"` GcpAttributes *ResourceJobTaskNewClusterGcpAttributes `json:"gcp_attributes,omitempty"` InitScripts []ResourceJobTaskNewClusterInitScripts `json:"init_scripts,omitempty"` + Library []ResourceJobTaskNewClusterLibrary `json:"library,omitempty"` WorkloadType *ResourceJobTaskNewClusterWorkloadType `json:"workload_type,omitempty"` } @@ -1086,9 +1243,21 @@ type ResourceJobTaskPythonWheelTask struct { Parameters []string `json:"parameters,omitempty"` } +type ResourceJobTaskRunJobTaskPipelineParams struct { + FullRefresh bool `json:"full_refresh,omitempty"` +} + type ResourceJobTaskRunJobTask struct { - JobId int `json:"job_id"` - JobParameters map[string]string `json:"job_parameters,omitempty"` + DbtCommands []string `json:"dbt_commands,omitempty"` + JarParams []string `json:"jar_params,omitempty"` + JobId int `json:"job_id"` + JobParameters map[string]string `json:"job_parameters,omitempty"` + NotebookParams map[string]string `json:"notebook_params,omitempty"` + PythonNamedParams map[string]string `json:"python_named_params,omitempty"` + PythonParams []string `json:"python_params,omitempty"` + SparkSubmitParams []string `json:"spark_submit_params,omitempty"` + SqlParams map[string]string `json:"sql_params,omitempty"` + PipelineParams *ResourceJobTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"` } type ResourceJobTaskSparkJarTask struct { @@ -1141,7 +1310,7 @@ type ResourceJobTaskSqlTaskQuery struct { type ResourceJobTaskSqlTask struct { Parameters map[string]string `json:"parameters,omitempty"` - WarehouseId string `json:"warehouse_id,omitempty"` + WarehouseId string `json:"warehouse_id"` Alert *ResourceJobTaskSqlTaskAlert `json:"alert,omitempty"` Dashboard *ResourceJobTaskSqlTaskDashboard `json:"dashboard,omitempty"` File *ResourceJobTaskSqlTaskFile `json:"file,omitempty"` @@ -1172,34 +1341,35 @@ type ResourceJobTaskWebhookNotifications struct { } type ResourceJobTask struct { - Description string `json:"description,omitempty"` - EnvironmentKey string `json:"environment_key,omitempty"` - ExistingClusterId string `json:"existing_cluster_id,omitempty"` - JobClusterKey string `json:"job_cluster_key,omitempty"` - MaxRetries int `json:"max_retries,omitempty"` - MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` - RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` - RunIf string `json:"run_if,omitempty"` - TaskKey string `json:"task_key,omitempty"` - TimeoutSeconds int `json:"timeout_seconds,omitempty"` - ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"` - DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"` - DependsOn []ResourceJobTaskDependsOn `json:"depends_on,omitempty"` - EmailNotifications *ResourceJobTaskEmailNotifications `json:"email_notifications,omitempty"` - ForEachTask *ResourceJobTaskForEachTask `json:"for_each_task,omitempty"` - Health *ResourceJobTaskHealth `json:"health,omitempty"` - Library []ResourceJobTaskLibrary `json:"library,omitempty"` - NewCluster *ResourceJobTaskNewCluster `json:"new_cluster,omitempty"` - NotebookTask *ResourceJobTaskNotebookTask `json:"notebook_task,omitempty"` - NotificationSettings *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"` - PipelineTask *ResourceJobTaskPipelineTask `json:"pipeline_task,omitempty"` - PythonWheelTask *ResourceJobTaskPythonWheelTask `json:"python_wheel_task,omitempty"` - RunJobTask *ResourceJobTaskRunJobTask `json:"run_job_task,omitempty"` - SparkJarTask *ResourceJobTaskSparkJarTask `json:"spark_jar_task,omitempty"` - SparkPythonTask *ResourceJobTaskSparkPythonTask `json:"spark_python_task,omitempty"` - SparkSubmitTask *ResourceJobTaskSparkSubmitTask `json:"spark_submit_task,omitempty"` - SqlTask *ResourceJobTaskSqlTask `json:"sql_task,omitempty"` - WebhookNotifications *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"` + Description string `json:"description,omitempty"` + DisableAutoOptimization bool `json:"disable_auto_optimization,omitempty"` + EnvironmentKey string `json:"environment_key,omitempty"` + ExistingClusterId string `json:"existing_cluster_id,omitempty"` + JobClusterKey string `json:"job_cluster_key,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"` + RetryOnTimeout bool `json:"retry_on_timeout,omitempty"` + RunIf string `json:"run_if,omitempty"` + TaskKey string `json:"task_key"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"` + DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"` + DependsOn []ResourceJobTaskDependsOn `json:"depends_on,omitempty"` + EmailNotifications *ResourceJobTaskEmailNotifications `json:"email_notifications,omitempty"` + ForEachTask *ResourceJobTaskForEachTask `json:"for_each_task,omitempty"` + Health *ResourceJobTaskHealth `json:"health,omitempty"` + Library []ResourceJobTaskLibrary `json:"library,omitempty"` + NewCluster *ResourceJobTaskNewCluster `json:"new_cluster,omitempty"` + NotebookTask *ResourceJobTaskNotebookTask `json:"notebook_task,omitempty"` + NotificationSettings *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"` + PipelineTask *ResourceJobTaskPipelineTask `json:"pipeline_task,omitempty"` + PythonWheelTask *ResourceJobTaskPythonWheelTask `json:"python_wheel_task,omitempty"` + RunJobTask *ResourceJobTaskRunJobTask `json:"run_job_task,omitempty"` + SparkJarTask *ResourceJobTaskSparkJarTask `json:"spark_jar_task,omitempty"` + SparkPythonTask *ResourceJobTaskSparkPythonTask `json:"spark_python_task,omitempty"` + SparkSubmitTask *ResourceJobTaskSparkSubmitTask `json:"spark_submit_task,omitempty"` + SqlTask *ResourceJobTaskSqlTask `json:"sql_task,omitempty"` + WebhookNotifications *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"` } type ResourceJobTriggerFileArrival struct { @@ -1208,6 +1378,13 @@ type ResourceJobTriggerFileArrival struct { WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` } +type ResourceJobTriggerTable struct { + Condition string `json:"condition,omitempty"` + MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` + TableNames []string `json:"table_names,omitempty"` + WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"` +} + type ResourceJobTriggerTableUpdate struct { Condition string `json:"condition,omitempty"` MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"` @@ -1218,6 +1395,7 @@ type ResourceJobTriggerTableUpdate struct { type ResourceJobTrigger struct { PauseStatus string `json:"pause_status,omitempty"` FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"` + Table *ResourceJobTriggerTable `json:"table,omitempty"` TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_library.go b/bundle/internal/tf/schema/resource_library.go index e2e83fb4f..385d992df 100644 --- a/bundle/internal/tf/schema/resource_library.go +++ b/bundle/internal/tf/schema/resource_library.go @@ -19,12 +19,13 @@ type ResourceLibraryPypi struct { } type ResourceLibrary struct { - ClusterId string `json:"cluster_id"` - Egg string `json:"egg,omitempty"` - Id string `json:"id,omitempty"` - Jar string `json:"jar,omitempty"` - Whl string `json:"whl,omitempty"` - Cran *ResourceLibraryCran `json:"cran,omitempty"` - Maven *ResourceLibraryMaven `json:"maven,omitempty"` - Pypi *ResourceLibraryPypi `json:"pypi,omitempty"` + ClusterId string `json:"cluster_id"` + Egg string `json:"egg,omitempty"` + Id string `json:"id,omitempty"` + Jar string `json:"jar,omitempty"` + Requirements string `json:"requirements,omitempty"` + Whl string `json:"whl,omitempty"` + Cran *ResourceLibraryCran `json:"cran,omitempty"` + Maven *ResourceLibraryMaven `json:"maven,omitempty"` + Pypi *ResourceLibraryPypi `json:"pypi,omitempty"` } diff --git a/bundle/internal/tf/schema/resource_model_serving.go b/bundle/internal/tf/schema/resource_model_serving.go index a74a544ed..f5ffbbe5e 100644 --- a/bundle/internal/tf/schema/resource_model_serving.go +++ b/bundle/internal/tf/schema/resource_model_serving.go @@ -34,12 +34,15 @@ type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServing } type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct { - OpenaiApiBase string `json:"openai_api_base,omitempty"` - OpenaiApiKey string `json:"openai_api_key"` - OpenaiApiType string `json:"openai_api_type,omitempty"` - OpenaiApiVersion string `json:"openai_api_version,omitempty"` - OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` - OpenaiOrganization string `json:"openai_organization,omitempty"` + MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"` + MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"` + MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"` + OpenaiApiBase string `json:"openai_api_base,omitempty"` + OpenaiApiKey string `json:"openai_api_key,omitempty"` + OpenaiApiType string `json:"openai_api_type,omitempty"` + OpenaiApiVersion string `json:"openai_api_version,omitempty"` + OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"` + OpenaiOrganization string `json:"openai_organization,omitempty"` } type ResourceModelServingConfigServedEntitiesExternalModelPalmConfig struct { @@ -114,6 +117,7 @@ type ResourceModelServingTags struct { type ResourceModelServing struct { Id string `json:"id,omitempty"` Name string `json:"name"` + RouteOptimized bool `json:"route_optimized,omitempty"` ServingEndpointId string `json:"serving_endpoint_id,omitempty"` Config *ResourceModelServingConfig `json:"config,omitempty"` RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"` diff --git a/bundle/internal/tf/schema/resource_mws_ncc_binding.go b/bundle/internal/tf/schema/resource_mws_ncc_binding.go new file mode 100644 index 000000000..8beafb6f5 --- /dev/null +++ b/bundle/internal/tf/schema/resource_mws_ncc_binding.go @@ -0,0 +1,9 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceMwsNccBinding struct { + Id string `json:"id,omitempty"` + NetworkConnectivityConfigId string `json:"network_connectivity_config_id"` + WorkspaceId int `json:"workspace_id"` +} diff --git a/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go b/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go new file mode 100644 index 000000000..2acb374bc --- /dev/null +++ b/bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go @@ -0,0 +1,17 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceMwsNccPrivateEndpointRule struct { + ConnectionState string `json:"connection_state,omitempty"` + CreationTime int `json:"creation_time,omitempty"` + Deactivated bool `json:"deactivated,omitempty"` + DeactivatedAt int `json:"deactivated_at,omitempty"` + EndpointName string `json:"endpoint_name,omitempty"` + GroupId string `json:"group_id"` + Id string `json:"id,omitempty"` + NetworkConnectivityConfigId string `json:"network_connectivity_config_id"` + ResourceId string `json:"resource_id"` + RuleId string `json:"rule_id,omitempty"` + UpdatedTime int `json:"updated_time,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go b/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go new file mode 100644 index 000000000..64ebab224 --- /dev/null +++ b/bundle/internal/tf/schema/resource_mws_network_connectivity_config.go @@ -0,0 +1,51 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule struct { + CidrBlocks []string `json:"cidr_blocks,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule struct { + Subnets []string `json:"subnets,omitempty"` + TargetRegion string `json:"target_region,omitempty"` + TargetServices []string `json:"target_services,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules struct { + AwsStableIpRule *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule `json:"aws_stable_ip_rule,omitempty"` + AzureServiceEndpointRule *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule `json:"azure_service_endpoint_rule,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules struct { + ConnectionState string `json:"connection_state,omitempty"` + CreationTime int `json:"creation_time,omitempty"` + Deactivated bool `json:"deactivated,omitempty"` + DeactivatedAt int `json:"deactivated_at,omitempty"` + EndpointName string `json:"endpoint_name,omitempty"` + GroupId string `json:"group_id,omitempty"` + NetworkConnectivityConfigId string `json:"network_connectivity_config_id,omitempty"` + ResourceId string `json:"resource_id,omitempty"` + RuleId string `json:"rule_id,omitempty"` + UpdatedTime int `json:"updated_time,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules struct { + AzurePrivateEndpointRules []ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules `json:"azure_private_endpoint_rules,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfigEgressConfig struct { + DefaultRules *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules `json:"default_rules,omitempty"` + TargetRules *ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules `json:"target_rules,omitempty"` +} + +type ResourceMwsNetworkConnectivityConfig struct { + AccountId string `json:"account_id,omitempty"` + CreationTime int `json:"creation_time,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name"` + NetworkConnectivityConfigId string `json:"network_connectivity_config_id,omitempty"` + Region string `json:"region"` + UpdatedTime int `json:"updated_time,omitempty"` + EgressConfig *ResourceMwsNetworkConnectivityConfigEgressConfig `json:"egress_config,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_quality_monitor.go b/bundle/internal/tf/schema/resource_quality_monitor.go new file mode 100644 index 000000000..0fc2abd66 --- /dev/null +++ b/bundle/internal/tf/schema/resource_quality_monitor.go @@ -0,0 +1,76 @@ +// Generated from Databricks Terraform provider schema. DO NOT EDIT. + +package schema + +type ResourceQualityMonitorCustomMetrics struct { + Definition string `json:"definition"` + InputColumns []string `json:"input_columns"` + Name string `json:"name"` + OutputDataType string `json:"output_data_type"` + Type string `json:"type"` +} + +type ResourceQualityMonitorDataClassificationConfig struct { + Enabled bool `json:"enabled,omitempty"` +} + +type ResourceQualityMonitorInferenceLog struct { + Granularities []string `json:"granularities"` + LabelCol string `json:"label_col,omitempty"` + ModelIdCol string `json:"model_id_col"` + PredictionCol string `json:"prediction_col"` + PredictionProbaCol string `json:"prediction_proba_col,omitempty"` + ProblemType string `json:"problem_type"` + TimestampCol string `json:"timestamp_col"` +} + +type ResourceQualityMonitorNotificationsOnFailure struct { + EmailAddresses []string `json:"email_addresses,omitempty"` +} + +type ResourceQualityMonitorNotificationsOnNewClassificationTagDetected struct { + EmailAddresses []string `json:"email_addresses,omitempty"` +} + +type ResourceQualityMonitorNotifications struct { + OnFailure *ResourceQualityMonitorNotificationsOnFailure `json:"on_failure,omitempty"` + OnNewClassificationTagDetected *ResourceQualityMonitorNotificationsOnNewClassificationTagDetected `json:"on_new_classification_tag_detected,omitempty"` +} + +type ResourceQualityMonitorSchedule struct { + PauseStatus string `json:"pause_status,omitempty"` + QuartzCronExpression string `json:"quartz_cron_expression"` + TimezoneId string `json:"timezone_id"` +} + +type ResourceQualityMonitorSnapshot struct { +} + +type ResourceQualityMonitorTimeSeries struct { + Granularities []string `json:"granularities"` + TimestampCol string `json:"timestamp_col"` +} + +type ResourceQualityMonitor struct { + AssetsDir string `json:"assets_dir"` + BaselineTableName string `json:"baseline_table_name,omitempty"` + DashboardId string `json:"dashboard_id,omitempty"` + DriftMetricsTableName string `json:"drift_metrics_table_name,omitempty"` + Id string `json:"id,omitempty"` + LatestMonitorFailureMsg string `json:"latest_monitor_failure_msg,omitempty"` + MonitorVersion string `json:"monitor_version,omitempty"` + OutputSchemaName string `json:"output_schema_name"` + ProfileMetricsTableName string `json:"profile_metrics_table_name,omitempty"` + SkipBuiltinDashboard bool `json:"skip_builtin_dashboard,omitempty"` + SlicingExprs []string `json:"slicing_exprs,omitempty"` + Status string `json:"status,omitempty"` + TableName string `json:"table_name"` + WarehouseId string `json:"warehouse_id,omitempty"` + CustomMetrics []ResourceQualityMonitorCustomMetrics `json:"custom_metrics,omitempty"` + DataClassificationConfig *ResourceQualityMonitorDataClassificationConfig `json:"data_classification_config,omitempty"` + InferenceLog *ResourceQualityMonitorInferenceLog `json:"inference_log,omitempty"` + Notifications *ResourceQualityMonitorNotifications `json:"notifications,omitempty"` + Schedule *ResourceQualityMonitorSchedule `json:"schedule,omitempty"` + Snapshot *ResourceQualityMonitorSnapshot `json:"snapshot,omitempty"` + TimeSeries *ResourceQualityMonitorTimeSeries `json:"time_series,omitempty"` +} diff --git a/bundle/internal/tf/schema/resource_sql_table.go b/bundle/internal/tf/schema/resource_sql_table.go index 97a8977bc..51fb3bc0d 100644 --- a/bundle/internal/tf/schema/resource_sql_table.go +++ b/bundle/internal/tf/schema/resource_sql_table.go @@ -18,6 +18,7 @@ type ResourceSqlTable struct { Id string `json:"id,omitempty"` Name string `json:"name"` Options map[string]string `json:"options,omitempty"` + Owner string `json:"owner,omitempty"` Partitions []string `json:"partitions,omitempty"` Properties map[string]string `json:"properties,omitempty"` SchemaName string `json:"schema_name"` diff --git a/bundle/internal/tf/schema/resource_vector_search_index.go b/bundle/internal/tf/schema/resource_vector_search_index.go index 06f666656..2ce51576d 100644 --- a/bundle/internal/tf/schema/resource_vector_search_index.go +++ b/bundle/internal/tf/schema/resource_vector_search_index.go @@ -13,11 +13,12 @@ type ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns struct { } type ResourceVectorSearchIndexDeltaSyncIndexSpec struct { - PipelineId string `json:"pipeline_id,omitempty"` - PipelineType string `json:"pipeline_type,omitempty"` - SourceTable string `json:"source_table,omitempty"` - EmbeddingSourceColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"` - EmbeddingVectorColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"` + EmbeddingWritebackTable string `json:"embedding_writeback_table,omitempty"` + PipelineId string `json:"pipeline_id,omitempty"` + PipelineType string `json:"pipeline_type,omitempty"` + SourceTable string `json:"source_table,omitempty"` + EmbeddingSourceColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"` + EmbeddingVectorColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"` } type ResourceVectorSearchIndexDirectAccessIndexSpecEmbeddingSourceColumns struct { diff --git a/bundle/internal/tf/schema/resources.go b/bundle/internal/tf/schema/resources.go index b1b1841d6..79d71a65f 100644 --- a/bundle/internal/tf/schema/resources.go +++ b/bundle/internal/tf/schema/resources.go @@ -3,112 +3,122 @@ package schema type Resources struct { - AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"` - ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"` - AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"` - AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` - AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` - AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` - Catalog map[string]any `json:"databricks_catalog,omitempty"` - CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` - Cluster map[string]any `json:"databricks_cluster,omitempty"` - ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` - Connection map[string]any `json:"databricks_connection,omitempty"` - DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` - DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` - Directory map[string]any `json:"databricks_directory,omitempty"` - Entitlements map[string]any `json:"databricks_entitlements,omitempty"` - ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` - File map[string]any `json:"databricks_file,omitempty"` - GitCredential map[string]any `json:"databricks_git_credential,omitempty"` - GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"` - Grant map[string]any `json:"databricks_grant,omitempty"` - Grants map[string]any `json:"databricks_grants,omitempty"` - Group map[string]any `json:"databricks_group,omitempty"` - GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"` - GroupMember map[string]any `json:"databricks_group_member,omitempty"` - GroupRole map[string]any `json:"databricks_group_role,omitempty"` - InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` - InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"` - IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"` - Job map[string]any `json:"databricks_job,omitempty"` - LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"` - Library map[string]any `json:"databricks_library,omitempty"` - Metastore map[string]any `json:"databricks_metastore,omitempty"` - MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"` - MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"` - MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` - MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` - MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"` - ModelServing map[string]any `json:"databricks_model_serving,omitempty"` - Mount map[string]any `json:"databricks_mount,omitempty"` - MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` - MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"` - MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"` - MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"` - MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"` - MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"` - MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"` - MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` - MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` - Notebook map[string]any `json:"databricks_notebook,omitempty"` - OboToken map[string]any `json:"databricks_obo_token,omitempty"` - OnlineTable map[string]any `json:"databricks_online_table,omitempty"` - PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` - Permissions map[string]any `json:"databricks_permissions,omitempty"` - Pipeline map[string]any `json:"databricks_pipeline,omitempty"` - Provider map[string]any `json:"databricks_provider,omitempty"` - Recipient map[string]any `json:"databricks_recipient,omitempty"` - RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` - Repo map[string]any `json:"databricks_repo,omitempty"` - RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"` - Schema map[string]any `json:"databricks_schema,omitempty"` - Secret map[string]any `json:"databricks_secret,omitempty"` - SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"` - SecretScope map[string]any `json:"databricks_secret_scope,omitempty"` - ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` - ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"` - ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"` - Share map[string]any `json:"databricks_share,omitempty"` - SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"` - SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"` - SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"` - SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"` - SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"` - SqlQuery map[string]any `json:"databricks_sql_query,omitempty"` - SqlTable map[string]any `json:"databricks_sql_table,omitempty"` - SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"` - SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"` - StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` - SystemSchema map[string]any `json:"databricks_system_schema,omitempty"` - Table map[string]any `json:"databricks_table,omitempty"` - Token map[string]any `json:"databricks_token,omitempty"` - User map[string]any `json:"databricks_user,omitempty"` - UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"` - UserRole map[string]any `json:"databricks_user_role,omitempty"` - VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` - VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` - Volume map[string]any `json:"databricks_volume,omitempty"` - WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` - WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` + AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"` + ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"` + AutomaticClusterUpdateWorkspaceSetting map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"` + AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"` + AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"` + AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"` + AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"` + Catalog map[string]any `json:"databricks_catalog,omitempty"` + CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"` + Cluster map[string]any `json:"databricks_cluster,omitempty"` + ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"` + ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"` + Connection map[string]any `json:"databricks_connection,omitempty"` + DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"` + DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"` + Directory map[string]any `json:"databricks_directory,omitempty"` + EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"` + Entitlements map[string]any `json:"databricks_entitlements,omitempty"` + ExternalLocation map[string]any `json:"databricks_external_location,omitempty"` + File map[string]any `json:"databricks_file,omitempty"` + GitCredential map[string]any `json:"databricks_git_credential,omitempty"` + GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"` + Grant map[string]any `json:"databricks_grant,omitempty"` + Grants map[string]any `json:"databricks_grants,omitempty"` + Group map[string]any `json:"databricks_group,omitempty"` + GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"` + GroupMember map[string]any `json:"databricks_group_member,omitempty"` + GroupRole map[string]any `json:"databricks_group_role,omitempty"` + InstancePool map[string]any `json:"databricks_instance_pool,omitempty"` + InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"` + IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"` + Job map[string]any `json:"databricks_job,omitempty"` + LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"` + Library map[string]any `json:"databricks_library,omitempty"` + Metastore map[string]any `json:"databricks_metastore,omitempty"` + MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"` + MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"` + MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"` + MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"` + MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"` + ModelServing map[string]any `json:"databricks_model_serving,omitempty"` + Mount map[string]any `json:"databricks_mount,omitempty"` + MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"` + MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"` + MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"` + MwsNccBinding map[string]any `json:"databricks_mws_ncc_binding,omitempty"` + MwsNccPrivateEndpointRule map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"` + MwsNetworkConnectivityConfig map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"` + MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"` + MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"` + MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"` + MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"` + MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"` + MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"` + Notebook map[string]any `json:"databricks_notebook,omitempty"` + OboToken map[string]any `json:"databricks_obo_token,omitempty"` + OnlineTable map[string]any `json:"databricks_online_table,omitempty"` + PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"` + Permissions map[string]any `json:"databricks_permissions,omitempty"` + Pipeline map[string]any `json:"databricks_pipeline,omitempty"` + Provider map[string]any `json:"databricks_provider,omitempty"` + QualityMonitor map[string]any `json:"databricks_quality_monitor,omitempty"` + Recipient map[string]any `json:"databricks_recipient,omitempty"` + RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"` + Repo map[string]any `json:"databricks_repo,omitempty"` + RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"` + Schema map[string]any `json:"databricks_schema,omitempty"` + Secret map[string]any `json:"databricks_secret,omitempty"` + SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"` + SecretScope map[string]any `json:"databricks_secret_scope,omitempty"` + ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"` + ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"` + ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"` + Share map[string]any `json:"databricks_share,omitempty"` + SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"` + SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"` + SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"` + SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"` + SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"` + SqlQuery map[string]any `json:"databricks_sql_query,omitempty"` + SqlTable map[string]any `json:"databricks_sql_table,omitempty"` + SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"` + SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"` + StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"` + SystemSchema map[string]any `json:"databricks_system_schema,omitempty"` + Table map[string]any `json:"databricks_table,omitempty"` + Token map[string]any `json:"databricks_token,omitempty"` + User map[string]any `json:"databricks_user,omitempty"` + UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"` + UserRole map[string]any `json:"databricks_user_role,omitempty"` + VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"` + VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"` + Volume map[string]any `json:"databricks_volume,omitempty"` + WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"` + WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"` } func NewResources() *Resources { return &Resources{ - AccessControlRuleSet: make(map[string]any), - ArtifactAllowlist: make(map[string]any), - AwsS3Mount: make(map[string]any), - AzureAdlsGen1Mount: make(map[string]any), - AzureAdlsGen2Mount: make(map[string]any), - AzureBlobMount: make(map[string]any), - Catalog: make(map[string]any), - CatalogWorkspaceBinding: make(map[string]any), - Cluster: make(map[string]any), - ClusterPolicy: make(map[string]any), - Connection: make(map[string]any), - DbfsFile: make(map[string]any), - DefaultNamespaceSetting: make(map[string]any), - Directory: make(map[string]any), + AccessControlRuleSet: make(map[string]any), + ArtifactAllowlist: make(map[string]any), + AutomaticClusterUpdateWorkspaceSetting: make(map[string]any), + AwsS3Mount: make(map[string]any), + AzureAdlsGen1Mount: make(map[string]any), + AzureAdlsGen2Mount: make(map[string]any), + AzureBlobMount: make(map[string]any), + Catalog: make(map[string]any), + CatalogWorkspaceBinding: make(map[string]any), + Cluster: make(map[string]any), + ClusterPolicy: make(map[string]any), + ComplianceSecurityProfileWorkspaceSetting: make(map[string]any), + Connection: make(map[string]any), + DbfsFile: make(map[string]any), + DefaultNamespaceSetting: make(map[string]any), + Directory: make(map[string]any), + EnhancedSecurityMonitoringWorkspaceSetting: make(map[string]any), Entitlements: make(map[string]any), ExternalLocation: make(map[string]any), File: make(map[string]any), @@ -137,6 +147,9 @@ func NewResources() *Resources { MwsCredentials: make(map[string]any), MwsCustomerManagedKeys: make(map[string]any), MwsLogDelivery: make(map[string]any), + MwsNccBinding: make(map[string]any), + MwsNccPrivateEndpointRule: make(map[string]any), + MwsNetworkConnectivityConfig: make(map[string]any), MwsNetworks: make(map[string]any), MwsPermissionAssignment: make(map[string]any), MwsPrivateAccessSettings: make(map[string]any), @@ -150,6 +163,7 @@ func NewResources() *Resources { Permissions: make(map[string]any), Pipeline: make(map[string]any), Provider: make(map[string]any), + QualityMonitor: make(map[string]any), Recipient: make(map[string]any), RegisteredModel: make(map[string]any), Repo: make(map[string]any), diff --git a/bundle/internal/tf/schema/root.go b/bundle/internal/tf/schema/root.go index be6852bc0..e4ca67740 100644 --- a/bundle/internal/tf/schema/root.go +++ b/bundle/internal/tf/schema/root.go @@ -21,7 +21,7 @@ type Root struct { const ProviderHost = "registry.terraform.io" const ProviderSource = "databricks/databricks" -const ProviderVersion = "1.40.0" +const ProviderVersion = "1.46.0" func NewRoot() *Root { return &Root{ diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index a79adedbf..84ead052b 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -30,6 +30,10 @@ func FindAllEnvironments(b *bundle.Bundle) map[string]([]jobs.JobEnvironment) { func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool { for _, e := range envs { + if e.Spec == nil { + continue + } + for _, l := range e.Spec.Dependencies { if IsEnvironmentDependencyLocal(l) { return true diff --git a/bundle/libraries/match.go b/bundle/libraries/match.go index 096cdf4a5..4feb4225d 100644 --- a/bundle/libraries/match.go +++ b/bundle/libraries/match.go @@ -62,6 +62,10 @@ func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error { func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error { for _, env := range envs { + if env.Spec == nil { + continue + } + for _, dep := range env.Spec.Dependencies { matches, err := filepath.Glob(filepath.Join(b.RootPath, dep)) if err != nil { diff --git a/bundle/parallel_test.go b/bundle/parallel_test.go index be1e33637..dfc7ddac9 100644 --- a/bundle/parallel_test.go +++ b/bundle/parallel_test.go @@ -2,6 +2,7 @@ package bundle import ( "context" + "sync" "testing" "github.com/databricks/cli/bundle/config" @@ -10,9 +11,14 @@ import ( ) type addToContainer struct { + t *testing.T container *[]int value int err bool + + // mu is a mutex that protects container. It is used to ensure that the + // container slice is only modified by one goroutine at a time. + mu *sync.Mutex } func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagnostics { @@ -20,9 +26,10 @@ func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagn return diag.Errorf("error") } - c := *m.container - c = append(c, m.value) - *m.container = c + m.mu.Lock() + *m.container = append(*m.container, m.value) + m.mu.Unlock() + return nil } @@ -36,9 +43,10 @@ func TestParallelMutatorWork(t *testing.T) { } container := []int{} - m1 := &addToContainer{container: &container, value: 1} - m2 := &addToContainer{container: &container, value: 2} - m3 := &addToContainer{container: &container, value: 3} + var mu sync.Mutex + m1 := &addToContainer{t: t, container: &container, value: 1, mu: &mu} + m2 := &addToContainer{t: t, container: &container, value: 2, mu: &mu} + m3 := &addToContainer{t: t, container: &container, value: 3, mu: &mu} m := Parallel(m1, m2, m3) @@ -57,9 +65,10 @@ func TestParallelMutatorWorkWithErrors(t *testing.T) { } container := []int{} - m1 := &addToContainer{container: &container, value: 1} - m2 := &addToContainer{container: &container, err: true, value: 2} - m3 := &addToContainer{container: &container, value: 3} + var mu sync.Mutex + m1 := &addToContainer{container: &container, value: 1, mu: &mu} + m2 := &addToContainer{container: &container, err: true, value: 2, mu: &mu} + m3 := &addToContainer{container: &container, value: 3, mu: &mu} m := Parallel(m1, m2, m3) diff --git a/bundle/permissions/filter_test.go b/bundle/permissions/filter_test.go index 410fa4be8..121ce10dc 100644 --- a/bundle/permissions/filter_test.go +++ b/bundle/permissions/filter_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" ) @@ -45,9 +46,15 @@ func testFixture(userName string) *bundle.Bundle { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { + JobSettings: &jobs.JobSettings{ + Name: "job1", + }, Permissions: p, }, "job2": { + JobSettings: &jobs.JobSettings{ + Name: "job2", + }, Permissions: p, }, }, diff --git a/bundle/permissions/mutator_test.go b/bundle/permissions/mutator_test.go index 438a15061..1a177d902 100644 --- a/bundle/permissions/mutator_test.go +++ b/bundle/permissions/mutator_test.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -23,8 +24,16 @@ func TestApplyBundlePermissions(t *testing.T) { }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ - "job_1": {}, - "job_2": {}, + "job_1": { + JobSettings: &jobs.JobSettings{ + Name: "job_1", + }, + }, + "job_2": { + JobSettings: &jobs.JobSettings{ + Name: "job_2", + }, + }, }, Pipelines: map[string]*resources.Pipeline{ "pipeline_1": {}, @@ -109,11 +118,17 @@ func TestWarningOnOverlapPermission(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job_1": { + JobSettings: &jobs.JobSettings{ + Name: "job_1", + }, Permissions: []resources.Permission{ {Level: CAN_VIEW, UserName: "TestUser"}, }, }, "job_2": { + JobSettings: &jobs.JobSettings{ + Name: "job_2", + }, Permissions: []resources.Permission{ {Level: CAN_VIEW, UserName: "TestUser2"}, }, diff --git a/bundle/permissions/workspace_root_test.go b/bundle/permissions/workspace_root_test.go index 7dd97b62d..5e23a1da8 100644 --- a/bundle/permissions/workspace_root_test.go +++ b/bundle/permissions/workspace_root_test.go @@ -30,8 +30,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) { }, Resources: config.Resources{ Jobs: map[string]*resources.Job{ - "job_1": {JobSettings: &jobs.JobSettings{}}, - "job_2": {JobSettings: &jobs.JobSettings{}}, + "job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}}, + "job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}}, }, Pipelines: map[string]*resources.Pipeline{ "pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}}, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index fce98b038..46c389189 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -36,7 +36,7 @@ func Deploy() bundle.Mutator { permissions.ApplyWorkspaceRootPermissions(), terraform.Interpolate(), terraform.Write(), - deploy.CheckRunningResource(), + terraform.CheckRunningResource(), bundle.Defer( terraform.Apply(), bundle.Seq( diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 940af953e..563657677 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -46,6 +46,7 @@ func Initialize() bundle.Mutator { permissions.ApplyBundlePermissions(), permissions.FilterCurrentUser(), metadata.AnnotateJobs(), + metadata.AnnotatePipelines(), terraform.Initialize(), scripts.Execute(config.ScriptPostInit), }, diff --git a/bundle/schema/docs.go b/bundle/schema/docs.go index fe63e4328..5b960ea55 100644 --- a/bundle/schema/docs.go +++ b/bundle/schema/docs.go @@ -70,7 +70,7 @@ func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) { } openapiReader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } // Generate descriptions for the "resources" field diff --git a/bundle/schema/docs/bundle_descriptions.json b/bundle/schema/docs/bundle_descriptions.json index ca889ae52..b6d0235aa 100644 --- a/bundle/schema/docs/bundle_descriptions.json +++ b/bundle/schema/docs/bundle_descriptions.json @@ -46,6 +46,17 @@ "properties": { "fail_on_active_runs": { "description": "" + }, + "lock": { + "description": "", + "properties": { + "enabled": { + "description": "" + }, + "force": { + "description": "" + } + } } } }, @@ -76,6 +87,9 @@ "additionalproperties": { "description": "" } + }, + "use_legacy_run_as": { + "description": "" } } }, @@ -242,7 +256,7 @@ "description": "", "properties": { "client": { - "description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version" + "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." }, "dependencies": { "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", @@ -334,7 +348,7 @@ "description": "If new_cluster, a description of a cluster that is created for each task.", "properties": { "apply_policy_default_values": { - "description": "" + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." }, "autoscale": { "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", @@ -410,14 +424,6 @@ } } }, - "clone_from": { - "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.", - "properties": { - "source_cluster_id": { - "description": "The cluster that is being cloned." - } - } - }, "cluster_log_conf": { "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", "properties": { @@ -460,9 +466,6 @@ "cluster_name": { "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" }, - "cluster_source": { - "description": "" - }, "custom_tags": { "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", "additionalproperties": { @@ -742,7 +745,7 @@ "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", "properties": { "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." + "description": "Indicate whether this schedule is paused or not." }, "quartz_cron_expression": { "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." @@ -799,7 +802,7 @@ "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." }, "warehouse_id": { "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." @@ -909,10 +912,10 @@ } }, "egg": { - "description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." }, "jar": { - "description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." }, "maven": { "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", @@ -942,8 +945,11 @@ } } }, + "requirements": { + "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" + }, "whl": { - "description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." } } } @@ -955,10 +961,10 @@ "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." }, "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", + "description": "If new_cluster, a description of a new cluster that is created for each run.", "properties": { "apply_policy_default_values": { - "description": "" + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." }, "autoscale": { "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", @@ -1034,14 +1040,6 @@ } } }, - "clone_from": { - "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.", - "properties": { - "source_cluster_id": { - "description": "The cluster that is being cloned." - } - } - }, "cluster_log_conf": { "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", "properties": { @@ -1084,9 +1082,6 @@ "cluster_name": { "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" }, - "cluster_source": { - "description": "" - }, "custom_tags": { "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", "additionalproperties": { @@ -1303,6 +1298,9 @@ }, "source": { "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + }, + "warehouse_id": { + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." } } }, @@ -1399,7 +1397,7 @@ } }, "python_named_params": { - "description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.", + "description": "", "additionalproperties": { "description": "" } @@ -1454,7 +1452,7 @@ "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." } } }, @@ -1526,13 +1524,13 @@ } }, "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.", + "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", "properties": { "path": { "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." } } }, @@ -1634,10 +1632,10 @@ } }, "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." + "description": "Whether this trigger is paused or not." }, "table": { - "description": "", + "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", "properties": { "condition": { "description": "The table(s) condition based on which to trigger a job run." @@ -1679,7 +1677,7 @@ } }, "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", + "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", "properties": { "on_duration_warning_threshold_exceeded": { "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", @@ -1833,6 +1831,15 @@ "openai_config": { "description": "OpenAI Config. Only required if the provider is 'openai'.", "properties": { + "microsoft_entra_client_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" + }, + "microsoft_entra_client_secret": { + "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + }, + "microsoft_entra_tenant_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" + }, "openai_api_base": { "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" }, @@ -1989,6 +1996,9 @@ } } }, + "route_optimized": { + "description": "Enable route optimization for the serving endpoint." + }, "tags": { "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", "items": { @@ -2415,6 +2425,17 @@ "continuous": { "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." }, + "deployment": { + "description": "Deployment type of this pipeline.", + "properties": { + "kind": { + "description": "The deployment method that manages the pipeline." + }, + "metadata_file_path": { + "description": "The path to the file containing metadata about the deployment." + } + } + }, "development": { "description": "Whether the pipeline is in Development mode. Defaults to false." }, @@ -2438,9 +2459,136 @@ } } }, + "gateway_definition": { + "description": "The definition of a gateway pipeline to support CDC.", + "properties": { + "connection_id": { + "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." + }, + "gateway_storage_catalog": { + "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." + }, + "gateway_storage_name": { + "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + }, + "gateway_storage_schema": { + "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." + } + } + }, "id": { "description": "Unique identifier for this pipeline." }, + "ingestion_definition": { + "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", + "properties": { + "connection_name": { + "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." + }, + "ingestion_gateway_id": { + "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." + }, + "objects": { + "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", + "items": { + "description": "", + "properties": { + "schema": { + "description": "Select tables from a specific source schema.", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store tables." + }, + "destination_schema": { + "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." + }, + "source_catalog": { + "description": "The source catalog name. Might be optional depending on the type of source." + }, + "source_schema": { + "description": "Required. Schema name in the source database." + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + }, + "table": { + "description": "Select tables from a specific source table.", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table." + }, + "destination_schema": { + "description": "Required. Destination schema to store table." + }, + "destination_table": { + "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." + }, + "source_catalog": { + "description": "Source catalog name. Might be optional depending on the type of source." + }, + "source_schema": { + "description": "Schema name in the source database. Might be optional depending on the type of source." + }, + "source_table": { + "description": "Required. Table name in the source database." + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + } + } + } + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + }, "libraries": { "description": "Libraries or code needed by this deployment.", "items": { @@ -2682,6 +2830,17 @@ "properties": { "fail_on_active_runs": { "description": "" + }, + "lock": { + "description": "", + "properties": { + "enabled": { + "description": "" + }, + "force": { + "description": "" + } + } } } }, @@ -2878,7 +3037,7 @@ "description": "", "properties": { "client": { - "description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version" + "description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version." }, "dependencies": { "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]", @@ -2970,7 +3129,7 @@ "description": "If new_cluster, a description of a cluster that is created for each task.", "properties": { "apply_policy_default_values": { - "description": "" + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." }, "autoscale": { "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", @@ -3046,14 +3205,6 @@ } } }, - "clone_from": { - "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.", - "properties": { - "source_cluster_id": { - "description": "The cluster that is being cloned." - } - } - }, "cluster_log_conf": { "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", "properties": { @@ -3096,9 +3247,6 @@ "cluster_name": { "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" }, - "cluster_source": { - "description": "" - }, "custom_tags": { "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", "additionalproperties": { @@ -3378,7 +3526,7 @@ "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", "properties": { "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." + "description": "Indicate whether this schedule is paused or not." }, "quartz_cron_expression": { "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required." @@ -3435,7 +3583,7 @@ "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider." }, "warehouse_id": { "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument." @@ -3545,10 +3693,10 @@ } }, "egg": { - "description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." }, "jar": { - "description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." }, "maven": { "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", @@ -3578,8 +3726,11 @@ } } }, + "requirements": { + "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`" + }, "whl": { - "description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." + "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI." } } } @@ -3591,10 +3742,10 @@ "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried." }, "new_cluster": { - "description": "If new_cluster, a description of a cluster that is created for each task.", + "description": "If new_cluster, a description of a new cluster that is created for each run.", "properties": { "apply_policy_default_values": { - "description": "" + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied." }, "autoscale": { "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", @@ -3670,14 +3821,6 @@ } } }, - "clone_from": { - "description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.", - "properties": { - "source_cluster_id": { - "description": "The cluster that is being cloned." - } - } - }, "cluster_log_conf": { "description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", "properties": { @@ -3720,9 +3863,6 @@ "cluster_name": { "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n" }, - "cluster_source": { - "description": "" - }, "custom_tags": { "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", "additionalproperties": { @@ -3939,6 +4079,9 @@ }, "source": { "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + }, + "warehouse_id": { + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail." } } }, @@ -4035,7 +4178,7 @@ } }, "python_named_params": { - "description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.", + "description": "", "additionalproperties": { "description": "" } @@ -4090,7 +4233,7 @@ "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository." } } }, @@ -4162,13 +4305,13 @@ } }, "file": { - "description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.", + "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", "properties": { "path": { "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths." }, "source": { - "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider." + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider." } } }, @@ -4270,10 +4413,10 @@ } }, "pause_status": { - "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED." + "description": "Whether this trigger is paused or not." }, "table": { - "description": "", + "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", "properties": { "condition": { "description": "The table(s) condition based on which to trigger a job run." @@ -4315,7 +4458,7 @@ } }, "webhook_notifications": { - "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", + "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", "properties": { "on_duration_warning_threshold_exceeded": { "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", @@ -4469,6 +4612,15 @@ "openai_config": { "description": "OpenAI Config. Only required if the provider is 'openai'.", "properties": { + "microsoft_entra_client_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n" + }, + "microsoft_entra_client_secret": { + "description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n" + }, + "microsoft_entra_tenant_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n" + }, "openai_api_base": { "description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n" }, @@ -4625,6 +4777,9 @@ } } }, + "route_optimized": { + "description": "Enable route optimization for the serving endpoint." + }, "tags": { "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", "items": { @@ -5051,6 +5206,17 @@ "continuous": { "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`." }, + "deployment": { + "description": "Deployment type of this pipeline.", + "properties": { + "kind": { + "description": "The deployment method that manages the pipeline." + }, + "metadata_file_path": { + "description": "The path to the file containing metadata about the deployment." + } + } + }, "development": { "description": "Whether the pipeline is in Development mode. Defaults to false." }, @@ -5074,9 +5240,136 @@ } } }, + "gateway_definition": { + "description": "The definition of a gateway pipeline to support CDC.", + "properties": { + "connection_id": { + "description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source." + }, + "gateway_storage_catalog": { + "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location." + }, + "gateway_storage_name": { + "description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n" + }, + "gateway_storage_schema": { + "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location." + } + } + }, "id": { "description": "Unique identifier for this pipeline." }, + "ingestion_definition": { + "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.", + "properties": { + "connection_name": { + "description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name." + }, + "ingestion_gateway_id": { + "description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name." + }, + "objects": { + "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", + "items": { + "description": "", + "properties": { + "schema": { + "description": "Select tables from a specific source schema.", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store tables." + }, + "destination_schema": { + "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists." + }, + "source_catalog": { + "description": "The source catalog name. Might be optional depending on the type of source." + }, + "source_schema": { + "description": "Required. Schema name in the source database." + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + }, + "table": { + "description": "Select tables from a specific source table.", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table." + }, + "destination_schema": { + "description": "Required. Destination schema to store table." + }, + "destination_table": { + "description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used." + }, + "source_catalog": { + "description": "Source catalog name. Might be optional depending on the type of source." + }, + "source_schema": { + "description": "Schema name in the source database. Might be optional depending on the type of source." + }, + "source_table": { + "description": "Required. Table name in the source database." + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + } + } + } + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", + "properties": { + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "items": { + "description": "" + } + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector" + }, + "scd_type": { + "description": "The SCD type to use to ingest the table." + } + } + } + } + }, "libraries": { "description": "Libraries or code needed by this deployment.", "items": { diff --git a/bundle/schema/openapi.go b/bundle/schema/openapi.go index fe329e7ac..1756d5165 100644 --- a/bundle/schema/openapi.go +++ b/bundle/schema/openapi.go @@ -10,17 +10,21 @@ import ( ) type OpenapiReader struct { + // OpenAPI spec to read schemas from. OpenapiSpec *openapi.Specification - Memo map[string]*jsonschema.Schema + + // In-memory cache of schemas read from the OpenAPI spec. + memo map[string]jsonschema.Schema } const SchemaPathPrefix = "#/components/schemas/" -func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, error) { +// Read a schema directly from the OpenAPI spec. +func (reader *OpenapiReader) readOpenapiSchema(path string) (jsonschema.Schema, error) { schemaKey := strings.TrimPrefix(path, SchemaPathPrefix) // return early if we already have a computed schema - memoSchema, ok := reader.Memo[schemaKey] + memoSchema, ok := reader.memo[schemaKey] if ok { return memoSchema, nil } @@ -28,18 +32,18 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, // check path is present in openapi spec openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey] if !ok { - return nil, fmt.Errorf("schema with path %s not found in openapi spec", path) + return jsonschema.Schema{}, fmt.Errorf("schema with path %s not found in openapi spec", path) } // convert openapi schema to the native schema struct bytes, err := json.Marshal(*openapiSchema) if err != nil { - return nil, err + return jsonschema.Schema{}, err } - jsonSchema := &jsonschema.Schema{} - err = json.Unmarshal(bytes, jsonSchema) + jsonSchema := jsonschema.Schema{} + err = json.Unmarshal(bytes, &jsonSchema) if err != nil { - return nil, err + return jsonschema.Schema{}, err } // A hack to convert a map[string]interface{} to *Schema @@ -49,23 +53,28 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, if ok { b, err := json.Marshal(jsonSchema.AdditionalProperties) if err != nil { - return nil, err + return jsonschema.Schema{}, err } additionalProperties := &jsonschema.Schema{} err = json.Unmarshal(b, additionalProperties) if err != nil { - return nil, err + return jsonschema.Schema{}, err } jsonSchema.AdditionalProperties = additionalProperties } // store read schema into memo - reader.Memo[schemaKey] = jsonSchema + reader.memo[schemaKey] = jsonSchema return jsonSchema, nil } -// safe againt loops in refs +// Resolve all nested "$ref" references in the schema. This function unrolls a single +// level of "$ref" in the schema and calls into traverseSchema to resolve nested references. +// Thus this function and traverseSchema are mutually recursive. +// +// This function is safe against reference loops. If a reference loop is detected, an error +// is returned. func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { if root.Reference == nil { return reader.traverseSchema(root, tracker) @@ -91,12 +100,12 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t // in the memo root.Reference = nil - // unroll one level of reference + // unroll one level of reference. selfRef, err := reader.readOpenapiSchema(ref) if err != nil { return nil, err } - root = selfRef + root = &selfRef root.Description = description // traverse again to find new references @@ -108,6 +117,8 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t return root, err } +// Traverse the nested properties of the schema to resolve "$ref" references. This function +// and safeResolveRefs are mutually recursive. func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) { // case primitive (or invalid) if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType { @@ -154,11 +165,11 @@ func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema } tracker := newTracker() tracker.push(path, path) - root, err = reader.safeResolveRefs(root, tracker) + resolvedRoot, err := reader.safeResolveRefs(&root, tracker) if err != nil { return nil, tracker.errWithTrace(err.Error(), "") } - return root, nil + return resolvedRoot, nil } func (reader *OpenapiReader) jobsDocs() (*Docs, error) { diff --git a/bundle/schema/openapi_test.go b/bundle/schema/openapi_test.go index 0d71fa440..359b1e58a 100644 --- a/bundle/schema/openapi_test.go +++ b/bundle/schema/openapi_test.go @@ -48,7 +48,7 @@ func TestReadSchemaForObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -106,7 +106,7 @@ func TestReadSchemaForArray(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -152,7 +152,7 @@ func TestReadSchemaForMap(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -201,7 +201,7 @@ func TestRootReferenceIsResolved(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -251,7 +251,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -285,7 +285,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -330,7 +330,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -400,7 +400,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { spec := &openapi.Specification{} reader := &OpenapiReader{ OpenapiSpec: spec, - Memo: make(map[string]*jsonschema.Schema), + memo: make(map[string]jsonschema.Schema), } err := json.Unmarshal([]byte(specString), spec) require.NoError(t, err) @@ -434,3 +434,61 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) { t.Log("[DEBUG] expected: ", expected) assert.Equal(t, expected, string(fruitsSchemaJson)) } + +func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) { + specString := `{ + "components": { + "schemas": { + "foo": { + "type": "number" + }, + "fruits": { + "type": "object", + "properties": { + "guava": { + "type": "object", + "description": "Guava is a fruit", + "$ref": "#/components/schemas/foo" + }, + "mango": { + "type": "object", + "description": "What is a mango?", + "$ref": "#/components/schemas/foo" + } + } + } + } + } + }` + spec := &openapi.Specification{} + reader := &OpenapiReader{ + OpenapiSpec: spec, + memo: make(map[string]jsonschema.Schema), + } + err := json.Unmarshal([]byte(specString), spec) + require.NoError(t, err) + + fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits") + require.NoError(t, err) + + fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ") + require.NoError(t, err) + + expected := `{ + "type": "object", + "properties": { + "guava": { + "type": "number", + "description": "Guava is a fruit" + }, + "mango": { + "type": "number", + "description": "What is a mango?" + } + } + }` + + t.Log("[DEBUG] actual: ", string(fruitsSchemaJson)) + t.Log("[DEBUG] expected: ", expected) + assert.Equal(t, expected, string(fruitsSchemaJson)) +} diff --git a/bundle/schema/schema.go b/bundle/schema/schema.go index b37f72d9b..ac0b4f2ec 100644 --- a/bundle/schema/schema.go +++ b/bundle/schema/schema.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/jsonschema" ) @@ -167,6 +168,22 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschem } jsonSchema := &jsonschema.Schema{Type: rootJavascriptType} + // If the type is a non-string primitive, then we allow it to be a string + // provided it's a pure variable reference (ie only a single variable reference). + if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType { + jsonSchema = &jsonschema.Schema{ + AnyOf: []*jsonschema.Schema{ + { + Type: rootJavascriptType, + }, + { + Type: jsonschema.StringType, + Pattern: dynvar.VariableRegex, + }, + }, + } + } + if docs != nil { jsonSchema.Description = docs.Description } diff --git a/bundle/schema/schema_test.go b/bundle/schema/schema_test.go index d44a2082a..ea4fd1020 100644 --- a/bundle/schema/schema_test.go +++ b/bundle/schema/schema_test.go @@ -14,7 +14,15 @@ func TestIntSchema(t *testing.T) { expected := `{ - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }` schema, err := New(reflect.TypeOf(elemInt), nil) @@ -33,7 +41,15 @@ func TestBooleanSchema(t *testing.T) { expected := `{ - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }` schema, err := New(reflect.TypeOf(elem), nil) @@ -101,46 +117,150 @@ func TestStructOfPrimitivesSchema(t *testing.T) { "type": "object", "properties": { "bool_val": { - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "float32_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "float64_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "int16_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "int32_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "int64_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "int8_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "int_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "string_val": { "type": "string" }, "uint16_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "uint32_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "uint64_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "uint8_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "uint_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -200,7 +320,15 @@ func TestStructOfStructsSchema(t *testing.T) { "type": "object", "properties": { "a": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "b": { "type": "string" @@ -257,7 +385,15 @@ func TestStructOfMapsSchema(t *testing.T) { "my_map": { "type": "object", "additionalProperties": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } } }, @@ -339,7 +475,15 @@ func TestMapOfPrimitivesSchema(t *testing.T) { `{ "type": "object", "additionalProperties": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }` @@ -368,7 +512,15 @@ func TestMapOfStructSchema(t *testing.T) { "type": "object", "properties": { "my_int": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -398,7 +550,15 @@ func TestMapOfMapSchema(t *testing.T) { "additionalProperties": { "type": "object", "additionalProperties": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } } }` @@ -495,7 +655,15 @@ func TestSliceOfMapSchema(t *testing.T) { "items": { "type": "object", "additionalProperties": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } } }` @@ -525,7 +693,15 @@ func TestSliceOfStructSchema(t *testing.T) { "type": "object", "properties": { "my_int": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -575,7 +751,15 @@ func TestEmbeddedStructSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "country": { "type": "string" @@ -607,7 +791,15 @@ func TestEmbeddedStructSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "home": { "type": "object", @@ -694,7 +886,15 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) { "type": "object", "properties": { "bar": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -728,7 +928,15 @@ func TestDashFieldsAreSkipped(t *testing.T) { "type": "object", "properties": { "bar": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -773,7 +981,15 @@ func TestPointerInStructSchema(t *testing.T) { "type": "object", "properties": { "ptr_val2": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -782,13 +998,29 @@ func TestPointerInStructSchema(t *testing.T) { ] }, "float_val": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "ptr_bar": { "type": "object", "properties": { "ptr_val2": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -797,7 +1029,15 @@ func TestPointerInStructSchema(t *testing.T) { ] }, "ptr_int": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "ptr_string": { "type": "string" @@ -860,7 +1100,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -875,7 +1123,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -895,7 +1151,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -910,7 +1174,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -932,7 +1204,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -950,7 +1230,15 @@ func TestGenericSchema(t *testing.T) { "type": "object", "properties": { "age": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "name": { "type": "string" @@ -1028,16 +1316,40 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) { "type": "object", "properties": { "apple": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "bar": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "papaya": { "type": "object", "properties": { "a": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "b": { "type": "string" @@ -1111,7 +1423,15 @@ func TestDocIngestionForObject(t *testing.T) { "description": "docs for a" }, "b": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -1185,12 +1505,28 @@ func TestDocIngestionForSlice(t *testing.T) { "type": "object", "properties": { "guava": { - "type": "number", - "description": "docs for guava" + "description": "docs for guava", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "pineapple": { - "type": "number", - "description": "docs for pineapple" + "description": "docs for pineapple", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -1268,12 +1604,28 @@ func TestDocIngestionForMap(t *testing.T) { "type": "object", "properties": { "apple": { - "type": "number", - "description": "docs for apple" + "description": "docs for apple", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "mango": { - "type": "number", - "description": "docs for mango" + "description": "docs for mango", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -1324,8 +1676,16 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) { "description": "docs for root", "properties": { "my_val": { - "type": "number", - "description": "docs for my val" + "description": "docs for my val", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] } }, "additionalProperties": false, @@ -1395,7 +1755,15 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) { "type": "object", "properties": { "apple": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "mango": {} }, @@ -1436,7 +1804,15 @@ func TestBundleReadOnlytag(t *testing.T) { "type": "object", "properties": { "apple": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "pokemon": { "type": "object", @@ -1488,7 +1864,15 @@ func TestBundleInternalTag(t *testing.T) { "type": "object", "properties": { "apple": { - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}" + } + ] }, "pokemon": { "type": "object", diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index 3e12ddb68..aed3964db 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -1,8 +1,11 @@ package config_tests import ( + "context" "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/libraries" "github.com/stretchr/testify/require" ) @@ -10,3 +13,11 @@ func TestEnvironmentKeySupported(t *testing.T) { _, diags := loadTargetWithDiags("./python_wheel/environment_key", "default") require.Empty(t, diags) } + +func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { + b, diags := loadTargetWithDiags("./environment_key_only", "default") + require.Empty(t, diags) + + diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist()) + require.Empty(t, diags) +} diff --git a/bundle/tests/environment_git_test.go b/bundle/tests/environment_git_test.go index bb10825e4..ad4aec2e6 100644 --- a/bundle/tests/environment_git_test.go +++ b/bundle/tests/environment_git_test.go @@ -1,6 +1,8 @@ package config_tests import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -9,12 +11,14 @@ import ( func TestGitAutoLoadWithEnvironment(t *testing.T) { b := load(t, "./environments_autoload_git") assert.True(t, b.Config.Bundle.Git.Inferred) - assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli") + validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") + assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) } func TestGitManuallySetBranchWithEnvironment(t *testing.T) { b := loadTarget(t, "./environments_autoload_git", "production") assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) - assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli") + validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") + assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) } diff --git a/bundle/tests/environment_key_only/databricks.yml b/bundle/tests/environment_key_only/databricks.yml new file mode 100644 index 000000000..caa34f8e3 --- /dev/null +++ b/bundle/tests/environment_key_only/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: environment_key_only + +resources: + jobs: + test_job: + name: "My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + environment_key: "test_env" + environments: + - environment_key: "test_env" diff --git a/bundle/tests/git_test.go b/bundle/tests/git_test.go index b33ffc211..21eaaedd2 100644 --- a/bundle/tests/git_test.go +++ b/bundle/tests/git_test.go @@ -2,6 +2,8 @@ package config_tests import ( "context" + "fmt" + "strings" "testing" "github.com/databricks/cli/bundle" @@ -13,14 +15,16 @@ import ( func TestGitAutoLoad(t *testing.T) { b := load(t, "./autoload_git") assert.True(t, b.Config.Bundle.Git.Inferred) - assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli") + validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") + assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) } func TestGitManuallySetBranch(t *testing.T) { b := loadTarget(t, "./autoload_git", "production") assert.False(t, b.Config.Bundle.Git.Inferred) assert.Equal(t, "main", b.Config.Bundle.Git.Branch) - assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli") + validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks") + assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL)) } func TestGitBundleBranchValidation(t *testing.T) { diff --git a/bundle/tests/include_multiple/my_first_job/resource.yml b/bundle/tests/include_multiple/my_first_job/resource.yml index c2be5a160..4bd7c7164 100644 --- a/bundle/tests/include_multiple/my_first_job/resource.yml +++ b/bundle/tests/include_multiple/my_first_job/resource.yml @@ -2,3 +2,4 @@ resources: jobs: my_first_job: id: 1 + name: "My First Job" diff --git a/bundle/tests/include_multiple/my_second_job/resource.yml b/bundle/tests/include_multiple/my_second_job/resource.yml index 2c28c4622..3a1514055 100644 --- a/bundle/tests/include_multiple/my_second_job/resource.yml +++ b/bundle/tests/include_multiple/my_second_job/resource.yml @@ -2,3 +2,4 @@ resources: jobs: my_second_job: id: 2 + name: "My Second Job" diff --git a/bundle/tests/include_with_glob/job.yml b/bundle/tests/include_with_glob/job.yml index 3d609c529..a98577818 100644 --- a/bundle/tests/include_with_glob/job.yml +++ b/bundle/tests/include_with_glob/job.yml @@ -2,3 +2,4 @@ resources: jobs: my_job: id: 1 + name: "My Job" diff --git a/bundle/tests/quality_monitor/databricks.yml b/bundle/tests/quality_monitor/databricks.yml new file mode 100644 index 000000000..3abcdfdda --- /dev/null +++ b/bundle/tests/quality_monitor/databricks.yml @@ -0,0 +1,40 @@ +resources: + quality_monitors: + my_monitor: + table_name: "main.test.thing1" + assets_dir: "/Shared/provider-test/databricks_monitoring/main.test.thing1" + output_schema_name: "test" + inference_log: + granularities: ["1 day"] + timestamp_col: "timestamp" + prediction_col: "prediction" + model_id_col: "model_id" + problem_type: "PROBLEM_TYPE_REGRESSION" + +targets: + development: + mode: development + resources: + quality_monitors: + my_monitor: + table_name: "main.test.dev" + + staging: + resources: + quality_monitors: + my_monitor: + table_name: "main.test.staging" + output_schema_name: "staging" + + production: + resources: + quality_monitors: + my_monitor: + table_name: "main.test.prod" + output_schema_name: "prod" + inference_log: + granularities: ["1 hour"] + timestamp_col: "timestamp_prod" + prediction_col: "prediction_prod" + model_id_col: "model_id_prod" + problem_type: "PROBLEM_TYPE_REGRESSION" diff --git a/bundle/tests/quality_monitor_test.go b/bundle/tests/quality_monitor_test.go new file mode 100644 index 000000000..d5db05196 --- /dev/null +++ b/bundle/tests/quality_monitor_test.go @@ -0,0 +1,59 @@ +package config_tests + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/stretchr/testify/assert" +) + +func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) { + assert.Equal(t, "timestamp", p.InferenceLog.TimestampCol) + assert.Equal(t, "prediction", p.InferenceLog.PredictionCol) + assert.Equal(t, "model_id", p.InferenceLog.ModelIdCol) + assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType) +} + +func TestMonitorTableNames(t *testing.T) { + b := loadTarget(t, "./quality_monitor", "development") + assert.Len(t, b.Config.Resources.QualityMonitors, 1) + assert.Equal(t, b.Config.Bundle.Mode, config.Development) + + p := b.Config.Resources.QualityMonitors["my_monitor"] + assert.Equal(t, "main.test.dev", p.TableName) + assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) + assert.Equal(t, "test", p.OutputSchemaName) + + assertExpectedMonitor(t, p) +} + +func TestMonitorStaging(t *testing.T) { + b := loadTarget(t, "./quality_monitor", "staging") + assert.Len(t, b.Config.Resources.QualityMonitors, 1) + + p := b.Config.Resources.QualityMonitors["my_monitor"] + assert.Equal(t, "main.test.staging", p.TableName) + assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) + assert.Equal(t, "staging", p.OutputSchemaName) + + assertExpectedMonitor(t, p) +} + +func TestMonitorProduction(t *testing.T) { + b := loadTarget(t, "./quality_monitor", "production") + assert.Len(t, b.Config.Resources.QualityMonitors, 1) + + p := b.Config.Resources.QualityMonitors["my_monitor"] + assert.Equal(t, "main.test.prod", p.TableName) + assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir) + assert.Equal(t, "prod", p.OutputSchemaName) + + inferenceLog := p.InferenceLog + assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities) + assert.Equal(t, "timestamp_prod", p.InferenceLog.TimestampCol) + assert.Equal(t, "prediction_prod", p.InferenceLog.PredictionCol) + assert.Equal(t, "model_id_prod", p.InferenceLog.ModelIdCol) + assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType) +} diff --git a/bundle/tests/undefined_job/databricks.yml b/bundle/tests/undefined_job/databricks.yml new file mode 100644 index 000000000..12c19f946 --- /dev/null +++ b/bundle/tests/undefined_job/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: undefined-job + +resources: + jobs: + undefined: + test: + name: "Test Job" diff --git a/bundle/tests/undefined_job_test.go b/bundle/tests/undefined_job_test.go new file mode 100644 index 000000000..ed502c471 --- /dev/null +++ b/bundle/tests/undefined_job_test.go @@ -0,0 +1,12 @@ +package config_tests + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUndefinedJobLoadsWithError(t *testing.T) { + _, diags := loadTargetWithDiags("./undefined_job", "default") + assert.ErrorContains(t, diags.Error(), "job undefined is not defined") +} diff --git a/bundle/tests/variables/variable_overrides_in_target/databricks.yml b/bundle/tests/variables/variable_overrides_in_target/databricks.yml new file mode 100644 index 000000000..4e52b5073 --- /dev/null +++ b/bundle/tests/variables/variable_overrides_in_target/databricks.yml @@ -0,0 +1,41 @@ +bundle: + name: foobar + +resources: + pipelines: + my_pipeline: + name: ${var.foo} + continuous: ${var.baz} + clusters: + - num_workers: ${var.bar} + + + +variables: + foo: + default: "a_string" + description: "A string variable" + + bar: + default: 42 + description: "An integer variable" + + baz: + default: true + description: "A boolean variable" + +targets: + use-default-variable-values: + + override-string-variable: + variables: + foo: "overridden_string" + + override-int-variable: + variables: + bar: 43 + + override-both-bool-and-string-variables: + variables: + foo: "overridden_string" + baz: false diff --git a/bundle/tests/variables_test.go b/bundle/tests/variables_test.go index fde36344f..f51802684 100644 --- a/bundle/tests/variables_test.go +++ b/bundle/tests/variables_test.go @@ -120,3 +120,52 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) { assert.Equal(t, "cluster: some-test-cluster", b.Config.Variables["d"].Lookup.String()) assert.Equal(t, "instance-pool: some-test-instance-pool", b.Config.Variables["e"].Lookup.String()) } + +func TestVariableTargetOverrides(t *testing.T) { + var tcases = []struct { + targetName string + pipelineName string + pipelineContinuous bool + pipelineNumWorkers int + }{ + { + "use-default-variable-values", + "a_string", + true, + 42, + }, + { + "override-string-variable", + "overridden_string", + true, + 42, + }, + { + "override-int-variable", + "a_string", + true, + 43, + }, + { + "override-both-bool-and-string-variables", + "overridden_string", + false, + 42, + }, + } + + for _, tcase := range tcases { + t.Run(tcase.targetName, func(t *testing.T) { + b := loadTarget(t, "./variables/variable_overrides_in_target", tcase.targetName) + diags := bundle.Apply(context.Background(), b, bundle.Seq( + mutator.SetVariables(), + mutator.ResolveVariableReferences("variables")), + ) + require.NoError(t, diags.Error()) + + assert.Equal(t, tcase.pipelineName, b.Config.Resources.Pipelines["my_pipeline"].Name) + assert.Equal(t, tcase.pipelineContinuous, b.Config.Resources.Pipelines["my_pipeline"].Continuous) + assert.Equal(t, tcase.pipelineNumWorkers, b.Config.Resources.Pipelines["my_pipeline"].Clusters[0].NumWorkers) + }) + } +} diff --git a/cmd/account/csp-enablement-account/csp-enablement-account.go b/cmd/account/csp-enablement-account/csp-enablement-account.go index 79819003b..d6fce9537 100755 --- a/cmd/account/csp-enablement-account/csp-enablement-account.go +++ b/cmd/account/csp-enablement-account/csp-enablement-account.go @@ -156,4 +156,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service CSPEnablementAccount +// end service CspEnablementAccount diff --git a/cmd/account/esm-enablement-account/esm-enablement-account.go b/cmd/account/esm-enablement-account/esm-enablement-account.go index dd407e2e5..71149e5ad 100755 --- a/cmd/account/esm-enablement-account/esm-enablement-account.go +++ b/cmd/account/esm-enablement-account/esm-enablement-account.go @@ -154,4 +154,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service ESMEnablementAccount +// end service EsmEnablementAccount diff --git a/cmd/auth/env.go b/cmd/auth/env.go index 04aef36a8..e72d15399 100644 --- a/cmd/auth/env.go +++ b/cmd/auth/env.go @@ -10,7 +10,7 @@ import ( "net/url" "strings" - "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" "gopkg.in/ini.v1" @@ -70,7 +70,7 @@ func resolveSection(cfg *config.Config, iniFile *config.File) (*ini.Section, err } func loadFromDatabricksCfg(ctx context.Context, cfg *config.Config) error { - iniFile, err := databrickscfg.Get(ctx) + iniFile, err := profile.DefaultProfiler.Get(ctx) if errors.Is(err, fs.ErrNotExist) { // it's fine not to have ~/.databrickscfg return nil diff --git a/cmd/auth/login.go b/cmd/auth/login.go index c033054b8..11cba8e5f 100644 --- a/cmd/auth/login.go +++ b/cmd/auth/login.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/cfgpickers" + "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/spf13/cobra" @@ -31,6 +32,7 @@ func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, arg } const minimalDbConnectVersion = "13.1" +const defaultTimeout = 1 * time.Hour func newLoginCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { defaultConfigPath := "~/.databrickscfg" @@ -84,7 +86,7 @@ depends on the existing profiles you have set in your configuration file var loginTimeout time.Duration var configureCluster bool - cmd.Flags().DurationVar(&loginTimeout, "timeout", auth.DefaultTimeout, + cmd.Flags().DurationVar(&loginTimeout, "timeout", defaultTimeout, "Timeout for completing login challenge in the browser") cmd.Flags().BoolVar(&configureCluster, "configure-cluster", false, "Prompts to configure cluster") @@ -108,7 +110,7 @@ depends on the existing profiles you have set in your configuration file profileName = profile } - err := setHost(ctx, profileName, persistentAuth, args) + err := setHostAndAccountId(ctx, profileName, persistentAuth, args) if err != nil { return err } @@ -117,17 +119,10 @@ depends on the existing profiles you have set in your configuration file // We need the config without the profile before it's used to initialise new workspace client below. // Otherwise it will complain about non existing profile because it was not yet saved. cfg := config.Config{ - Host: persistentAuth.Host, - AuthType: "databricks-cli", + Host: persistentAuth.Host, + AccountID: persistentAuth.AccountID, + AuthType: "databricks-cli", } - if cfg.IsAccountClient() && persistentAuth.AccountID == "" { - accountId, err := promptForAccountID(ctx) - if err != nil { - return err - } - persistentAuth.AccountID = accountId - } - cfg.AccountID = persistentAuth.AccountID ctx, cancel := context.WithTimeout(ctx, loginTimeout) defer cancel() @@ -172,15 +167,15 @@ depends on the existing profiles you have set in your configuration file return cmd } -func setHost(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error { +func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error { + profiler := profile.GetProfiler(ctx) // If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile. - _, profiles, err := databrickscfg.LoadProfiles(ctx, func(p databrickscfg.Profile) bool { - return p.Name == profileName - }) + profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName)) // Tolerate ErrNoConfiguration here, as we will write out a configuration as part of the login flow. - if err != nil && !errors.Is(err, databrickscfg.ErrNoConfiguration) { + if err != nil && !errors.Is(err, profile.ErrNoConfiguration) { return err } + if persistentAuth.Host == "" { if len(profiles) > 0 && profiles[0].Host != "" { persistentAuth.Host = profiles[0].Host @@ -188,5 +183,17 @@ func setHost(ctx context.Context, profileName string, persistentAuth *auth.Persi configureHost(ctx, persistentAuth, args, 0) } } + isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient() + if isAccountClient && persistentAuth.AccountID == "" { + if len(profiles) > 0 && profiles[0].AccountID != "" { + persistentAuth.AccountID = profiles[0].AccountID + } else { + accountId, err := promptForAccountID(ctx) + if err != nil { + return err + } + persistentAuth.AccountID = accountId + } + } return nil } diff --git a/cmd/auth/login_test.go b/cmd/auth/login_test.go index 9b834bd0a..ce3ca5ae5 100644 --- a/cmd/auth/login_test.go +++ b/cmd/auth/login_test.go @@ -12,6 +12,6 @@ import ( func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) { ctx := context.Background() ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./imaginary-file/databrickscfg") - err := setHost(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{}) + err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{}) assert.NoError(t, err) } diff --git a/cmd/auth/profiles.go b/cmd/auth/profiles.go index 797eb3b5f..61a6c1f33 100644 --- a/cmd/auth/profiles.go +++ b/cmd/auth/profiles.go @@ -8,7 +8,7 @@ import ( "time" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -94,7 +94,7 @@ func newProfilesCommand() *cobra.Command { cmd.RunE = func(cmd *cobra.Command, args []string) error { var profiles []*profileMetadata - iniFile, err := databrickscfg.Get(cmd.Context()) + iniFile, err := profile.DefaultProfiler.Get(cmd.Context()) if os.IsNotExist(err) { // return empty list for non-configured machines iniFile = &config.File{ diff --git a/cmd/auth/token.go b/cmd/auth/token.go index d763b9564..3f9af43fa 100644 --- a/cmd/auth/token.go +++ b/cmd/auth/token.go @@ -4,12 +4,44 @@ import ( "context" "encoding/json" "errors" + "fmt" + "os" + "strings" "time" "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go/httpclient" "github.com/spf13/cobra" ) +type tokenErrorResponse struct { + Error string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +func buildLoginCommand(profile string, persistentAuth *auth.PersistentAuth) string { + executable := os.Args[0] + cmd := []string{ + executable, + "auth", + "login", + } + if profile != "" { + cmd = append(cmd, "--profile", profile) + } else { + cmd = append(cmd, "--host", persistentAuth.Host) + if persistentAuth.AccountID != "" { + cmd = append(cmd, "--account-id", persistentAuth.AccountID) + } + } + return strings.Join(cmd, " ") +} + +func helpfulError(profile string, persistentAuth *auth.PersistentAuth) string { + loginMsg := buildLoginCommand(profile, persistentAuth) + return fmt.Sprintf("Try logging in again with `%s` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new", loginMsg) +} + func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { cmd := &cobra.Command{ Use: "token [HOST]", @@ -17,7 +49,7 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { } var tokenTimeout time.Duration - cmd.Flags().DurationVar(&tokenTimeout, "timeout", auth.DefaultTimeout, + cmd.Flags().DurationVar(&tokenTimeout, "timeout", defaultTimeout, "Timeout for acquiring a token.") cmd.RunE = func(cmd *cobra.Command, args []string) error { @@ -29,11 +61,11 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { profileName = profileFlag.Value.String() // If a profile is provided we read the host from the .databrickscfg file if profileName != "" && len(args) > 0 { - return errors.New("providing both a profile and a host parameters is not supported") + return errors.New("providing both a profile and host is not supported") } } - err := setHost(ctx, profileName, persistentAuth, args) + err := setHostAndAccountId(ctx, profileName, persistentAuth, args) if err != nil { return err } @@ -42,8 +74,21 @@ func newTokenCommand(persistentAuth *auth.PersistentAuth) *cobra.Command { ctx, cancel := context.WithTimeout(ctx, tokenTimeout) defer cancel() t, err := persistentAuth.Load(ctx) - if err != nil { - return err + var httpErr *httpclient.HttpError + if errors.As(err, &httpErr) { + helpMsg := helpfulError(profileName, persistentAuth) + t := &tokenErrorResponse{} + err = json.Unmarshal([]byte(httpErr.Message), t) + if err != nil { + return fmt.Errorf("unexpected parsing token response: %w. %s", err, helpMsg) + } + if t.ErrorDescription == "Refresh token is invalid" { + return fmt.Errorf("a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run `%s`", buildLoginCommand(profileName, persistentAuth)) + } else { + return fmt.Errorf("unexpected error refreshing token: %s. %s", t.ErrorDescription, helpMsg) + } + } else if err != nil { + return fmt.Errorf("unexpected error refreshing token: %w. %s", err, helpfulError(profileName, persistentAuth)) } raw, err := json.MarshalIndent(t, "", " ") if err != nil { diff --git a/cmd/auth/token_test.go b/cmd/auth/token_test.go new file mode 100644 index 000000000..df98cc151 --- /dev/null +++ b/cmd/auth/token_test.go @@ -0,0 +1,168 @@ +package auth_test + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/databricks/cli/cmd" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/auth/cache" + "github.com/databricks/cli/libs/databrickscfg/profile" + "github.com/databricks/databricks-sdk-go/httpclient" + "github.com/databricks/databricks-sdk-go/httpclient/fixtures" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" +) + +var refreshFailureTokenResponse = fixtures.HTTPFixture{ + MatchAny: true, + Status: 401, + Response: map[string]string{ + "error": "invalid_request", + "error_description": "Refresh token is invalid", + }, +} + +var refreshFailureInvalidResponse = fixtures.HTTPFixture{ + MatchAny: true, + Status: 401, + Response: "Not json", +} + +var refreshFailureOtherError = fixtures.HTTPFixture{ + MatchAny: true, + Status: 401, + Response: map[string]string{ + "error": "other_error", + "error_description": "Databricks is down", + }, +} + +var refreshSuccessTokenResponse = fixtures.HTTPFixture{ + MatchAny: true, + Status: 200, + Response: map[string]string{ + "access_token": "new-access-token", + "token_type": "Bearer", + "expires_in": "3600", + }, +} + +func validateToken(t *testing.T, resp string) { + res := map[string]string{} + err := json.Unmarshal([]byte(resp), &res) + assert.NoError(t, err) + assert.Equal(t, "new-access-token", res["access_token"]) + assert.Equal(t, "Bearer", res["token_type"]) +} + +func getContextForTest(f fixtures.HTTPFixture) context.Context { + profiler := profile.InMemoryProfiler{ + Profiles: profile.Profiles{ + { + Name: "expired", + Host: "https://accounts.cloud.databricks.com", + AccountID: "expired", + }, + { + Name: "active", + Host: "https://accounts.cloud.databricks.com", + AccountID: "active", + }, + }, + } + tokenCache := &cache.InMemoryTokenCache{ + Tokens: map[string]*oauth2.Token{ + "https://accounts.cloud.databricks.com/oidc/accounts/expired": { + RefreshToken: "expired", + }, + "https://accounts.cloud.databricks.com/oidc/accounts/active": { + RefreshToken: "active", + Expiry: time.Now().Add(1 * time.Hour), // Hopefully unit tests don't take an hour to run + }, + }, + } + client := httpclient.NewApiClient(httpclient.ClientConfig{ + Transport: fixtures.SliceTransport{f}, + }) + ctx := profile.WithProfiler(context.Background(), profiler) + ctx = cache.WithTokenCache(ctx, tokenCache) + ctx = auth.WithApiClientForOAuth(ctx, client) + return ctx +} + +func getCobraCmdForTest(f fixtures.HTTPFixture) (*cobra.Command, *bytes.Buffer) { + ctx := getContextForTest(f) + c := cmd.New(ctx) + output := &bytes.Buffer{} + c.SetOut(output) + return c, output +} + +func TestTokenCmdWithProfilePrintsHelpfulLoginMessageOnRefreshFailure(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshFailureTokenResponse) + cmd.SetArgs([]string{"auth", "token", "--profile", "expired"}) + err := cmd.Execute() + + out := output.String() + assert.Empty(t, out) + assert.ErrorContains(t, err, "a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run ") + assert.ErrorContains(t, err, "auth login --profile expired") +} + +func TestTokenCmdWithHostPrintsHelpfulLoginMessageOnRefreshFailure(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshFailureTokenResponse) + cmd.SetArgs([]string{"auth", "token", "--host", "https://accounts.cloud.databricks.com", "--account-id", "expired"}) + err := cmd.Execute() + + out := output.String() + assert.Empty(t, out) + assert.ErrorContains(t, err, "a new access token could not be retrieved because the refresh token is invalid. To reauthenticate, run ") + assert.ErrorContains(t, err, "auth login --host https://accounts.cloud.databricks.com --account-id expired") +} + +func TestTokenCmdInvalidResponse(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshFailureInvalidResponse) + cmd.SetArgs([]string{"auth", "token", "--profile", "active"}) + err := cmd.Execute() + + out := output.String() + assert.Empty(t, out) + assert.ErrorContains(t, err, "unexpected parsing token response: invalid character 'N' looking for beginning of value. Try logging in again with ") + assert.ErrorContains(t, err, "auth login --profile active` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new") +} + +func TestTokenCmdOtherErrorResponse(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshFailureOtherError) + cmd.SetArgs([]string{"auth", "token", "--profile", "active"}) + err := cmd.Execute() + + out := output.String() + assert.Empty(t, out) + assert.ErrorContains(t, err, "unexpected error refreshing token: Databricks is down. Try logging in again with ") + assert.ErrorContains(t, err, "auth login --profile active` before retrying. If this fails, please report this issue to the Databricks CLI maintainers at https://github.com/databricks/cli/issues/new") +} + +func TestTokenCmdWithProfileSuccess(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshSuccessTokenResponse) + cmd.SetArgs([]string{"auth", "token", "--profile", "active"}) + err := cmd.Execute() + + out := output.String() + validateToken(t, out) + assert.NoError(t, err) +} + +func TestTokenCmdWithHostSuccess(t *testing.T) { + cmd, output := getCobraCmdForTest(refreshSuccessTokenResponse) + cmd.SetArgs([]string{"auth", "token", "--host", "https://accounts.cloud.databricks.com", "--account-id", "expired"}) + err := cmd.Execute() + + out := output.String() + validateToken(t, out) + assert.NoError(t, err) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 1db60d585..0880c9c44 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -9,7 +9,7 @@ func New() *cobra.Command { cmd := &cobra.Command{ Use: "bundle", Short: "Databricks Asset Bundles let you express data/AI/analytics projects as code.", - Long: "Databricks Asset Bundles let you express data/AI/analytics projects as code.\n\nOnline documentation: https://docs.databricks.com/en/dev-tools/bundles", + Long: "Databricks Asset Bundles let you express data/AI/analytics projects as code.\n\nOnline documentation: https://docs.databricks.com/en/dev-tools/bundles/index.html", GroupID: "development", } diff --git a/cmd/bundle/schema.go b/cmd/bundle/schema.go index 0f27142bd..b0d6b3dd5 100644 --- a/cmd/bundle/schema.go +++ b/cmd/bundle/schema.go @@ -7,9 +7,58 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/schema" "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/jsonschema" "github.com/spf13/cobra" ) +func overrideVariables(s *jsonschema.Schema) error { + // Override schema for default values to allow for multiple primitive types. + // These are normalized to strings when converted to the typed representation. + err := s.SetByPath("variables.*.default", jsonschema.Schema{ + AnyOf: []*jsonschema.Schema{ + { + Type: jsonschema.StringType, + }, + { + Type: jsonschema.BooleanType, + }, + { + Type: jsonschema.NumberType, + }, + { + Type: jsonschema.IntegerType, + }, + }, + }) + if err != nil { + return err + } + + // Override schema for variables in targets to allow just specifying the value + // along side overriding the variable definition if needed. + ns, err := s.GetByPath("variables.*") + if err != nil { + return err + } + return s.SetByPath("targets.*.variables.*", jsonschema.Schema{ + AnyOf: []*jsonschema.Schema{ + { + Type: jsonschema.StringType, + }, + { + Type: jsonschema.BooleanType, + }, + { + Type: jsonschema.NumberType, + }, + { + Type: jsonschema.IntegerType, + }, + &ns, + }, + }) +} + func newSchemaCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schema", @@ -30,6 +79,13 @@ func newSchemaCommand() *cobra.Command { return err } + // Override schema for variables to take into account normalization of default + // variable values and variable overrides in a target. + err = overrideVariables(schema) + if err != nil { + return err + } + // Print the JSON schema to stdout. result, err := json.MarshalIndent(schema, "", " ") if err != nil { diff --git a/cmd/labs/project/installer.go b/cmd/labs/project/installer.go index 42c4a8496..92dfe9e7c 100644 --- a/cmd/labs/project/installer.go +++ b/cmd/labs/project/installer.go @@ -11,8 +11,8 @@ import ( "github.com/databricks/cli/cmd/labs/github" "github.com/databricks/cli/cmd/labs/unpack" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/cfgpickers" + "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/process" "github.com/databricks/cli/libs/python" @@ -89,7 +89,7 @@ func (i *installer) Install(ctx context.Context) error { return err } w, err := i.login(ctx) - if err != nil && errors.Is(err, databrickscfg.ErrNoConfiguration) { + if err != nil && errors.Is(err, profile.ErrNoConfiguration) { cfg, err := i.Installer.envAwareConfig(ctx) if err != nil { return err diff --git a/cmd/root/auth.go b/cmd/root/auth.go index 387b67f0d..107679105 100644 --- a/cmd/root/auth.go +++ b/cmd/root/auth.go @@ -7,7 +7,7 @@ import ( "net/http" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/databrickscfg/profile" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" "github.com/manifoldco/promptui" @@ -37,7 +37,7 @@ func (e ErrNoAccountProfiles) Error() string { func initProfileFlag(cmd *cobra.Command) { cmd.PersistentFlags().StringP("profile", "p", "", "~/.databrickscfg profile") - cmd.RegisterFlagCompletionFunc("profile", databrickscfg.ProfileCompletion) + cmd.RegisterFlagCompletionFunc("profile", profile.ProfileCompletion) } func profileFlagValue(cmd *cobra.Command) (string, bool) { @@ -111,27 +111,29 @@ func MustAccountClient(cmd *cobra.Command, args []string) error { cfg := &config.Config{} // The command-line profile flag takes precedence over DATABRICKS_CONFIG_PROFILE. - profile, hasProfileFlag := profileFlagValue(cmd) + pr, hasProfileFlag := profileFlagValue(cmd) if hasProfileFlag { - cfg.Profile = profile + cfg.Profile = pr } ctx := cmd.Context() ctx = context.WithValue(ctx, &configUsed, cfg) cmd.SetContext(ctx) + profiler := profile.GetProfiler(ctx) + if cfg.Profile == "" { // account-level CLI was not really done before, so here are the assumptions: // 1. only admins will have account configured // 2. 99% of admins will have access to just one account // hence, we don't need to create a special "DEFAULT_ACCOUNT" profile yet - _, profiles, err := databrickscfg.LoadProfiles(cmd.Context(), databrickscfg.MatchAccountProfiles) + profiles, err := profiler.LoadProfiles(cmd.Context(), profile.MatchAccountProfiles) if err == nil && len(profiles) == 1 { cfg.Profile = profiles[0].Name } // if there is no config file, we don't want to fail and instead just skip it - if err != nil && !errors.Is(err, databrickscfg.ErrNoConfiguration) { + if err != nil && !errors.Is(err, profile.ErrNoConfiguration) { return err } } @@ -233,11 +235,12 @@ func SetAccountClient(ctx context.Context, a *databricks.AccountClient) context. } func AskForWorkspaceProfile(ctx context.Context) (string, error) { - path, err := databrickscfg.GetPath(ctx) + profiler := profile.GetProfiler(ctx) + path, err := profiler.GetPath(ctx) if err != nil { return "", fmt.Errorf("cannot determine Databricks config file path: %w", err) } - file, profiles, err := databrickscfg.LoadProfiles(ctx, databrickscfg.MatchWorkspaceProfiles) + profiles, err := profiler.LoadProfiles(ctx, profile.MatchWorkspaceProfiles) if err != nil { return "", err } @@ -248,7 +251,7 @@ func AskForWorkspaceProfile(ctx context.Context) (string, error) { return profiles[0].Name, nil } i, _, err := cmdio.RunSelect(ctx, &promptui.Select{ - Label: fmt.Sprintf("Workspace profiles defined in %s", file), + Label: fmt.Sprintf("Workspace profiles defined in %s", path), Items: profiles, Searcher: profiles.SearchCaseInsensitive, StartInSearchMode: true, @@ -266,11 +269,12 @@ func AskForWorkspaceProfile(ctx context.Context) (string, error) { } func AskForAccountProfile(ctx context.Context) (string, error) { - path, err := databrickscfg.GetPath(ctx) + profiler := profile.GetProfiler(ctx) + path, err := profiler.GetPath(ctx) if err != nil { return "", fmt.Errorf("cannot determine Databricks config file path: %w", err) } - file, profiles, err := databrickscfg.LoadProfiles(ctx, databrickscfg.MatchAccountProfiles) + profiles, err := profiler.LoadProfiles(ctx, profile.MatchAccountProfiles) if err != nil { return "", err } @@ -281,7 +285,7 @@ func AskForAccountProfile(ctx context.Context) (string, error) { return profiles[0].Name, nil } i, _, err := cmdio.RunSelect(ctx, &promptui.Select{ - Label: fmt.Sprintf("Account profiles defined in %s", file), + Label: fmt.Sprintf("Account profiles defined in %s", path), Items: profiles, Searcher: profiles.SearchCaseInsensitive, StartInSearchMode: true, diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 42550722b..e5f1bfc9e 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/sync" + "github.com/databricks/cli/libs/vfs" "github.com/spf13/cobra" ) @@ -46,7 +47,7 @@ func (f *syncFlags) syncOptionsFromArgs(cmd *cobra.Command, args []string) (*syn } opts := sync.SyncOptions{ - LocalPath: args[0], + LocalPath: vfs.MustNew(args[0]), RemotePath: args[1], Full: f.full, PollInterval: f.interval, diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 026d840f7..b741e7b16 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -31,7 +31,7 @@ func TestSyncOptionsFromBundle(t *testing.T) { f := syncFlags{} opts, err := f.syncOptionsFromBundle(New(), []string{}, b) require.NoError(t, err) - assert.Equal(t, tempDir, opts.LocalPath) + assert.Equal(t, tempDir, opts.LocalPath.Native()) assert.Equal(t, "/Users/jane@doe.com/path", opts.RemotePath) assert.Equal(t, filepath.Join(tempDir, ".databricks", "bundle", "default"), opts.SnapshotBasePath) assert.NotNil(t, opts.WorkspaceClient) @@ -49,11 +49,14 @@ func TestSyncOptionsFromArgsRequiredTwoArgs(t *testing.T) { } func TestSyncOptionsFromArgs(t *testing.T) { + local := t.TempDir() + remote := "/remote" + f := syncFlags{} cmd := New() cmd.SetContext(root.SetWorkspaceClient(context.Background(), nil)) - opts, err := f.syncOptionsFromArgs(cmd, []string{"/local", "/remote"}) + opts, err := f.syncOptionsFromArgs(cmd, []string{local, remote}) require.NoError(t, err) - assert.Equal(t, "/local", opts.LocalPath) - assert.Equal(t, "/remote", opts.RemotePath) + assert.Equal(t, local, opts.LocalPath.Native()) + assert.Equal(t, remote, opts.RemotePath) } diff --git a/cmd/workspace/apps/apps.go b/cmd/workspace/apps/apps.go index 1ea50e830..1d6de4775 100755 --- a/cmd/workspace/apps/apps.go +++ b/cmd/workspace/apps/apps.go @@ -4,6 +4,7 @@ package apps import ( "fmt" + "time" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" @@ -19,10 +20,10 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ Use: "apps", - Short: `Lakehouse Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`, - Long: `Lakehouse Apps run directly on a customer’s Databricks instance, integrate - with their data, use and extend Databricks services, and enable users to - interact through single sign-on.`, + Short: `Apps run directly on a customer’s Databricks instance, integrate with their data, use and extend Databricks services, and enable users to interact through single sign-on.`, + Long: `Apps run directly on a customer’s Databricks instance, integrate with their + data, use and extend Databricks services, and enable users to interact through + single sign-on.`, GroupID: "serving", Annotations: map[string]string{ "package": "serving", @@ -34,11 +35,15 @@ func New() *cobra.Command { // Add methods cmd.AddCommand(newCreate()) - cmd.AddCommand(newDeleteApp()) - cmd.AddCommand(newGetApp()) - cmd.AddCommand(newGetAppDeploymentStatus()) - cmd.AddCommand(newGetApps()) - cmd.AddCommand(newGetEvents()) + cmd.AddCommand(newCreateDeployment()) + cmd.AddCommand(newDelete()) + cmd.AddCommand(newGet()) + cmd.AddCommand(newGetDeployment()) + cmd.AddCommand(newGetEnvironment()) + cmd.AddCommand(newList()) + cmd.AddCommand(newListDeployments()) + cmd.AddCommand(newStop()) + cmd.AddCommand(newUpdate()) // Apply optional overrides to this command. for _, fn := range cmdOverrides { @@ -54,28 +59,50 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var createOverrides []func( *cobra.Command, - *serving.DeployAppRequest, + *serving.CreateAppRequest, ) func newCreate() *cobra.Command { cmd := &cobra.Command{} - var createReq serving.DeployAppRequest + var createReq serving.CreateAppRequest var createJson flags.JsonFlag + var createSkipWait bool + var createTimeout time.Duration + + cmd.Flags().BoolVar(&createSkipWait, "no-wait", createSkipWait, `do not wait to reach IDLE state`) + cmd.Flags().DurationVar(&createTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach IDLE state`) // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - // TODO: any: resources + cmd.Flags().StringVar(&createReq.Description, "description", createReq.Description, `The description of the app.`) - cmd.Use = "create" - cmd.Short = `Create and deploy an application.` - cmd.Long = `Create and deploy an application. + cmd.Use = "create NAME" + cmd.Short = `Create an App.` + cmd.Long = `Create an App. - Creates and deploys an application.` + Creates a new app. + + Arguments: + NAME: The name of the app. The name must contain only lowercase alphanumeric + characters and hyphens and be between 2 and 30 characters long. It must be + unique within the workspace.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(0)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, no positional arguments are required. Provide 'name' in your JSON input") + } + return nil + } + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() @@ -86,15 +113,35 @@ func newCreate() *cobra.Command { if err != nil { return err } - } else { - return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") + } + if !cmd.Flags().Changed("json") { + createReq.Name = args[0] } - response, err := w.Apps.Create(ctx, createReq) + wait, err := w.Apps.Create(ctx, createReq) if err != nil { return err } - return cmdio.Render(ctx, response) + if createSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *serving.App) { + if i.Status == nil { + return + } + status := i.Status.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.Status != nil { + statusMessage = i.Status.Message + } + spinner <- statusMessage + }).GetWithTimeout(createTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) } // Disable completions since they are not applicable. @@ -109,30 +156,131 @@ func newCreate() *cobra.Command { return cmd } -// start delete-app command +// start create-deployment command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var deleteAppOverrides []func( +var createDeploymentOverrides []func( + *cobra.Command, + *serving.CreateAppDeploymentRequest, +) + +func newCreateDeployment() *cobra.Command { + cmd := &cobra.Command{} + + var createDeploymentReq serving.CreateAppDeploymentRequest + var createDeploymentJson flags.JsonFlag + + var createDeploymentSkipWait bool + var createDeploymentTimeout time.Duration + + cmd.Flags().BoolVar(&createDeploymentSkipWait, "no-wait", createDeploymentSkipWait, `do not wait to reach SUCCEEDED state`) + cmd.Flags().DurationVar(&createDeploymentTimeout, "timeout", 20*time.Minute, `maximum amount of time to reach SUCCEEDED state`) + // TODO: short flags + cmd.Flags().Var(&createDeploymentJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Use = "create-deployment APP_NAME SOURCE_CODE_PATH" + cmd.Short = `Create an App Deployment.` + cmd.Long = `Create an App Deployment. + + Creates an app deployment for the app with the supplied name. + + Arguments: + APP_NAME: The name of the app. + SOURCE_CODE_PATH: The source code path of the deployment.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("json") { + err := root.ExactArgs(1)(cmd, args) + if err != nil { + return fmt.Errorf("when --json flag is specified, provide only APP_NAME as positional arguments. Provide 'source_code_path' in your JSON input") + } + return nil + } + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = createDeploymentJson.Unmarshal(&createDeploymentReq) + if err != nil { + return err + } + } + createDeploymentReq.AppName = args[0] + if !cmd.Flags().Changed("json") { + createDeploymentReq.SourceCodePath = args[1] + } + + wait, err := w.Apps.CreateDeployment(ctx, createDeploymentReq) + if err != nil { + return err + } + if createDeploymentSkipWait { + return cmdio.Render(ctx, wait.Response) + } + spinner := cmdio.Spinner(ctx) + info, err := wait.OnProgress(func(i *serving.AppDeployment) { + if i.Status == nil { + return + } + status := i.Status.State + statusMessage := fmt.Sprintf("current status: %s", status) + if i.Status != nil { + statusMessage = i.Status.Message + } + spinner <- statusMessage + }).GetWithTimeout(createDeploymentTimeout) + close(spinner) + if err != nil { + return err + } + return cmdio.Render(ctx, info) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range createDeploymentOverrides { + fn(cmd, &createDeploymentReq) + } + + return cmd +} + +// start delete command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var deleteOverrides []func( *cobra.Command, *serving.DeleteAppRequest, ) -func newDeleteApp() *cobra.Command { +func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteAppReq serving.DeleteAppRequest + var deleteReq serving.DeleteAppRequest // TODO: short flags - cmd.Use = "delete-app NAME" - cmd.Short = `Delete an application.` - cmd.Long = `Delete an application. + cmd.Use = "delete NAME" + cmd.Short = `Delete an App.` + cmd.Long = `Delete an App. - Delete an application definition + Deletes an app. Arguments: - NAME: The name of an application. This field is required.` + NAME: The name of the app.` cmd.Annotations = make(map[string]string) @@ -146,13 +294,13 @@ func newDeleteApp() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - deleteAppReq.Name = args[0] + deleteReq.Name = args[0] - response, err := w.Apps.DeleteApp(ctx, deleteAppReq) + err = w.Apps.Delete(ctx, deleteReq) if err != nil { return err } - return cmdio.Render(ctx, response) + return nil } // Disable completions since they are not applicable. @@ -160,37 +308,37 @@ func newDeleteApp() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range deleteAppOverrides { - fn(cmd, &deleteAppReq) + for _, fn := range deleteOverrides { + fn(cmd, &deleteReq) } return cmd } -// start get-app command +// start get command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getAppOverrides []func( +var getOverrides []func( *cobra.Command, *serving.GetAppRequest, ) -func newGetApp() *cobra.Command { +func newGet() *cobra.Command { cmd := &cobra.Command{} - var getAppReq serving.GetAppRequest + var getReq serving.GetAppRequest // TODO: short flags - cmd.Use = "get-app NAME" - cmd.Short = `Get definition for an application.` - cmd.Long = `Get definition for an application. + cmd.Use = "get NAME" + cmd.Short = `Get an App.` + cmd.Long = `Get an App. - Get an application definition + Retrieves information for the app with the supplied name. Arguments: - NAME: The name of an application. This field is required.` + NAME: The name of the app.` cmd.Annotations = make(map[string]string) @@ -204,9 +352,9 @@ func newGetApp() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getAppReq.Name = args[0] + getReq.Name = args[0] - response, err := w.Apps.GetApp(ctx, getAppReq) + response, err := w.Apps.Get(ctx, getReq) if err != nil { return err } @@ -218,39 +366,98 @@ func newGetApp() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getAppOverrides { - fn(cmd, &getAppReq) + for _, fn := range getOverrides { + fn(cmd, &getReq) } return cmd } -// start get-app-deployment-status command +// start get-deployment command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getAppDeploymentStatusOverrides []func( +var getDeploymentOverrides []func( *cobra.Command, - *serving.GetAppDeploymentStatusRequest, + *serving.GetAppDeploymentRequest, ) -func newGetAppDeploymentStatus() *cobra.Command { +func newGetDeployment() *cobra.Command { cmd := &cobra.Command{} - var getAppDeploymentStatusReq serving.GetAppDeploymentStatusRequest + var getDeploymentReq serving.GetAppDeploymentRequest // TODO: short flags - cmd.Flags().StringVar(&getAppDeploymentStatusReq.IncludeAppLog, "include-app-log", getAppDeploymentStatusReq.IncludeAppLog, `Boolean flag to include application logs.`) - - cmd.Use = "get-app-deployment-status DEPLOYMENT_ID" - cmd.Short = `Get deployment status for an application.` - cmd.Long = `Get deployment status for an application. + cmd.Use = "get-deployment APP_NAME DEPLOYMENT_ID" + cmd.Short = `Get an App Deployment.` + cmd.Long = `Get an App Deployment. - Get deployment status for an application + Retrieves information for the app deployment with the supplied name and + deployment id. Arguments: - DEPLOYMENT_ID: The deployment id for an application. This field is required.` + APP_NAME: The name of the app. + DEPLOYMENT_ID: The unique id of the deployment.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(2) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getDeploymentReq.AppName = args[0] + getDeploymentReq.DeploymentId = args[1] + + response, err := w.Apps.GetDeployment(ctx, getDeploymentReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getDeploymentOverrides { + fn(cmd, &getDeploymentReq) + } + + return cmd +} + +// start get-environment command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getEnvironmentOverrides []func( + *cobra.Command, + *serving.GetAppEnvironmentRequest, +) + +func newGetEnvironment() *cobra.Command { + cmd := &cobra.Command{} + + var getEnvironmentReq serving.GetAppEnvironmentRequest + + // TODO: short flags + + cmd.Use = "get-environment NAME" + cmd.Short = `Get App Environment.` + cmd.Long = `Get App Environment. + + Retrieves app environment. + + Arguments: + NAME: The name of the app.` cmd.Annotations = make(map[string]string) @@ -264,9 +471,9 @@ func newGetAppDeploymentStatus() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getAppDeploymentStatusReq.DeploymentId = args[0] + getEnvironmentReq.Name = args[0] - response, err := w.Apps.GetAppDeploymentStatus(ctx, getAppDeploymentStatusReq) + response, err := w.Apps.GetEnvironment(ctx, getEnvironmentReq) if err != nil { return err } @@ -278,41 +485,52 @@ func newGetAppDeploymentStatus() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getAppDeploymentStatusOverrides { - fn(cmd, &getAppDeploymentStatusReq) + for _, fn := range getEnvironmentOverrides { + fn(cmd, &getEnvironmentReq) } return cmd } -// start get-apps command +// start list command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getAppsOverrides []func( +var listOverrides []func( *cobra.Command, + *serving.ListAppsRequest, ) -func newGetApps() *cobra.Command { +func newList() *cobra.Command { cmd := &cobra.Command{} - cmd.Use = "get-apps" - cmd.Short = `List all applications.` - cmd.Long = `List all applications. + var listReq serving.ListAppsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, `Upper bound for items returned.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Pagination token to go to the next page of apps.`) + + cmd.Use = "list" + cmd.Short = `List Apps.` + cmd.Long = `List Apps. - List all available applications` + Lists all apps in the workspace.` cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Apps.GetApps(ctx) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + + response := w.Apps.List(ctx, listReq) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -320,37 +538,40 @@ func newGetApps() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getAppsOverrides { - fn(cmd) + for _, fn := range listOverrides { + fn(cmd, &listReq) } return cmd } -// start get-events command +// start list-deployments command // Slice with functions to override default command behavior. // Functions can be added from the `init()` function in manually curated files in this directory. -var getEventsOverrides []func( +var listDeploymentsOverrides []func( *cobra.Command, - *serving.GetEventsRequest, + *serving.ListAppDeploymentsRequest, ) -func newGetEvents() *cobra.Command { +func newListDeployments() *cobra.Command { cmd := &cobra.Command{} - var getEventsReq serving.GetEventsRequest + var listDeploymentsReq serving.ListAppDeploymentsRequest // TODO: short flags - cmd.Use = "get-events NAME" - cmd.Short = `Get deployment events for an application.` - cmd.Long = `Get deployment events for an application. + cmd.Flags().IntVar(&listDeploymentsReq.PageSize, "page-size", listDeploymentsReq.PageSize, `Upper bound for items returned.`) + cmd.Flags().StringVar(&listDeploymentsReq.PageToken, "page-token", listDeploymentsReq.PageToken, `Pagination token to go to the next page of apps.`) + + cmd.Use = "list-deployments APP_NAME" + cmd.Short = `List App Deployments.` + cmd.Long = `List App Deployments. - Get deployment events for an application + Lists all app deployments for the app with the supplied name. Arguments: - NAME: The name of an application. This field is required.` + APP_NAME: The name of the app.` cmd.Annotations = make(map[string]string) @@ -364,9 +585,134 @@ func newGetEvents() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - getEventsReq.Name = args[0] + listDeploymentsReq.AppName = args[0] - response, err := w.Apps.GetEvents(ctx, getEventsReq) + response := w.Apps.ListDeployments(ctx, listDeploymentsReq) + return cmdio.RenderIterator(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range listDeploymentsOverrides { + fn(cmd, &listDeploymentsReq) + } + + return cmd +} + +// start stop command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var stopOverrides []func( + *cobra.Command, + *serving.StopAppRequest, +) + +func newStop() *cobra.Command { + cmd := &cobra.Command{} + + var stopReq serving.StopAppRequest + + // TODO: short flags + + cmd.Use = "stop NAME" + cmd.Short = `Stop an App.` + cmd.Long = `Stop an App. + + Stops the active deployment of the app in the workspace. + + Arguments: + NAME: The name of the app.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + stopReq.Name = args[0] + + err = w.Apps.Stop(ctx, stopReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range stopOverrides { + fn(cmd, &stopReq) + } + + return cmd +} + +// start update command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var updateOverrides []func( + *cobra.Command, + *serving.UpdateAppRequest, +) + +func newUpdate() *cobra.Command { + cmd := &cobra.Command{} + + var updateReq serving.UpdateAppRequest + var updateJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&updateJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&updateReq.Description, "description", updateReq.Description, `The description of the app.`) + + cmd.Use = "update NAME" + cmd.Short = `Update an App.` + cmd.Long = `Update an App. + + Updates the app with the supplied name. + + Arguments: + NAME: The name of the app. The name must contain only lowercase alphanumeric + characters and hyphens and be between 2 and 30 characters long. It must be + unique within the workspace.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = updateJson.Unmarshal(&updateReq) + if err != nil { + return err + } + } + updateReq.Name = args[0] + + response, err := w.Apps.Update(ctx, updateReq) if err != nil { return err } @@ -378,8 +724,8 @@ func newGetEvents() *cobra.Command { cmd.ValidArgsFunction = cobra.NoFileCompletions // Apply optional overrides to this command. - for _, fn := range getEventsOverrides { - fn(cmd, &getEventsReq) + for _, fn := range updateOverrides { + fn(cmd, &updateReq) } return cmd diff --git a/cmd/workspace/apps/overrides.go b/cmd/workspace/apps/overrides.go deleted file mode 100644 index e38e139b5..000000000 --- a/cmd/workspace/apps/overrides.go +++ /dev/null @@ -1,58 +0,0 @@ -package apps - -import ( - "fmt" - - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/flags" - "github.com/databricks/databricks-sdk-go/service/serving" - "github.com/spf13/cobra" -) - -func createOverride(cmd *cobra.Command, deployReq *serving.DeployAppRequest) { - var manifestYaml flags.YamlFlag - var resourcesYaml flags.YamlFlag - createJson := cmd.Flag("json").Value.(*flags.JsonFlag) - - // TODO: short flags - cmd.Flags().Var(&manifestYaml, "manifest", `either inline YAML string or @path/to/manifest.yaml`) - cmd.Flags().Var(&resourcesYaml, "resources", `either inline YAML string or @path/to/resources.yaml`) - - cmd.Annotations = make(map[string]string) - - cmd.PreRunE = root.MustWorkspaceClient - cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() - w := root.WorkspaceClient(ctx) - if cmd.Flags().Changed("json") { - err = createJson.Unmarshal(&deployReq) - if err != nil { - return err - } - } else if cmd.Flags().Changed("manifest") { - err = manifestYaml.Unmarshal(&deployReq.Manifest) - if err != nil { - return err - } - if cmd.Flags().Changed("resources") { - err = resourcesYaml.Unmarshal(&deployReq.Resources) - if err != nil { - return err - } - } - } else { - return fmt.Errorf("please provide command input in YAML format by specifying the --manifest flag or provide a json payload using the --json flag") - } - response, err := w.Apps.Create(ctx, *deployReq) - if err != nil { - return err - } - - return cmdio.Render(ctx, response) - } -} - -func init() { - createOverrides = append(createOverrides, createOverride) -} diff --git a/cmd/workspace/clusters/clusters.go b/cmd/workspace/clusters/clusters.go index e657fd9c3..f4baab3b2 100755 --- a/cmd/workspace/clusters/clusters.go +++ b/cmd/workspace/clusters/clusters.go @@ -188,7 +188,7 @@ func newCreate() *cobra.Command { // TODO: short flags cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, ``) + cmd.Flags().BoolVar(&createReq.ApplyPolicyDefaultValues, "apply-policy-default-values", createReq.ApplyPolicyDefaultValues, `When set to true, fixed and default values from the policy will be used for fields that are omitted.`) // TODO: complex arg: autoscale cmd.Flags().IntVar(&createReq.AutoterminationMinutes, "autotermination-minutes", createReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`) // TODO: complex arg: aws_attributes @@ -196,15 +196,6 @@ func newCreate() *cobra.Command { // TODO: complex arg: clone_from // TODO: complex arg: cluster_log_conf cmd.Flags().StringVar(&createReq.ClusterName, "cluster-name", createReq.ClusterName, `Cluster name requested by the user.`) - cmd.Flags().Var(&createReq.ClusterSource, "cluster-source", `Determines whether the cluster was created by a user through the UI, created by the Databricks Jobs Scheduler, or through an API request. Supported values: [ - API, - JOB, - MODELS, - PIPELINE, - PIPELINE_MAINTENANCE, - SQL, - UI, -]`) // TODO: map via StringToStringVar: custom_tags cmd.Flags().Var(&createReq.DataSecurityMode, "data-security-mode", `Data security mode decides what data governance model to use when accessing data from a cluster. Supported values: [ LEGACY_PASSTHROUGH, @@ -443,23 +434,13 @@ func newEdit() *cobra.Command { // TODO: short flags cmd.Flags().Var(&editJson, "json", `either inline JSON string or @path/to/file.json with request body`) - cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, ``) + cmd.Flags().BoolVar(&editReq.ApplyPolicyDefaultValues, "apply-policy-default-values", editReq.ApplyPolicyDefaultValues, `When set to true, fixed and default values from the policy will be used for fields that are omitted.`) // TODO: complex arg: autoscale cmd.Flags().IntVar(&editReq.AutoterminationMinutes, "autotermination-minutes", editReq.AutoterminationMinutes, `Automatically terminates the cluster after it is inactive for this time in minutes.`) // TODO: complex arg: aws_attributes // TODO: complex arg: azure_attributes - // TODO: complex arg: clone_from // TODO: complex arg: cluster_log_conf cmd.Flags().StringVar(&editReq.ClusterName, "cluster-name", editReq.ClusterName, `Cluster name requested by the user.`) - cmd.Flags().Var(&editReq.ClusterSource, "cluster-source", `Determines whether the cluster was created by a user through the UI, created by the Databricks Jobs Scheduler, or through an API request. Supported values: [ - API, - JOB, - MODELS, - PIPELINE, - PIPELINE_MAINTENANCE, - SQL, - UI, -]`) // TODO: map via StringToStringVar: custom_tags cmd.Flags().Var(&editReq.DataSecurityMode, "data-security-mode", `Data security mode decides what data governance model to use when accessing data from a cluster. Supported values: [ LEGACY_PASSTHROUGH, diff --git a/cmd/workspace/cmd.go b/cmd/workspace/cmd.go index a78b9bc1e..7ad9389a8 100755 --- a/cmd/workspace/cmd.go +++ b/cmd/workspace/cmd.go @@ -32,7 +32,6 @@ import ( instance_profiles "github.com/databricks/cli/cmd/workspace/instance-profiles" ip_access_lists "github.com/databricks/cli/cmd/workspace/ip-access-lists" jobs "github.com/databricks/cli/cmd/workspace/jobs" - lakehouse_monitors "github.com/databricks/cli/cmd/workspace/lakehouse-monitors" lakeview "github.com/databricks/cli/cmd/workspace/lakeview" libraries "github.com/databricks/cli/cmd/workspace/libraries" metastores "github.com/databricks/cli/cmd/workspace/metastores" @@ -51,6 +50,7 @@ import ( provider_provider_analytics_dashboards "github.com/databricks/cli/cmd/workspace/provider-provider-analytics-dashboards" provider_providers "github.com/databricks/cli/cmd/workspace/provider-providers" providers "github.com/databricks/cli/cmd/workspace/providers" + quality_monitors "github.com/databricks/cli/cmd/workspace/quality-monitors" queries "github.com/databricks/cli/cmd/workspace/queries" query_history "github.com/databricks/cli/cmd/workspace/query-history" query_visualizations "github.com/databricks/cli/cmd/workspace/query-visualizations" @@ -113,7 +113,6 @@ func All() []*cobra.Command { out = append(out, instance_profiles.New()) out = append(out, ip_access_lists.New()) out = append(out, jobs.New()) - out = append(out, lakehouse_monitors.New()) out = append(out, lakeview.New()) out = append(out, libraries.New()) out = append(out, metastores.New()) @@ -132,6 +131,7 @@ func All() []*cobra.Command { out = append(out, provider_provider_analytics_dashboards.New()) out = append(out, provider_providers.New()) out = append(out, providers.New()) + out = append(out, quality_monitors.New()) out = append(out, queries.New()) out = append(out, query_history.New()) out = append(out, query_visualizations.New()) diff --git a/cmd/workspace/csp-enablement/csp-enablement.go b/cmd/workspace/compliance-security-profile/compliance-security-profile.go similarity index 88% rename from cmd/workspace/csp-enablement/csp-enablement.go rename to cmd/workspace/compliance-security-profile/compliance-security-profile.go index 312591564..a7b45901f 100755 --- a/cmd/workspace/csp-enablement/csp-enablement.go +++ b/cmd/workspace/compliance-security-profile/compliance-security-profile.go @@ -1,6 +1,6 @@ // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. -package csp_enablement +package compliance_security_profile import ( "fmt" @@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "csp-enablement", + Use: "compliance-security-profile", Short: `Controls whether to enable the compliance security profile for the current workspace.`, Long: `Controls whether to enable the compliance security profile for the current workspace. Enabling it on a workspace is permanent. By default, it is turned @@ -45,13 +45,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *settings.GetCspEnablementSettingRequest, + *settings.GetComplianceSecurityProfileSettingRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq settings.GetCspEnablementSettingRequest + var getReq settings.GetComplianceSecurityProfileSettingRequest // TODO: short flags @@ -75,7 +75,7 @@ func newGet() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Settings.CspEnablement().Get(ctx, getReq) + response, err := w.Settings.ComplianceSecurityProfile().Get(ctx, getReq) if err != nil { return err } @@ -100,13 +100,13 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *settings.UpdateCspEnablementSettingRequest, + *settings.UpdateComplianceSecurityProfileSettingRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq settings.UpdateCspEnablementSettingRequest + var updateReq settings.UpdateComplianceSecurityProfileSettingRequest var updateJson flags.JsonFlag // TODO: short flags @@ -138,7 +138,7 @@ func newUpdate() *cobra.Command { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } - response, err := w.Settings.CspEnablement().Update(ctx, updateReq) + response, err := w.Settings.ComplianceSecurityProfile().Update(ctx, updateReq) if err != nil { return err } @@ -157,4 +157,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service CSPEnablement +// end service ComplianceSecurityProfile diff --git a/cmd/workspace/connections/connections.go b/cmd/workspace/connections/connections.go index bdb266685..f76420fbe 100755 --- a/cmd/workspace/connections/connections.go +++ b/cmd/workspace/connections/connections.go @@ -154,7 +154,7 @@ func newDelete() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) promptSpinner <- "No NAME argument specified. Loading names for Connections drop-down." - names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx) + names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx, catalog.ListConnectionsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Connections drop-down. Please manually specify required arguments. Original error: %w", err) @@ -224,7 +224,7 @@ func newGet() *cobra.Command { if len(args) == 0 { promptSpinner := cmdio.Spinner(ctx) promptSpinner <- "No NAME argument specified. Loading names for Connections drop-down." - names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx) + names, err := w.Connections.ConnectionInfoNameToFullNameMap(ctx, catalog.ListConnectionsRequest{}) close(promptSpinner) if err != nil { return fmt.Errorf("failed to load names for Connections drop-down. Please manually specify required arguments. Original error: %w", err) @@ -265,11 +265,19 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var listOverrides []func( *cobra.Command, + *catalog.ListConnectionsRequest, ) func newList() *cobra.Command { cmd := &cobra.Command{} + var listReq catalog.ListConnectionsRequest + + // TODO: short flags + + cmd.Flags().IntVar(&listReq.MaxResults, "max-results", listReq.MaxResults, `Maximum number of connections to return.`) + cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, `Opaque pagination token to go to next page based on previous query.`) + cmd.Use = "list" cmd.Short = `List connections.` cmd.Long = `List connections. @@ -278,11 +286,17 @@ func newList() *cobra.Command { cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(0) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response := w.Connections.List(ctx) + + response := w.Connections.List(ctx, listReq) return cmdio.RenderIterator(ctx, response) } @@ -292,7 +306,7 @@ func newList() *cobra.Command { // Apply optional overrides to this command. for _, fn := range listOverrides { - fn(cmd) + fn(cmd, &listReq) } return cmd diff --git a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go index cd92002a4..6f3ba4b42 100755 --- a/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go +++ b/cmd/workspace/consumer-fulfillments/consumer-fulfillments.go @@ -64,9 +64,6 @@ func newGet() *cobra.Command { Get a high level preview of the metadata of listing installable content.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -126,9 +123,6 @@ func newList() *cobra.Command { Personalized installations contain metadata about the attached share or git repo, as well as the Delta Sharing recipient type.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/consumer-installations/consumer-installations.go b/cmd/workspace/consumer-installations/consumer-installations.go index 9d6c7c894..d176e5b39 100755 --- a/cmd/workspace/consumer-installations/consumer-installations.go +++ b/cmd/workspace/consumer-installations/consumer-installations.go @@ -76,9 +76,6 @@ func newCreate() *cobra.Command { Install payload associated with a Databricks Marketplace listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -140,9 +137,6 @@ func newDelete() *cobra.Command { Uninstall an installation associated with a Databricks Marketplace listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -202,9 +196,6 @@ func newList() *cobra.Command { List all installations across all listings.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -258,9 +249,6 @@ func newListListingInstallations() *cobra.Command { List all installations for a particular listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -321,9 +309,6 @@ func newUpdate() *cobra.Command { the rotateToken flag is true 2. the token will be forcibly rotate if the rotateToken flag is true and the tokenInfo field is empty` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/consumer-listings/consumer-listings.go b/cmd/workspace/consumer-listings/consumer-listings.go index 70295dfb3..8669dfae5 100755 --- a/cmd/workspace/consumer-listings/consumer-listings.go +++ b/cmd/workspace/consumer-listings/consumer-listings.go @@ -66,9 +66,6 @@ func newGet() *cobra.Command { Get a published listing in the Databricks Marketplace that the consumer has access to.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -132,13 +129,14 @@ func newList() *cobra.Command { // TODO: array: assets // TODO: array: categories + cmd.Flags().BoolVar(&listReq.IsAscending, "is-ascending", listReq.IsAscending, ``) cmd.Flags().BoolVar(&listReq.IsFree, "is-free", listReq.IsFree, `Filters each listing based on if it is free.`) cmd.Flags().BoolVar(&listReq.IsPrivateExchange, "is-private-exchange", listReq.IsPrivateExchange, `Filters each listing based on if it is a private exchange.`) cmd.Flags().BoolVar(&listReq.IsStaffPick, "is-staff-pick", listReq.IsStaffPick, `Filters each listing based on whether it is a staff pick.`) cmd.Flags().IntVar(&listReq.PageSize, "page-size", listReq.PageSize, ``) cmd.Flags().StringVar(&listReq.PageToken, "page-token", listReq.PageToken, ``) // TODO: array: provider_ids - // TODO: complex arg: sort_by_spec + cmd.Flags().Var(&listReq.SortBy, "sort-by", `Criteria for sorting the resulting set of listings. Supported values: [SORT_BY_DATE, SORT_BY_RELEVANCE, SORT_BY_TITLE, SORT_BY_UNSPECIFIED]`) // TODO: array: tags cmd.Use = "list" @@ -148,9 +146,6 @@ func newList() *cobra.Command { List all published listings in the Databricks Marketplace that the consumer has access to.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -197,6 +192,7 @@ func newSearch() *cobra.Command { // TODO: array: assets // TODO: array: categories + cmd.Flags().BoolVar(&searchReq.IsAscending, "is-ascending", searchReq.IsAscending, ``) cmd.Flags().BoolVar(&searchReq.IsFree, "is-free", searchReq.IsFree, ``) cmd.Flags().BoolVar(&searchReq.IsPrivateExchange, "is-private-exchange", searchReq.IsPrivateExchange, ``) cmd.Flags().IntVar(&searchReq.PageSize, "page-size", searchReq.PageSize, ``) @@ -215,9 +211,6 @@ func newSearch() *cobra.Command { Arguments: QUERY: Fuzzy matches query` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient diff --git a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go index 40ae4c848..c55ca4ee1 100755 --- a/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go +++ b/cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go @@ -75,9 +75,6 @@ func newCreate() *cobra.Command { Create a personalization request for a listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -142,9 +139,6 @@ func newGet() *cobra.Command { Get the personalization request for a listing. Each consumer can make at *most* one personalization request for a listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -203,9 +197,6 @@ func newList() *cobra.Command { List personalization requests for a consumer across all listings.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/consumer-providers/consumer-providers.go b/cmd/workspace/consumer-providers/consumer-providers.go index 5a0849dce..d8ac0ec12 100755 --- a/cmd/workspace/consumer-providers/consumer-providers.go +++ b/cmd/workspace/consumer-providers/consumer-providers.go @@ -64,9 +64,6 @@ func newGet() *cobra.Command { Get a provider in the Databricks Marketplace with at least one visible listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -139,9 +136,6 @@ func newList() *cobra.Command { List all providers in the Databricks Marketplace with at least one visible listing.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/dashboards/dashboards.go b/cmd/workspace/dashboards/dashboards.go index 0500ebecf..1a143538b 100755 --- a/cmd/workspace/dashboards/dashboards.go +++ b/cmd/workspace/dashboards/dashboards.go @@ -386,6 +386,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `The title of this dashboard that appears in list views and at the top of the dashboard page.`) cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) + // TODO: array: tags cmd.Use = "update DASHBOARD_ID" cmd.Short = `Change a dashboard definition.` diff --git a/cmd/workspace/esm-enablement/esm-enablement.go b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go similarity index 89% rename from cmd/workspace/esm-enablement/esm-enablement.go rename to cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go index a65fe2f76..a8acc5cd1 100755 --- a/cmd/workspace/esm-enablement/esm-enablement.go +++ b/cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go @@ -1,6 +1,6 @@ // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. -package esm_enablement +package enhanced_security_monitoring import ( "fmt" @@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "esm-enablement", + Use: "enhanced-security-monitoring", Short: `Controls whether enhanced security monitoring is enabled for the current workspace.`, Long: `Controls whether enhanced security monitoring is enabled for the current workspace. If the compliance security profile is enabled, this is @@ -47,13 +47,13 @@ func New() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *settings.GetEsmEnablementSettingRequest, + *settings.GetEnhancedSecurityMonitoringSettingRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq settings.GetEsmEnablementSettingRequest + var getReq settings.GetEnhancedSecurityMonitoringSettingRequest // TODO: short flags @@ -77,7 +77,7 @@ func newGet() *cobra.Command { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Settings.EsmEnablement().Get(ctx, getReq) + response, err := w.Settings.EnhancedSecurityMonitoring().Get(ctx, getReq) if err != nil { return err } @@ -102,13 +102,13 @@ func newGet() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var updateOverrides []func( *cobra.Command, - *settings.UpdateEsmEnablementSettingRequest, + *settings.UpdateEnhancedSecurityMonitoringSettingRequest, ) func newUpdate() *cobra.Command { cmd := &cobra.Command{} - var updateReq settings.UpdateEsmEnablementSettingRequest + var updateReq settings.UpdateEnhancedSecurityMonitoringSettingRequest var updateJson flags.JsonFlag // TODO: short flags @@ -140,7 +140,7 @@ func newUpdate() *cobra.Command { return fmt.Errorf("please provide command input in JSON format by specifying the --json flag") } - response, err := w.Settings.EsmEnablement().Update(ctx, updateReq) + response, err := w.Settings.EnhancedSecurityMonitoring().Update(ctx, updateReq) if err != nil { return err } @@ -159,4 +159,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service ESMEnablement +// end service EnhancedSecurityMonitoring diff --git a/cmd/workspace/jobs/jobs.go b/cmd/workspace/jobs/jobs.go index 267dfc73b..e31c3f086 100755 --- a/cmd/workspace/jobs/jobs.go +++ b/cmd/workspace/jobs/jobs.go @@ -1513,6 +1513,7 @@ func newSubmit() *cobra.Command { // TODO: complex arg: pipeline_task // TODO: complex arg: python_wheel_task // TODO: complex arg: queue + // TODO: complex arg: run_as // TODO: complex arg: run_job_task cmd.Flags().StringVar(&submitReq.RunName, "run-name", submitReq.RunName, `An optional name for the run.`) // TODO: complex arg: spark_jar_task diff --git a/cmd/workspace/libraries/libraries.go b/cmd/workspace/libraries/libraries.go index e11e5a4c5..2c10d8161 100755 --- a/cmd/workspace/libraries/libraries.go +++ b/cmd/workspace/libraries/libraries.go @@ -25,18 +25,14 @@ func New() *cobra.Command { To make third-party or custom code available to notebooks and jobs running on your clusters, you can install a library. Libraries can be written in Python, - Java, Scala, and R. You can upload Java, Scala, and Python libraries and point - to external packages in PyPI, Maven, and CRAN repositories. + Java, Scala, and R. You can upload Python, Java, Scala and R libraries and + point to external packages in PyPI, Maven, and CRAN repositories. Cluster libraries can be used by all notebooks running on a cluster. You can install a cluster library directly from a public repository such as PyPI or Maven, using a previously installed workspace library, or using an init script. - When you install a library on a cluster, a notebook already attached to that - cluster will not immediately see the new library. You must first detach and - then reattach the notebook to the cluster. - When you uninstall a library from a cluster, the library is removed only when you restart the cluster. Until you restart the cluster, the status of the uninstalled library appears as Uninstall pending restart.`, @@ -75,9 +71,8 @@ func newAllClusterStatuses() *cobra.Command { cmd.Short = `Get all statuses.` cmd.Long = `Get all statuses. - Get the status of all libraries on all clusters. A status will be available - for all libraries installed on this cluster via the API or the libraries UI as - well as libraries set to be installed on all clusters via the libraries UI.` + Get the status of all libraries on all clusters. A status is returned for all + libraries installed on this cluster via the API or the libraries UI.` cmd.Annotations = make(map[string]string) @@ -85,11 +80,8 @@ func newAllClusterStatuses() *cobra.Command { cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - response, err := w.Libraries.AllClusterStatuses(ctx) - if err != nil { - return err - } - return cmdio.Render(ctx, response) + response := w.Libraries.AllClusterStatuses(ctx) + return cmdio.RenderIterator(ctx, response) } // Disable completions since they are not applicable. @@ -110,13 +102,13 @@ func newAllClusterStatuses() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var clusterStatusOverrides []func( *cobra.Command, - *compute.ClusterStatusRequest, + *compute.ClusterStatus, ) func newClusterStatus() *cobra.Command { cmd := &cobra.Command{} - var clusterStatusReq compute.ClusterStatusRequest + var clusterStatusReq compute.ClusterStatus // TODO: short flags @@ -124,21 +116,13 @@ func newClusterStatus() *cobra.Command { cmd.Short = `Get status.` cmd.Long = `Get status. - Get the status of libraries on a cluster. A status will be available for all - libraries installed on this cluster via the API or the libraries UI as well as - libraries set to be installed on all clusters via the libraries UI. The order - of returned libraries will be as follows. - - 1. Libraries set to be installed on this cluster will be returned first. - Within this group, the final order will be order in which the libraries were - added to the cluster. - - 2. Libraries set to be installed on all clusters are returned next. Within - this group there is no order guarantee. - - 3. Libraries that were previously requested on this cluster or on all - clusters, but now marked for removal. Within this group there is no order - guarantee. + Get the status of libraries on a cluster. A status is returned for all + libraries installed on this cluster via the API or the libraries UI. The order + of returned libraries is as follows: 1. Libraries set to be installed on this + cluster, in the order that the libraries were added to the cluster, are + returned first. 2. Libraries that were previously requested to be installed on + this cluster or, but are now marked for removal, in no particular order, are + returned last. Arguments: CLUSTER_ID: Unique identifier of the cluster whose status should be retrieved.` @@ -195,12 +179,8 @@ func newInstall() *cobra.Command { cmd.Short = `Add a library.` cmd.Long = `Add a library. - Add libraries to be installed on a cluster. The installation is asynchronous; - it happens in the background after the completion of this request. - - **Note**: The actual set of libraries to be installed on a cluster is the - union of the libraries specified via this method and the libraries set to be - installed on all clusters via the libraries UI.` + Add libraries to install on a cluster. The installation is asynchronous; it + happens in the background after the completion of this request.` cmd.Annotations = make(map[string]string) @@ -259,9 +239,9 @@ func newUninstall() *cobra.Command { cmd.Short = `Uninstall libraries.` cmd.Long = `Uninstall libraries. - Set libraries to be uninstalled on a cluster. The libraries won't be - uninstalled until the cluster is restarted. Uninstalling libraries that are - not installed on the cluster will have no impact but is not an error.` + Set libraries to uninstall from a cluster. The libraries won't be uninstalled + until the cluster is restarted. A request to uninstall a library that is not + currently installed is ignored.` cmd.Annotations = make(map[string]string) diff --git a/cmd/workspace/model-versions/model-versions.go b/cmd/workspace/model-versions/model-versions.go index 7b556c724..034cea2df 100755 --- a/cmd/workspace/model-versions/model-versions.go +++ b/cmd/workspace/model-versions/model-versions.go @@ -288,6 +288,7 @@ func newList() *cobra.Command { schema. There is no guarantee of a specific ordering of the elements in the response. + The elements in the response will not contain any aliases or tags. Arguments: FULL_NAME: The full three-level name of the registered model under which to list diff --git a/cmd/workspace/pipelines/pipelines.go b/cmd/workspace/pipelines/pipelines.go index b7c3235f8..f1cc4e3f7 100755 --- a/cmd/workspace/pipelines/pipelines.go +++ b/cmd/workspace/pipelines/pipelines.go @@ -940,11 +940,14 @@ func newUpdate() *cobra.Command { // TODO: array: clusters // TODO: map via StringToStringVar: configuration cmd.Flags().BoolVar(&updateReq.Continuous, "continuous", updateReq.Continuous, `Whether the pipeline is continuous or triggered.`) + // TODO: complex arg: deployment cmd.Flags().BoolVar(&updateReq.Development, "development", updateReq.Development, `Whether the pipeline is in Development mode.`) cmd.Flags().StringVar(&updateReq.Edition, "edition", updateReq.Edition, `Pipeline product edition.`) cmd.Flags().Int64Var(&updateReq.ExpectedLastModified, "expected-last-modified", updateReq.ExpectedLastModified, `If present, the last-modified time of the pipeline settings before the edit.`) // TODO: complex arg: filters + // TODO: complex arg: gateway_definition cmd.Flags().StringVar(&updateReq.Id, "id", updateReq.Id, `Unique identifier for this pipeline.`) + // TODO: complex arg: ingestion_definition // TODO: array: libraries cmd.Flags().StringVar(&updateReq.Name, "name", updateReq.Name, `Friendly identifier for this pipeline.`) // TODO: array: notifications diff --git a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go index 43ae6da7e..4ab36b5d0 100755 --- a/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go +++ b/cmd/workspace/provider-exchange-filters/provider-exchange-filters.go @@ -68,9 +68,6 @@ func newCreate() *cobra.Command { Add an exchange filter.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -128,9 +125,6 @@ func newDelete() *cobra.Command { Delete an exchange filter` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -201,9 +195,6 @@ func newList() *cobra.Command { List exchange filter` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -258,9 +249,6 @@ func newUpdate() *cobra.Command { Update an exchange filter.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/provider-exchanges/provider-exchanges.go b/cmd/workspace/provider-exchanges/provider-exchanges.go index fe1a9a3dc..7ff73e0d1 100755 --- a/cmd/workspace/provider-exchanges/provider-exchanges.go +++ b/cmd/workspace/provider-exchanges/provider-exchanges.go @@ -74,9 +74,6 @@ func newAddListingToExchange() *cobra.Command { Associate an exchange with a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -152,9 +149,6 @@ func newCreate() *cobra.Command { Create an exchange` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -212,9 +206,6 @@ func newDelete() *cobra.Command { This removes a listing from marketplace.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -270,9 +261,6 @@ func newDeleteListingFromExchange() *cobra.Command { Disassociate an exchange with a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -328,9 +316,6 @@ func newGet() *cobra.Command { Get an exchange.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -389,9 +374,6 @@ func newList() *cobra.Command { List exchanges visible to provider` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -445,9 +427,6 @@ func newListExchangesForListing() *cobra.Command { List exchanges associated with a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -503,33 +482,18 @@ func newListListingsForExchange() *cobra.Command { List listings associated with an exchange` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + cmd.PreRunE = root.MustWorkspaceClient cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() w := root.WorkspaceClient(ctx) - if len(args) == 0 { - promptSpinner := cmdio.Spinner(ctx) - promptSpinner <- "No EXCHANGE_ID argument specified. Loading names for Provider Exchanges drop-down." - names, err := w.ProviderExchanges.ExchangeListingExchangeNameToExchangeIdMap(ctx, marketplace.ListExchangesForListingRequest{}) - close(promptSpinner) - if err != nil { - return fmt.Errorf("failed to load names for Provider Exchanges drop-down. Please manually specify required arguments. Original error: %w", err) - } - id, err := cmdio.Select(ctx, names, "") - if err != nil { - return err - } - args = append(args, id) - } - if len(args) != 1 { - return fmt.Errorf("expected to have ") - } listListingsForExchangeReq.ExchangeId = args[0] response := w.ProviderExchanges.ListListingsForExchange(ctx, listListingsForExchangeReq) @@ -572,9 +536,6 @@ func newUpdate() *cobra.Command { Update an exchange` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/provider-files/provider-files.go b/cmd/workspace/provider-files/provider-files.go index b9357f131..25e1addf5 100755 --- a/cmd/workspace/provider-files/provider-files.go +++ b/cmd/workspace/provider-files/provider-files.go @@ -72,9 +72,6 @@ func newCreate() *cobra.Command { Create a file. Currently, only provider icons and attached notebooks are supported.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -132,9 +129,6 @@ func newDelete() *cobra.Command { Delete a file` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -202,9 +196,6 @@ func newGet() *cobra.Command { Get a file` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -277,9 +268,6 @@ func newList() *cobra.Command { List files attached to a parent entity.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient diff --git a/cmd/workspace/provider-listings/provider-listings.go b/cmd/workspace/provider-listings/provider-listings.go index 4f90f7b9e..0abdf51d8 100755 --- a/cmd/workspace/provider-listings/provider-listings.go +++ b/cmd/workspace/provider-listings/provider-listings.go @@ -70,9 +70,6 @@ func newCreate() *cobra.Command { Create a new listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -130,9 +127,6 @@ func newDelete() *cobra.Command { Delete a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -200,9 +194,6 @@ func newGet() *cobra.Command { Get a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -273,9 +264,6 @@ func newList() *cobra.Command { List listings owned by this provider` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -328,9 +316,6 @@ func newUpdate() *cobra.Command { Update a listing` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go index 58b3cba1d..a38d9f420 100755 --- a/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go +++ b/cmd/workspace/provider-personalization-requests/provider-personalization-requests.go @@ -69,9 +69,6 @@ func newList() *cobra.Command { List personalization requests to this provider. This will return all personalization requests, regardless of which listing they are for.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -128,9 +125,6 @@ func newUpdate() *cobra.Command { Update personalization request. This method only permits updating the status of the request.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go index 70ef0f320..8cee6e4eb 100755 --- a/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go +++ b/cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go @@ -60,9 +60,6 @@ func newCreate() *cobra.Command { Create provider analytics dashboard. Returns Marketplace specific id. Not to be confused with the Lakeview dashboard id.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -105,9 +102,6 @@ func newGet() *cobra.Command { Get provider analytics dashboard.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -150,9 +144,6 @@ func newGetLatestVersion() *cobra.Command { Get latest version of provider analytics dashboard.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -207,9 +198,6 @@ func newUpdate() *cobra.Command { Arguments: ID: id is immutable property and can't be updated.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/provider-providers/provider-providers.go b/cmd/workspace/provider-providers/provider-providers.go index 52f4c45ae..b7273a344 100755 --- a/cmd/workspace/provider-providers/provider-providers.go +++ b/cmd/workspace/provider-providers/provider-providers.go @@ -69,9 +69,6 @@ func newCreate() *cobra.Command { Create a provider` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -129,9 +126,6 @@ func newDelete() *cobra.Command { Delete provider` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -199,9 +193,6 @@ func newGet() *cobra.Command { Get provider profile` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.PreRunE = root.MustWorkspaceClient @@ -272,9 +263,6 @@ func newList() *cobra.Command { List provider profiles for account.` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { @@ -327,9 +315,6 @@ func newUpdate() *cobra.Command { Update provider profile` - // This command is being previewed; hide from help output. - cmd.Hidden = true - cmd.Annotations = make(map[string]string) cmd.Args = func(cmd *cobra.Command, args []string) error { diff --git a/cmd/workspace/lakehouse-monitors/lakehouse-monitors.go b/cmd/workspace/quality-monitors/quality-monitors.go similarity index 95% rename from cmd/workspace/lakehouse-monitors/lakehouse-monitors.go rename to cmd/workspace/quality-monitors/quality-monitors.go index 465ed6f92..95d992164 100755 --- a/cmd/workspace/lakehouse-monitors/lakehouse-monitors.go +++ b/cmd/workspace/quality-monitors/quality-monitors.go @@ -1,6 +1,6 @@ // Code generated from OpenAPI specs by Databricks SDK Generator. DO NOT EDIT. -package lakehouse_monitors +package quality_monitors import ( "fmt" @@ -18,7 +18,7 @@ var cmdOverrides []func(*cobra.Command) func New() *cobra.Command { cmd := &cobra.Command{ - Use: "lakehouse-monitors", + Use: "quality-monitors", Short: `A monitor computes and monitors data or model quality metrics for a table over time.`, Long: `A monitor computes and monitors data or model quality metrics for a table over time. It generates metrics tables and a dashboard that you can use to monitor @@ -105,7 +105,7 @@ func newCancelRefresh() *cobra.Command { cancelRefreshReq.TableName = args[0] cancelRefreshReq.RefreshId = args[1] - err = w.LakehouseMonitors.CancelRefresh(ctx, cancelRefreshReq) + err = w.QualityMonitors.CancelRefresh(ctx, cancelRefreshReq) if err != nil { return err } @@ -208,7 +208,7 @@ func newCreate() *cobra.Command { createReq.OutputSchemaName = args[2] } - response, err := w.LakehouseMonitors.Create(ctx, createReq) + response, err := w.QualityMonitors.Create(ctx, createReq) if err != nil { return err } @@ -233,13 +233,13 @@ func newCreate() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var deleteOverrides []func( *cobra.Command, - *catalog.DeleteLakehouseMonitorRequest, + *catalog.DeleteQualityMonitorRequest, ) func newDelete() *cobra.Command { cmd := &cobra.Command{} - var deleteReq catalog.DeleteLakehouseMonitorRequest + var deleteReq catalog.DeleteQualityMonitorRequest // TODO: short flags @@ -278,7 +278,7 @@ func newDelete() *cobra.Command { deleteReq.TableName = args[0] - err = w.LakehouseMonitors.Delete(ctx, deleteReq) + err = w.QualityMonitors.Delete(ctx, deleteReq) if err != nil { return err } @@ -303,13 +303,13 @@ func newDelete() *cobra.Command { // Functions can be added from the `init()` function in manually curated files in this directory. var getOverrides []func( *cobra.Command, - *catalog.GetLakehouseMonitorRequest, + *catalog.GetQualityMonitorRequest, ) func newGet() *cobra.Command { cmd := &cobra.Command{} - var getReq catalog.GetLakehouseMonitorRequest + var getReq catalog.GetQualityMonitorRequest // TODO: short flags @@ -347,7 +347,7 @@ func newGet() *cobra.Command { getReq.TableName = args[0] - response, err := w.LakehouseMonitors.Get(ctx, getReq) + response, err := w.QualityMonitors.Get(ctx, getReq) if err != nil { return err } @@ -416,7 +416,7 @@ func newGetRefresh() *cobra.Command { getRefreshReq.TableName = args[0] getRefreshReq.RefreshId = args[1] - response, err := w.LakehouseMonitors.GetRefresh(ctx, getRefreshReq) + response, err := w.QualityMonitors.GetRefresh(ctx, getRefreshReq) if err != nil { return err } @@ -484,7 +484,7 @@ func newListRefreshes() *cobra.Command { listRefreshesReq.TableName = args[0] - response, err := w.LakehouseMonitors.ListRefreshes(ctx, listRefreshesReq) + response, err := w.QualityMonitors.ListRefreshes(ctx, listRefreshesReq) if err != nil { return err } @@ -552,7 +552,7 @@ func newRunRefresh() *cobra.Command { runRefreshReq.TableName = args[0] - response, err := w.LakehouseMonitors.RunRefresh(ctx, runRefreshReq) + response, err := w.QualityMonitors.RunRefresh(ctx, runRefreshReq) if err != nil { return err } @@ -591,6 +591,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.BaselineTableName, "baseline-table-name", updateReq.BaselineTableName, `Name of the baseline table from which drift metrics are computed from.`) // TODO: array: custom_metrics + cmd.Flags().StringVar(&updateReq.DashboardId, "dashboard-id", updateReq.DashboardId, `Id of dashboard that visualizes the computed metrics.`) // TODO: complex arg: data_classification_config // TODO: complex arg: inference_log // TODO: complex arg: notifications @@ -651,7 +652,7 @@ func newUpdate() *cobra.Command { updateReq.OutputSchemaName = args[1] } - response, err := w.LakehouseMonitors.Update(ctx, updateReq) + response, err := w.QualityMonitors.Update(ctx, updateReq) if err != nil { return err } @@ -670,4 +671,4 @@ func newUpdate() *cobra.Command { return cmd } -// end service LakehouseMonitors +// end service QualityMonitors diff --git a/cmd/workspace/queries/queries.go b/cmd/workspace/queries/queries.go index 0126097fc..b96eb7154 100755 --- a/cmd/workspace/queries/queries.go +++ b/cmd/workspace/queries/queries.go @@ -401,6 +401,7 @@ func newUpdate() *cobra.Command { // TODO: any: options cmd.Flags().StringVar(&updateReq.Query, "query", updateReq.Query, `The text of the query to be run.`) cmd.Flags().Var(&updateReq.RunAsRole, "run-as-role", `Sets the **Run as** role for the object. Supported values: [owner, viewer]`) + // TODO: array: tags cmd.Use = "update QUERY_ID" cmd.Short = `Change a query definition.` diff --git a/cmd/workspace/serving-endpoints/serving-endpoints.go b/cmd/workspace/serving-endpoints/serving-endpoints.go index 6706b99ea..b92f824d3 100755 --- a/cmd/workspace/serving-endpoints/serving-endpoints.go +++ b/cmd/workspace/serving-endpoints/serving-endpoints.go @@ -46,6 +46,7 @@ func New() *cobra.Command { cmd.AddCommand(newDelete()) cmd.AddCommand(newExportMetrics()) cmd.AddCommand(newGet()) + cmd.AddCommand(newGetOpenApi()) cmd.AddCommand(newGetPermissionLevels()) cmd.AddCommand(newGetPermissions()) cmd.AddCommand(newList()) @@ -151,6 +152,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) // TODO: array: rate_limits + cmd.Flags().BoolVar(&createReq.RouteOptimized, "route-optimized", createReq.RouteOptimized, `Enable route optimization for the serving endpoint.`) // TODO: array: tags cmd.Use = "create" @@ -302,11 +304,12 @@ func newExportMetrics() *cobra.Command { exportMetricsReq.Name = args[0] - err = w.ServingEndpoints.ExportMetrics(ctx, exportMetricsReq) + response, err := w.ServingEndpoints.ExportMetrics(ctx, exportMetricsReq) if err != nil { return err } - return nil + defer response.Contents.Close() + return cmdio.Render(ctx, response.Contents) } // Disable completions since they are not applicable. @@ -379,6 +382,67 @@ func newGet() *cobra.Command { return cmd } +// start get-open-api command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var getOpenApiOverrides []func( + *cobra.Command, + *serving.GetOpenApiRequest, +) + +func newGetOpenApi() *cobra.Command { + cmd := &cobra.Command{} + + var getOpenApiReq serving.GetOpenApiRequest + + // TODO: short flags + + cmd.Use = "get-open-api NAME" + cmd.Short = `Get the schema for a serving endpoint.` + cmd.Long = `Get the schema for a serving endpoint. + + Get the query schema of the serving endpoint in OpenAPI format. The schema + contains information for the supported paths, input and output format and + datatypes. + + Arguments: + NAME: The name of the serving endpoint that the served model belongs to. This + field is required.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + getOpenApiReq.Name = args[0] + + err = w.ServingEndpoints.GetOpenApi(ctx, getOpenApiReq) + if err != nil { + return err + } + return nil + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range getOpenApiOverrides { + fn(cmd, &getOpenApiReq) + } + + return cmd +} + // start get-permission-levels command // Slice with functions to override default command behavior. diff --git a/cmd/workspace/settings/settings.go b/cmd/workspace/settings/settings.go index 38e19e839..214986c76 100755 --- a/cmd/workspace/settings/settings.go +++ b/cmd/workspace/settings/settings.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/cobra" automatic_cluster_update "github.com/databricks/cli/cmd/workspace/automatic-cluster-update" - csp_enablement "github.com/databricks/cli/cmd/workspace/csp-enablement" + compliance_security_profile "github.com/databricks/cli/cmd/workspace/compliance-security-profile" default_namespace "github.com/databricks/cli/cmd/workspace/default-namespace" - esm_enablement "github.com/databricks/cli/cmd/workspace/esm-enablement" + enhanced_security_monitoring "github.com/databricks/cli/cmd/workspace/enhanced-security-monitoring" restrict_workspace_admins "github.com/databricks/cli/cmd/workspace/restrict-workspace-admins" ) @@ -29,9 +29,9 @@ func New() *cobra.Command { // Add subservices cmd.AddCommand(automatic_cluster_update.New()) - cmd.AddCommand(csp_enablement.New()) + cmd.AddCommand(compliance_security_profile.New()) cmd.AddCommand(default_namespace.New()) - cmd.AddCommand(esm_enablement.New()) + cmd.AddCommand(enhanced_security_monitoring.New()) cmd.AddCommand(restrict_workspace_admins.New()) // Apply optional overrides to this command. diff --git a/cmd/workspace/shares/shares.go b/cmd/workspace/shares/shares.go index 0e3523cec..c2fd779a7 100755 --- a/cmd/workspace/shares/shares.go +++ b/cmd/workspace/shares/shares.go @@ -67,6 +67,7 @@ func newCreate() *cobra.Command { cmd.Flags().Var(&createJson, "json", `either inline JSON string or @path/to/file.json with request body`) cmd.Flags().StringVar(&createReq.Comment, "comment", createReq.Comment, `User-provided free-form text description.`) + cmd.Flags().StringVar(&createReq.StorageRoot, "storage-root", createReq.StorageRoot, `Storage root URL for the share.`) cmd.Use = "create NAME" cmd.Short = `Create a share.` @@ -368,6 +369,7 @@ func newUpdate() *cobra.Command { cmd.Flags().StringVar(&updateReq.Comment, "comment", updateReq.Comment, `User-provided free-form text description.`) cmd.Flags().StringVar(&updateReq.NewName, "new-name", updateReq.NewName, `New name for the share.`) cmd.Flags().StringVar(&updateReq.Owner, "owner", updateReq.Owner, `Username of current owner of share.`) + cmd.Flags().StringVar(&updateReq.StorageRoot, "storage-root", updateReq.StorageRoot, `Storage root URL for the share.`) // TODO: array: updates cmd.Use = "update NAME" @@ -382,6 +384,9 @@ func newUpdate() *cobra.Command { In the case that the share name is changed, **updateShare** requires that the caller is both the share owner and a metastore admin. + If there are notebook files in the share, the __storage_root__ field cannot be + updated. + For each table that is added through this method, the share owner must also have **SELECT** privilege on the table. This privilege must be maintained indefinitely for recipients to be able to access the table. Typically, you diff --git a/cmd/workspace/system-schemas/system-schemas.go b/cmd/workspace/system-schemas/system-schemas.go index 070701d2f..3fe0580d7 100755 --- a/cmd/workspace/system-schemas/system-schemas.go +++ b/cmd/workspace/system-schemas/system-schemas.go @@ -3,8 +3,6 @@ package system_schemas import ( - "fmt" - "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -81,10 +79,7 @@ func newDisable() *cobra.Command { w := root.WorkspaceClient(ctx) disableReq.MetastoreId = args[0] - _, err = fmt.Sscan(args[1], &disableReq.SchemaName) - if err != nil { - return fmt.Errorf("invalid SCHEMA_NAME: %s", args[1]) - } + disableReq.SchemaName = args[1] err = w.SystemSchemas.Disable(ctx, disableReq) if err != nil { @@ -145,10 +140,7 @@ func newEnable() *cobra.Command { w := root.WorkspaceClient(ctx) enableReq.MetastoreId = args[0] - _, err = fmt.Sscan(args[1], &enableReq.SchemaName) - if err != nil { - return fmt.Errorf("invalid SCHEMA_NAME: %s", args[1]) - } + enableReq.SchemaName = args[1] err = w.SystemSchemas.Enable(ctx, enableReq) if err != nil { diff --git a/cmd/workspace/vector-search-indexes/vector-search-indexes.go b/cmd/workspace/vector-search-indexes/vector-search-indexes.go index 32e023d44..dff8176ea 100755 --- a/cmd/workspace/vector-search-indexes/vector-search-indexes.go +++ b/cmd/workspace/vector-search-indexes/vector-search-indexes.go @@ -42,6 +42,7 @@ func New() *cobra.Command { cmd.AddCommand(newGetIndex()) cmd.AddCommand(newListIndexes()) cmd.AddCommand(newQueryIndex()) + cmd.AddCommand(newScanIndex()) cmd.AddCommand(newSyncIndex()) cmd.AddCommand(newUpsertDataVectorIndex()) @@ -468,6 +469,76 @@ func newQueryIndex() *cobra.Command { return cmd } +// start scan-index command + +// Slice with functions to override default command behavior. +// Functions can be added from the `init()` function in manually curated files in this directory. +var scanIndexOverrides []func( + *cobra.Command, + *vectorsearch.ScanVectorIndexRequest, +) + +func newScanIndex() *cobra.Command { + cmd := &cobra.Command{} + + var scanIndexReq vectorsearch.ScanVectorIndexRequest + var scanIndexJson flags.JsonFlag + + // TODO: short flags + cmd.Flags().Var(&scanIndexJson, "json", `either inline JSON string or @path/to/file.json with request body`) + + cmd.Flags().StringVar(&scanIndexReq.LastPrimaryKey, "last-primary-key", scanIndexReq.LastPrimaryKey, `Primary key of the last entry returned in the previous scan.`) + cmd.Flags().IntVar(&scanIndexReq.NumResults, "num-results", scanIndexReq.NumResults, `Number of results to return.`) + + cmd.Use = "scan-index INDEX_NAME" + cmd.Short = `Scan an index.` + cmd.Long = `Scan an index. + + Scan the specified vector index and return the first num_results entries + after the exclusive primary_key. + + Arguments: + INDEX_NAME: Name of the vector index to scan.` + + cmd.Annotations = make(map[string]string) + + cmd.Args = func(cmd *cobra.Command, args []string) error { + check := root.ExactArgs(1) + return check(cmd, args) + } + + cmd.PreRunE = root.MustWorkspaceClient + cmd.RunE = func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + w := root.WorkspaceClient(ctx) + + if cmd.Flags().Changed("json") { + err = scanIndexJson.Unmarshal(&scanIndexReq) + if err != nil { + return err + } + } + scanIndexReq.IndexName = args[0] + + response, err := w.VectorSearchIndexes.ScanIndex(ctx, scanIndexReq) + if err != nil { + return err + } + return cmdio.Render(ctx, response) + } + + // Disable completions since they are not applicable. + // Can be overridden by manual implementation in `override.go`. + cmd.ValidArgsFunction = cobra.NoFileCompletions + + // Apply optional overrides to this command. + for _, fn := range scanIndexOverrides { + fn(cmd, &scanIndexReq) + } + + return cmd +} + // start sync-index command // Slice with functions to override default command behavior. diff --git a/go.mod b/go.mod index 6a991b0ec..1b6c9aeb3 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.21 require ( github.com/Masterminds/semver/v3 v3.2.1 // MIT github.com/briandowns/spinner v1.23.0 // Apache 2.0 - github.com/databricks/databricks-sdk-go v0.38.0 // Apache 2.0 - github.com/fatih/color v1.16.0 // MIT + github.com/databricks/databricks-sdk-go v0.41.0 // Apache 2.0 + github.com/fatih/color v1.17.0 // MIT github.com/ghodss/yaml v1.0.0 // MIT + NOTICE github.com/google/uuid v1.6.0 // BSD-3-Clause - github.com/hashicorp/go-version v1.6.0 // MPL 2.0 - github.com/hashicorp/hc-install v0.6.4 // MPL 2.0 - github.com/hashicorp/terraform-exec v0.20.0 // MPL 2.0 - github.com/hashicorp/terraform-json v0.21.0 // MPL 2.0 + github.com/hashicorp/go-version v1.7.0 // MPL 2.0 + github.com/hashicorp/hc-install v0.7.0 // MPL 2.0 + github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0 + github.com/hashicorp/terraform-json v0.22.1 // MPL 2.0 github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause github.com/mattn/go-isatty v0.0.20 // MIT github.com/nwidger/jsoncolor v0.3.2 // MIT @@ -23,17 +23,16 @@ require ( github.com/stretchr/testify v1.9.0 // MIT golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.17.0 - golang.org/x/oauth2 v0.19.0 + golang.org/x/oauth2 v0.20.0 golang.org/x/sync v0.7.0 - golang.org/x/term v0.19.0 - golang.org/x/text v0.14.0 + golang.org/x/term v0.20.0 + golang.org/x/text v0.15.0 gopkg.in/ini.v1 v1.67.0 // Apache 2.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - cloud.google.com/go/compute v1.23.4 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect @@ -52,15 +51,15 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/zclconf/go-cty v1.14.1 // indirect + github.com/zclconf/go-cty v1.14.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.19.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect diff --git a/go.sum b/go.sum index 8fe9109b5..723057ad9 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= -cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -30,8 +28,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/databricks/databricks-sdk-go v0.38.0 h1:MQhOCWTkdKItG+n6ZwcXQv9FWBVXq9fax8VSZns2e+0= -github.com/databricks/databricks-sdk-go v0.38.0/go.mod h1:Yjy1gREDLK65g4axpVbVNKYAHYE2Sqzj0AB9QWHCBVM= +github.com/databricks/databricks-sdk-go v0.41.0 h1:OyhYY+Q6+gqkWeXmpGEiacoU2RStTeWPF0x4vmqbQdc= +github.com/databricks/databricks-sdk-go v0.41.0/go.mod h1:rLIhh7DvifVLmf2QxMr/vMRGqdrTZazn8VYo4LilfCo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -42,8 +40,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -99,14 +97,14 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hc-install v0.6.4 h1:QLqlM56/+SIIGvGcfFiwMY3z5WGXT066suo/v9Km8e0= -github.com/hashicorp/hc-install v0.6.4/go.mod h1:05LWLy8TD842OtgcfBbOT0WMoInBMUSHjmDx10zuBIA= -github.com/hashicorp/terraform-exec v0.20.0 h1:DIZnPsqzPGuUnq6cH8jWcPunBfY+C+M8JyYF3vpnuEo= -github.com/hashicorp/terraform-exec v0.20.0/go.mod h1:ckKGkJWbsNqFKV1itgMnE0hY9IYf1HoiekpuN0eWoDw= -github.com/hashicorp/terraform-json v0.21.0 h1:9NQxbLNqPbEMze+S6+YluEdXgJmhQykRyRNd+zTI05U= -github.com/hashicorp/terraform-json v0.21.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hc-install v0.7.0 h1:Uu9edVqjKQxxuD28mR5TikkKDd/p55S8vzPC1659aBk= +github.com/hashicorp/hc-install v0.7.0/go.mod h1:ELmmzZlGnEcqoUMKUuykHaPCIR1sYLYX+KSggWSKZuA= +github.com/hashicorp/terraform-exec v0.21.0 h1:uNkLAe95ey5Uux6KJdua6+cv8asgILFVWkd/RG0D2XQ= +github.com/hashicorp/terraform-exec v0.21.0/go.mod h1:1PPeMYou+KDUSSeRE9szMZ/oHf4fYUmB923Wzbq1ICg= +github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= +github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -156,8 +154,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= -github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= @@ -172,8 +170,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= @@ -188,11 +186,11 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -208,14 +206,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json b/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json index 8fca7a7c4..f03ad1c2b 100644 --- a/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json +++ b/internal/bundle/bundles/deploy_then_remove_resources/databricks_template_schema.json @@ -3,6 +3,14 @@ "unique_id": { "type": "string", "description": "Unique ID for pipeline name" + }, + "spark_version": { + "type": "string", + "description": "Spark version used for job cluster" + }, + "node_type_id": { + "type": "string", + "description": "Node type id for job cluster" } } } diff --git a/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py b/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py new file mode 100644 index 000000000..4914a7436 --- /dev/null +++ b/internal/bundle/bundles/deploy_then_remove_resources/template/bar.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("hello") diff --git a/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl b/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl index e3a676770..f3be9aafd 100644 --- a/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl +++ b/internal/bundle/bundles/deploy_then_remove_resources/template/resources.yml.tmpl @@ -1,4 +1,15 @@ resources: + jobs: + foo: + name: test-bundle-job-{{.unique_id}} + tasks: + - task_key: my_notebook_task + new_cluster: + num_workers: 1 + spark_version: "{{.spark_version}}" + node_type_id: "{{.node_type_id}}" + notebook_task: + notebook_path: "./bar.py" pipelines: bar: name: test-bundle-pipeline-{{.unique_id}} diff --git a/internal/bundle/deploy_then_remove_resources_test.go b/internal/bundle/deploy_then_remove_resources_test.go index 72baf798c..66ec5c16a 100644 --- a/internal/bundle/deploy_then_remove_resources_test.go +++ b/internal/bundle/deploy_then_remove_resources_test.go @@ -5,7 +5,9 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/env" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,9 +17,12 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) w := wt.W + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) uniqueId := uuid.New().String() bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{ - "unique_id": uniqueId, + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, }) require.NoError(t, err) @@ -31,6 +36,12 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) { require.NoError(t, err) assert.Equal(t, pipeline.Name, pipelineName) + // assert job is created + jobName := "test-bundle-job-" + uniqueId + job, err := w.Jobs.GetBySettingsName(ctx, jobName) + require.NoError(t, err) + assert.Equal(t, job.Settings.Name, jobName) + // delete resources.yml err = os.Remove(filepath.Join(bundleRoot, "resources.yml")) require.NoError(t, err) @@ -43,6 +54,10 @@ func TestAccBundleDeployThenRemoveResources(t *testing.T) { _, err = w.Pipelines.GetByName(ctx, pipelineName) assert.ErrorContains(t, err, "does not exist") + // assert job is deleted + _, err = w.Jobs.GetBySettingsName(ctx, jobName) + assert.ErrorContains(t, err, "does not exist") + t.Cleanup(func() { err = destroyBundle(t, ctx, bundleRoot) require.NoError(t, err) diff --git a/internal/bundle/destroy_test.go b/internal/bundle/destroy_test.go index 43c05fbae..baccf4e6f 100644 --- a/internal/bundle/destroy_test.go +++ b/internal/bundle/destroy_test.go @@ -6,7 +6,9 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/internal" "github.com/databricks/cli/internal/acc" + "github.com/databricks/cli/libs/env" "github.com/databricks/databricks-sdk-go/apierr" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -17,9 +19,12 @@ func TestAccBundleDestroy(t *testing.T) { ctx, wt := acc.WorkspaceTest(t) w := wt.W + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) uniqueId := uuid.New().String() bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{ - "unique_id": uniqueId, + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, }) require.NoError(t, err) @@ -29,7 +34,7 @@ func TestAccBundleDestroy(t *testing.T) { _, err = os.ReadDir(snapshotsDir) assert.ErrorIs(t, err, os.ErrNotExist) - // deploy pipeline + // deploy resources err = deployBundle(t, ctx, bundleRoot) require.NoError(t, err) @@ -49,6 +54,12 @@ func TestAccBundleDestroy(t *testing.T) { require.NoError(t, err) assert.Equal(t, pipeline.Name, pipelineName) + // assert job is created + jobName := "test-bundle-job-" + uniqueId + job, err := w.Jobs.GetBySettingsName(ctx, jobName) + require.NoError(t, err) + assert.Equal(t, job.Settings.Name, jobName) + // destroy bundle err = destroyBundle(t, ctx, bundleRoot) require.NoError(t, err) @@ -57,6 +68,10 @@ func TestAccBundleDestroy(t *testing.T) { _, err = w.Pipelines.GetByName(ctx, pipelineName) assert.ErrorContains(t, err, "does not exist") + // assert job is deleted + _, err = w.Jobs.GetBySettingsName(ctx, jobName) + assert.ErrorContains(t, err, "does not exist") + // Assert snapshot file is deleted entries, err = os.ReadDir(snapshotsDir) require.NoError(t, err) diff --git a/internal/filer_test.go b/internal/filer_test.go index d333a1b70..3361de5bc 100644 --- a/internal/filer_test.go +++ b/internal/filer_test.go @@ -3,9 +3,12 @@ package internal import ( "bytes" "context" + "encoding/json" "errors" + "fmt" "io" "io/fs" + "path" "regexp" "strings" "testing" @@ -37,6 +40,36 @@ func (f filerTest) assertContents(ctx context.Context, name string, contents str assert.Equal(f, contents, body.String()) } +func (f filerTest) assertContentsJupyter(ctx context.Context, name string) { + reader, err := f.Read(ctx, name) + if !assert.NoError(f, err) { + return + } + + defer reader.Close() + + var body bytes.Buffer + _, err = io.Copy(&body, reader) + if !assert.NoError(f, err) { + return + } + + var actual map[string]any + err = json.Unmarshal(body.Bytes(), &actual) + if !assert.NoError(f, err) { + return + } + + // Since a roundtrip to the workspace changes a Jupyter notebook's payload, + // the best we can do is assert that the nbformat is correct. + assert.EqualValues(f, 4, actual["nbformat"]) +} + +func (f filerTest) assertNotExists(ctx context.Context, name string) { + _, err := f.Stat(ctx, name) + assert.ErrorIs(f, err, fs.ErrNotExist) +} + func commonFilerRecursiveDeleteTest(t *testing.T, ctx context.Context, f filer.Filer) { var err error @@ -94,6 +127,7 @@ func TestAccFilerRecursiveDelete(t *testing.T) { {"workspace files", setupWsfsFiler}, {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, + {"workspace files extensions", setupWsfsExtensionsFiler}, } { tc := testCase @@ -204,6 +238,7 @@ func TestAccFilerReadWrite(t *testing.T) { {"workspace files", setupWsfsFiler}, {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, + {"workspace files extensions", setupWsfsExtensionsFiler}, } { tc := testCase @@ -312,6 +347,7 @@ func TestAccFilerReadDir(t *testing.T) { {"workspace files", setupWsfsFiler}, {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, + {"workspace files extensions", setupWsfsExtensionsFiler}, } { tc := testCase @@ -374,6 +410,8 @@ var jupyterNotebookContent2 = ` ` func TestAccFilerWorkspaceNotebookConflict(t *testing.T) { + t.Parallel() + f, _ := setupWsfsFiler(t) ctx := context.Background() var err error @@ -420,6 +458,8 @@ func TestAccFilerWorkspaceNotebookConflict(t *testing.T) { } func TestAccFilerWorkspaceNotebookWithOverwriteFlag(t *testing.T) { + t.Parallel() + f, _ := setupWsfsFiler(t) ctx := context.Background() var err error @@ -462,3 +502,309 @@ func TestAccFilerWorkspaceNotebookWithOverwriteFlag(t *testing.T) { filerTest{t, f}.assertContents(ctx, "scalaNb", "// Databricks notebook source\n println(\"second upload\"))") filerTest{t, f}.assertContents(ctx, "jupyterNb", "# Databricks notebook source\nprint(\"Jupyter Notebook Version 2\")") } + +func TestAccFilerWorkspaceFilesExtensionsReadDir(t *testing.T) { + t.Parallel() + + files := []struct { + name string + content string + }{ + {"dir1/dir2/dir3/file.txt", "file content"}, + {"foo.py", "print('foo')"}, + {"foo.r", "print('foo')"}, + {"foo.scala", "println('foo')"}, + {"foo.sql", "SELECT 'foo'"}, + {"jupyterNb.ipynb", jupyterNotebookContent1}, + {"jupyterNb2.ipynb", jupyterNotebookContent2}, + {"pyNb.py", "# Databricks notebook source\nprint('first upload'))"}, + {"rNb.r", "# Databricks notebook source\nprint('first upload'))"}, + {"scalaNb.scala", "// Databricks notebook source\n println(\"first upload\"))"}, + {"sqlNb.sql", "-- Databricks notebook source\n SELECT \"first upload\""}, + } + + ctx := context.Background() + wf, _ := setupWsfsExtensionsFiler(t) + + for _, f := range files { + err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories) + require.NoError(t, err) + } + + // Read entries + entries, err := wf.ReadDir(ctx, ".") + require.NoError(t, err) + assert.Len(t, entries, len(files)) + names := []string{} + for _, e := range entries { + names = append(names, e.Name()) + } + assert.Equal(t, []string{ + "dir1", + "foo.py", + "foo.r", + "foo.scala", + "foo.sql", + "jupyterNb.ipynb", + "jupyterNb2.ipynb", + "pyNb.py", + "rNb.r", + "scalaNb.scala", + "sqlNb.sql", + }, names) +} + +func setupFilerWithExtensionsTest(t *testing.T) filer.Filer { + files := []struct { + name string + content string + }{ + {"foo.py", "# Databricks notebook source\nprint('first upload'))"}, + {"bar.py", "print('foo')"}, + {"jupyter.ipynb", jupyterNotebookContent1}, + {"pretender", "not a notebook"}, + {"dir/file.txt", "file content"}, + {"scala-notebook.scala", "// Databricks notebook source\nprintln('first upload')"}, + } + + ctx := context.Background() + wf, _ := setupWsfsExtensionsFiler(t) + + for _, f := range files { + err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories) + require.NoError(t, err) + } + + return wf +} + +func TestAccFilerWorkspaceFilesExtensionsRead(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf := setupFilerWithExtensionsTest(t) + + // Read contents of test fixtures as a sanity check. + filerTest{t, wf}.assertContents(ctx, "foo.py", "# Databricks notebook source\nprint('first upload'))") + filerTest{t, wf}.assertContents(ctx, "bar.py", "print('foo')") + filerTest{t, wf}.assertContentsJupyter(ctx, "jupyter.ipynb") + filerTest{t, wf}.assertContents(ctx, "dir/file.txt", "file content") + filerTest{t, wf}.assertContents(ctx, "scala-notebook.scala", "// Databricks notebook source\nprintln('first upload')") + filerTest{t, wf}.assertContents(ctx, "pretender", "not a notebook") + + // Read non-existent file + _, err := wf.Read(ctx, "non-existent.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Ensure we do not read a regular file as a notebook + _, err = wf.Read(ctx, "pretender.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + _, err = wf.Read(ctx, "pretender.ipynb") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Read directory + _, err = wf.Read(ctx, "dir") + assert.ErrorIs(t, err, fs.ErrInvalid) + + // Ensure we do not read a Scala notebook as a Python notebook + _, err = wf.Read(ctx, "scala-notebook.py") + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestAccFilerWorkspaceFilesExtensionsDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf := setupFilerWithExtensionsTest(t) + + // Delete notebook + err := wf.Delete(ctx, "foo.py") + require.NoError(t, err) + filerTest{t, wf}.assertNotExists(ctx, "foo.py") + + // Delete file + err = wf.Delete(ctx, "bar.py") + require.NoError(t, err) + filerTest{t, wf}.assertNotExists(ctx, "bar.py") + + // Delete jupyter notebook + err = wf.Delete(ctx, "jupyter.ipynb") + require.NoError(t, err) + filerTest{t, wf}.assertNotExists(ctx, "jupyter.ipynb") + + // Delete non-existent file + err = wf.Delete(ctx, "non-existent.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Ensure we do not delete a file as a notebook + err = wf.Delete(ctx, "pretender.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Ensure we do not delete a Scala notebook as a Python notebook + _, err = wf.Read(ctx, "scala-notebook.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Delete directory + err = wf.Delete(ctx, "dir") + assert.ErrorIs(t, err, fs.ErrInvalid) + + // Delete directory recursively + err = wf.Delete(ctx, "dir", filer.DeleteRecursively) + require.NoError(t, err) + filerTest{t, wf}.assertNotExists(ctx, "dir") +} + +func TestAccFilerWorkspaceFilesExtensionsStat(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf := setupFilerWithExtensionsTest(t) + + // Stat on a notebook + info, err := wf.Stat(ctx, "foo.py") + require.NoError(t, err) + assert.Equal(t, "foo.py", info.Name()) + assert.False(t, info.IsDir()) + + // Stat on a file + info, err = wf.Stat(ctx, "bar.py") + require.NoError(t, err) + assert.Equal(t, "bar.py", info.Name()) + assert.False(t, info.IsDir()) + + // Stat on a Jupyter notebook + info, err = wf.Stat(ctx, "jupyter.ipynb") + require.NoError(t, err) + assert.Equal(t, "jupyter.ipynb", info.Name()) + assert.False(t, info.IsDir()) + + // Stat on a directory + info, err = wf.Stat(ctx, "dir") + require.NoError(t, err) + assert.Equal(t, "dir", info.Name()) + assert.True(t, info.IsDir()) + + // Stat on a non-existent file + _, err = wf.Stat(ctx, "non-existent.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Ensure we do not stat a file as a notebook + _, err = wf.Stat(ctx, "pretender.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Ensure we do not stat a Scala notebook as a Python notebook + _, err = wf.Stat(ctx, "scala-notebook.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + + _, err = wf.Stat(ctx, "pretender.ipynb") + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestAccFilerWorkspaceFilesExtensionsErrorsOnDupName(t *testing.T) { + t.Parallel() + + tcases := []struct { + files []struct{ name, content string } + name string + }{ + { + name: "python", + files: []struct{ name, content string }{ + {"foo.py", "print('foo')"}, + {"foo.py", "# Databricks notebook source\nprint('foo')"}, + }, + }, + { + name: "r", + files: []struct{ name, content string }{ + {"foo.r", "print('foo')"}, + {"foo.r", "# Databricks notebook source\nprint('foo')"}, + }, + }, + { + name: "sql", + files: []struct{ name, content string }{ + {"foo.sql", "SELECT 'foo'"}, + {"foo.sql", "-- Databricks notebook source\nSELECT 'foo'"}, + }, + }, + { + name: "scala", + files: []struct{ name, content string }{ + {"foo.scala", "println('foo')"}, + {"foo.scala", "// Databricks notebook source\nprintln('foo')"}, + }, + }, + // We don't need to test this for ipynb notebooks. The import API + // fails when the file extension is .ipynb but the content is not a + // valid juptyer notebook. + } + + for i := range tcases { + tc := tcases[i] + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf, tmpDir := setupWsfsExtensionsFiler(t) + + for _, f := range tc.files { + err := wf.Write(ctx, f.name, strings.NewReader(f.content), filer.CreateParentDirectories) + require.NoError(t, err) + } + + _, err := wf.ReadDir(ctx, ".") + assert.ErrorAs(t, err, &filer.DuplicatePathError{}) + assert.ErrorContains(t, err, fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both NOTEBOOK at %s and FILE at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", path.Join(tmpDir, "foo"), path.Join(tmpDir, tc.files[0].name), tc.files[0].name)) + }) + } + +} + +func TestAccWorkspaceFilesExtensionsDirectoriesAreNotNotebooks(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf, _ := setupWsfsExtensionsFiler(t) + + // Create a directory with an extension + err := wf.Mkdir(ctx, "foo") + require.NoError(t, err) + + // Reading foo.py should fail. foo is a directory, not a notebook. + _, err = wf.Read(ctx, "foo.py") + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestAccWorkspaceFilesExtensions_ExportFormatIsPreserved(t *testing.T) { + t.Parallel() + + ctx := context.Background() + wf, _ := setupWsfsExtensionsFiler(t) + + // Case 1: Source Notebook + err := wf.Write(ctx, "foo.py", strings.NewReader("# Databricks notebook source\nprint('foo')")) + require.NoError(t, err) + + // The source notebook should exist but not the Jupyter notebook + filerTest{t, wf}.assertContents(ctx, "foo.py", "# Databricks notebook source\nprint('foo')") + _, err = wf.Stat(ctx, "foo.ipynb") + assert.ErrorIs(t, err, fs.ErrNotExist) + _, err = wf.Read(ctx, "foo.ipynb") + assert.ErrorIs(t, err, fs.ErrNotExist) + err = wf.Delete(ctx, "foo.ipynb") + assert.ErrorIs(t, err, fs.ErrNotExist) + + // Case 2: Jupyter Notebook + err = wf.Write(ctx, "bar.ipynb", strings.NewReader(jupyterNotebookContent1)) + require.NoError(t, err) + + // The Jupyter notebook should exist but not the source notebook + filerTest{t, wf}.assertContentsJupyter(ctx, "bar.ipynb") + _, err = wf.Stat(ctx, "bar.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + _, err = wf.Read(ctx, "bar.py") + assert.ErrorIs(t, err, fs.ErrNotExist) + err = wf.Delete(ctx, "bar.py") + assert.ErrorIs(t, err, fs.ErrNotExist) +} diff --git a/internal/helpers.go b/internal/helpers.go index 49dc9f4ca..3923e7e1e 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -559,6 +559,17 @@ func setupWsfsFiler(t *testing.T) (filer.Filer, string) { return f, tmpdir } +func setupWsfsExtensionsFiler(t *testing.T) (filer.Filer, string) { + t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) + + w := databricks.Must(databricks.NewWorkspaceClient()) + tmpdir := TemporaryWorkspaceDir(t, w) + f, err := filer.NewWorkspaceFilesExtensionsClient(w, tmpdir) + require.NoError(t, err) + + return f, tmpdir +} + func setupDbfsFiler(t *testing.T) (filer.Filer, string) { t.Log(GetEnvOrSkipTest(t, "CLOUD_ENV")) diff --git a/internal/sync_test.go b/internal/sync_test.go index f970a7ce0..4021e6490 100644 --- a/internal/sync_test.go +++ b/internal/sync_test.go @@ -313,7 +313,7 @@ func TestAccSyncNestedFolderSync(t *testing.T) { assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"}) assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"}) assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("dir1/dir2/dir3/foo.txt"))) + assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/dir2/dir3/foo.txt")) // delete f.Remove(t) @@ -374,7 +374,7 @@ func TestAccSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) { assertSync.remoteDirContent(ctx, "dir1", []string{"a b+c"}) assertSync.remoteDirContent(ctx, "dir1/a b+c", []string{"c+d e"}) assertSync.remoteDirContent(ctx, "dir1/a b+c/c+d e", []string{"e+f g#i.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("dir1/a b+c/c+d e/e+f g#i.txt"))) + assertSync.snapshotContains(append(repoFiles, ".gitignore", "dir1/a b+c/c+d e/e+f g#i.txt")) // delete f.Remove(t) @@ -404,7 +404,7 @@ func TestAccSyncIncrementalFileOverwritesFolder(t *testing.T) { assertSync.waitForCompletionMarker() assertSync.remoteDirContent(ctx, "", append(repoFiles, ".gitignore", "foo")) assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"}) - assertSync.snapshotContains(append(repoFiles, ".gitignore", filepath.FromSlash("foo/bar.txt"))) + assertSync.snapshotContains(append(repoFiles, ".gitignore", "foo/bar.txt")) // delete foo/bar.txt f.Remove(t) diff --git a/libs/auth/cache/cache.go b/libs/auth/cache/cache.go index 5511c1922..097353e74 100644 --- a/libs/auth/cache/cache.go +++ b/libs/auth/cache/cache.go @@ -1,106 +1,26 @@ package cache import ( - "encoding/json" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" + "context" "golang.org/x/oauth2" ) -const ( - // where the token cache is stored - tokenCacheFile = ".databricks/token-cache.json" - - // only the owner of the file has full execute, read, and write access - ownerExecReadWrite = 0o700 - - // only the owner of the file has full read and write access - ownerReadWrite = 0o600 - - // format versioning leaves some room for format improvement - tokenCacheVersion = 1 -) - -var ErrNotConfigured = errors.New("databricks OAuth is not configured for this host") - -// this implementation requires the calling code to do a machine-wide lock, -// otherwise the file might get corrupt. -type TokenCache struct { - Version int `json:"version"` - Tokens map[string]*oauth2.Token `json:"tokens"` - - fileLocation string +type TokenCache interface { + Store(key string, t *oauth2.Token) error + Lookup(key string) (*oauth2.Token, error) } -func (c *TokenCache) Store(key string, t *oauth2.Token) error { - err := c.load() - if errors.Is(err, fs.ErrNotExist) { - dir := filepath.Dir(c.fileLocation) - err = os.MkdirAll(dir, ownerExecReadWrite) - if err != nil { - return fmt.Errorf("mkdir: %w", err) - } - } else if err != nil { - return fmt.Errorf("load: %w", err) - } - c.Version = tokenCacheVersion - if c.Tokens == nil { - c.Tokens = map[string]*oauth2.Token{} - } - c.Tokens[key] = t - raw, err := json.MarshalIndent(c, "", " ") - if err != nil { - return fmt.Errorf("marshal: %w", err) - } - return os.WriteFile(c.fileLocation, raw, ownerReadWrite) +var tokenCache int + +func WithTokenCache(ctx context.Context, c TokenCache) context.Context { + return context.WithValue(ctx, &tokenCache, c) } -func (c *TokenCache) Lookup(key string) (*oauth2.Token, error) { - err := c.load() - if errors.Is(err, fs.ErrNotExist) { - return nil, ErrNotConfigured - } else if err != nil { - return nil, fmt.Errorf("load: %w", err) - } - t, ok := c.Tokens[key] +func GetTokenCache(ctx context.Context) TokenCache { + c, ok := ctx.Value(&tokenCache).(TokenCache) if !ok { - return nil, ErrNotConfigured + return &FileTokenCache{} } - return t, nil -} - -func (c *TokenCache) location() (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("home: %w", err) - } - return filepath.Join(home, tokenCacheFile), nil -} - -func (c *TokenCache) load() error { - loc, err := c.location() - if err != nil { - return err - } - c.fileLocation = loc - raw, err := os.ReadFile(loc) - if err != nil { - return fmt.Errorf("read: %w", err) - } - err = json.Unmarshal(raw, c) - if err != nil { - return fmt.Errorf("parse: %w", err) - } - if c.Version != tokenCacheVersion { - // in the later iterations we could do state upgraders, - // so that we transform token cache from v1 to v2 without - // losing the tokens and asking the user to re-authenticate. - return fmt.Errorf("needs version %d, got version %d", - tokenCacheVersion, c.Version) - } - return nil + return c } diff --git a/libs/auth/cache/file.go b/libs/auth/cache/file.go new file mode 100644 index 000000000..38dfea9f2 --- /dev/null +++ b/libs/auth/cache/file.go @@ -0,0 +1,108 @@ +package cache + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "golang.org/x/oauth2" +) + +const ( + // where the token cache is stored + tokenCacheFile = ".databricks/token-cache.json" + + // only the owner of the file has full execute, read, and write access + ownerExecReadWrite = 0o700 + + // only the owner of the file has full read and write access + ownerReadWrite = 0o600 + + // format versioning leaves some room for format improvement + tokenCacheVersion = 1 +) + +var ErrNotConfigured = errors.New("databricks OAuth is not configured for this host") + +// this implementation requires the calling code to do a machine-wide lock, +// otherwise the file might get corrupt. +type FileTokenCache struct { + Version int `json:"version"` + Tokens map[string]*oauth2.Token `json:"tokens"` + + fileLocation string +} + +func (c *FileTokenCache) Store(key string, t *oauth2.Token) error { + err := c.load() + if errors.Is(err, fs.ErrNotExist) { + dir := filepath.Dir(c.fileLocation) + err = os.MkdirAll(dir, ownerExecReadWrite) + if err != nil { + return fmt.Errorf("mkdir: %w", err) + } + } else if err != nil { + return fmt.Errorf("load: %w", err) + } + c.Version = tokenCacheVersion + if c.Tokens == nil { + c.Tokens = map[string]*oauth2.Token{} + } + c.Tokens[key] = t + raw, err := json.MarshalIndent(c, "", " ") + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + return os.WriteFile(c.fileLocation, raw, ownerReadWrite) +} + +func (c *FileTokenCache) Lookup(key string) (*oauth2.Token, error) { + err := c.load() + if errors.Is(err, fs.ErrNotExist) { + return nil, ErrNotConfigured + } else if err != nil { + return nil, fmt.Errorf("load: %w", err) + } + t, ok := c.Tokens[key] + if !ok { + return nil, ErrNotConfigured + } + return t, nil +} + +func (c *FileTokenCache) location() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("home: %w", err) + } + return filepath.Join(home, tokenCacheFile), nil +} + +func (c *FileTokenCache) load() error { + loc, err := c.location() + if err != nil { + return err + } + c.fileLocation = loc + raw, err := os.ReadFile(loc) + if err != nil { + return fmt.Errorf("read: %w", err) + } + err = json.Unmarshal(raw, c) + if err != nil { + return fmt.Errorf("parse: %w", err) + } + if c.Version != tokenCacheVersion { + // in the later iterations we could do state upgraders, + // so that we transform token cache from v1 to v2 without + // losing the tokens and asking the user to re-authenticate. + return fmt.Errorf("needs version %d, got version %d", + tokenCacheVersion, c.Version) + } + return nil +} + +var _ TokenCache = (*FileTokenCache)(nil) diff --git a/libs/auth/cache/cache_test.go b/libs/auth/cache/file_test.go similarity index 93% rename from libs/auth/cache/cache_test.go rename to libs/auth/cache/file_test.go index 6529882c7..3e4aae36f 100644 --- a/libs/auth/cache/cache_test.go +++ b/libs/auth/cache/file_test.go @@ -27,7 +27,7 @@ func setup(t *testing.T) string { func TestStoreAndLookup(t *testing.T) { setup(t) - c := &TokenCache{} + c := &FileTokenCache{} err := c.Store("x", &oauth2.Token{ AccessToken: "abc", }) @@ -38,7 +38,7 @@ func TestStoreAndLookup(t *testing.T) { }) require.NoError(t, err) - l := &TokenCache{} + l := &FileTokenCache{} tok, err := l.Lookup("x") require.NoError(t, err) assert.Equal(t, "abc", tok.AccessToken) @@ -50,7 +50,7 @@ func TestStoreAndLookup(t *testing.T) { func TestNoCacheFileReturnsErrNotConfigured(t *testing.T) { setup(t) - l := &TokenCache{} + l := &FileTokenCache{} _, err := l.Lookup("x") assert.Equal(t, ErrNotConfigured, err) } @@ -63,7 +63,7 @@ func TestLoadCorruptFile(t *testing.T) { err = os.WriteFile(f, []byte("abc"), ownerExecReadWrite) require.NoError(t, err) - l := &TokenCache{} + l := &FileTokenCache{} _, err = l.Lookup("x") assert.EqualError(t, err, "load: parse: invalid character 'a' looking for beginning of value") } @@ -76,14 +76,14 @@ func TestLoadWrongVersion(t *testing.T) { err = os.WriteFile(f, []byte(`{"version": 823, "things": []}`), ownerExecReadWrite) require.NoError(t, err) - l := &TokenCache{} + l := &FileTokenCache{} _, err = l.Lookup("x") assert.EqualError(t, err, "load: needs version 1, got version 823") } func TestDevNull(t *testing.T) { t.Setenv(homeEnvVar, "/dev/null") - l := &TokenCache{} + l := &FileTokenCache{} _, err := l.Lookup("x") // macOS/Linux: load: read: open /dev/null/.databricks/token-cache.json: // windows: databricks OAuth is not configured for this host @@ -95,7 +95,7 @@ func TestStoreOnDev(t *testing.T) { t.SkipNow() } t.Setenv(homeEnvVar, "/dev") - c := &TokenCache{} + c := &FileTokenCache{} err := c.Store("x", &oauth2.Token{ AccessToken: "abc", }) diff --git a/libs/auth/cache/in_memory.go b/libs/auth/cache/in_memory.go new file mode 100644 index 000000000..469d45575 --- /dev/null +++ b/libs/auth/cache/in_memory.go @@ -0,0 +1,26 @@ +package cache + +import ( + "golang.org/x/oauth2" +) + +type InMemoryTokenCache struct { + Tokens map[string]*oauth2.Token +} + +// Lookup implements TokenCache. +func (i *InMemoryTokenCache) Lookup(key string) (*oauth2.Token, error) { + token, ok := i.Tokens[key] + if !ok { + return nil, ErrNotConfigured + } + return token, nil +} + +// Store implements TokenCache. +func (i *InMemoryTokenCache) Store(key string, t *oauth2.Token) error { + i.Tokens[key] = t + return nil +} + +var _ TokenCache = (*InMemoryTokenCache)(nil) diff --git a/libs/auth/cache/in_memory_test.go b/libs/auth/cache/in_memory_test.go new file mode 100644 index 000000000..d8394d3b2 --- /dev/null +++ b/libs/auth/cache/in_memory_test.go @@ -0,0 +1,44 @@ +package cache + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/oauth2" +) + +func TestInMemoryCacheHit(t *testing.T) { + token := &oauth2.Token{ + AccessToken: "abc", + } + c := &InMemoryTokenCache{ + Tokens: map[string]*oauth2.Token{ + "key": token, + }, + } + res, err := c.Lookup("key") + assert.Equal(t, res, token) + assert.NoError(t, err) +} + +func TestInMemoryCacheMiss(t *testing.T) { + c := &InMemoryTokenCache{ + Tokens: map[string]*oauth2.Token{}, + } + _, err := c.Lookup("key") + assert.ErrorIs(t, err, ErrNotConfigured) +} + +func TestInMemoryCacheStore(t *testing.T) { + token := &oauth2.Token{ + AccessToken: "abc", + } + c := &InMemoryTokenCache{ + Tokens: map[string]*oauth2.Token{}, + } + err := c.Store("key", token) + assert.NoError(t, err) + res, err := c.Lookup("key") + assert.Equal(t, res, token) + assert.NoError(t, err) +} diff --git a/libs/auth/oauth.go b/libs/auth/oauth.go index 4ce0d4def..1f3e032de 100644 --- a/libs/auth/oauth.go +++ b/libs/auth/oauth.go @@ -20,6 +20,20 @@ import ( "golang.org/x/oauth2/authhandler" ) +var apiClientForOauth int + +func WithApiClientForOAuth(ctx context.Context, c *httpclient.ApiClient) context.Context { + return context.WithValue(ctx, &apiClientForOauth, c) +} + +func GetApiClientForOAuth(ctx context.Context) *httpclient.ApiClient { + c, ok := ctx.Value(&apiClientForOauth).(*httpclient.ApiClient) + if !ok { + return httpclient.NewApiClient(httpclient.ClientConfig{}) + } + return c +} + const ( // these values are predefined by Databricks as a public client // and is specific to this application only. Using these values @@ -28,7 +42,7 @@ const ( appRedirectAddr = "localhost:8020" // maximum amount of time to acquire listener on appRedirectAddr - DefaultTimeout = 45 * time.Second + listenerTimeout = 45 * time.Second ) var ( // Databricks SDK API: `databricks OAuth is not` will be checked for presence @@ -42,14 +56,13 @@ type PersistentAuth struct { AccountID string http *httpclient.ApiClient - cache tokenCache + cache cache.TokenCache ln net.Listener browser func(string) error } -type tokenCache interface { - Store(key string, t *oauth2.Token) error - Lookup(key string) (*oauth2.Token, error) +func (a *PersistentAuth) SetApiClient(h *httpclient.ApiClient) { + a.http = h } func (a *PersistentAuth) Load(ctx context.Context) (*oauth2.Token, error) { @@ -136,12 +149,10 @@ func (a *PersistentAuth) init(ctx context.Context) error { return ErrFetchCredentials } if a.http == nil { - a.http = httpclient.NewApiClient(httpclient.ClientConfig{ - // noop - }) + a.http = GetApiClientForOAuth(ctx) } if a.cache == nil { - a.cache = &cache.TokenCache{} + a.cache = cache.GetTokenCache(ctx) } if a.browser == nil { a.browser = browser.OpenURL @@ -149,7 +160,7 @@ func (a *PersistentAuth) init(ctx context.Context) error { // try acquire listener, which we also use as a machine-local // exclusive lock to prevent token cache corruption in the scope // of developer machine, where this command runs. - listener, err := retries.Poll(ctx, DefaultTimeout, + listener, err := retries.Poll(ctx, listenerTimeout, func() (*net.Listener, *retries.Err) { var lc net.ListenConfig l, err := lc.Listen(ctx, "tcp", appRedirectAddr) diff --git a/libs/databrickscfg/loader_test.go b/libs/databrickscfg/loader_test.go index 4525115e0..c42fcdbdd 100644 --- a/libs/databrickscfg/loader_test.go +++ b/libs/databrickscfg/loader_test.go @@ -68,7 +68,7 @@ func TestLoaderErrorsOnInvalidFile(t *testing.T) { Loaders: []config.Loader{ ResolveProfileFromHost, }, - ConfigFile: "testdata/badcfg", + ConfigFile: "profile/testdata/badcfg", Host: "https://default", } @@ -81,7 +81,7 @@ func TestLoaderSkipsNoMatchingHost(t *testing.T) { Loaders: []config.Loader{ ResolveProfileFromHost, }, - ConfigFile: "testdata/databrickscfg", + ConfigFile: "profile/testdata/databrickscfg", Host: "https://noneofthehostsmatch", } @@ -95,7 +95,7 @@ func TestLoaderMatchingHost(t *testing.T) { Loaders: []config.Loader{ ResolveProfileFromHost, }, - ConfigFile: "testdata/databrickscfg", + ConfigFile: "profile/testdata/databrickscfg", Host: "https://default", } @@ -110,7 +110,7 @@ func TestLoaderMatchingHostWithQuery(t *testing.T) { Loaders: []config.Loader{ ResolveProfileFromHost, }, - ConfigFile: "testdata/databrickscfg", + ConfigFile: "profile/testdata/databrickscfg", Host: "https://query/?foo=bar", } @@ -125,7 +125,7 @@ func TestLoaderErrorsOnMultipleMatches(t *testing.T) { Loaders: []config.Loader{ ResolveProfileFromHost, }, - ConfigFile: "testdata/databrickscfg", + ConfigFile: "profile/testdata/databrickscfg", Host: "https://foo/bar", } diff --git a/libs/databrickscfg/ops_test.go b/libs/databrickscfg/ops_test.go index 233555fe2..3ea92024c 100644 --- a/libs/databrickscfg/ops_test.go +++ b/libs/databrickscfg/ops_test.go @@ -30,7 +30,7 @@ func TestLoadOrCreate_NotAllowed(t *testing.T) { } func TestLoadOrCreate_Bad(t *testing.T) { - path := "testdata/badcfg" + path := "profile/testdata/badcfg" file, err := loadOrCreateConfigFile(path) assert.Error(t, err) assert.Nil(t, file) @@ -40,7 +40,7 @@ func TestMatchOrCreateSection_Direct(t *testing.T) { cfg := &config.Config{ Profile: "query", } - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() @@ -54,7 +54,7 @@ func TestMatchOrCreateSection_AccountID(t *testing.T) { cfg := &config.Config{ AccountID: "abc", } - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() @@ -68,7 +68,7 @@ func TestMatchOrCreateSection_NormalizeHost(t *testing.T) { cfg := &config.Config{ Host: "https://query/?o=abracadabra", } - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() @@ -80,7 +80,7 @@ func TestMatchOrCreateSection_NormalizeHost(t *testing.T) { func TestMatchOrCreateSection_NoProfileOrHost(t *testing.T) { cfg := &config.Config{} - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() @@ -92,7 +92,7 @@ func TestMatchOrCreateSection_MultipleProfiles(t *testing.T) { cfg := &config.Config{ Host: "https://foo", } - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() @@ -105,7 +105,7 @@ func TestMatchOrCreateSection_NewProfile(t *testing.T) { Host: "https://bar", Profile: "delirium", } - file, err := loadOrCreateConfigFile("testdata/databrickscfg") + file, err := loadOrCreateConfigFile("profile/testdata/databrickscfg") assert.NoError(t, err) ctx := context.Background() diff --git a/libs/databrickscfg/profile/context.go b/libs/databrickscfg/profile/context.go new file mode 100644 index 000000000..fa4d2ad8a --- /dev/null +++ b/libs/databrickscfg/profile/context.go @@ -0,0 +1,17 @@ +package profile + +import "context" + +var profiler int + +func WithProfiler(ctx context.Context, p Profiler) context.Context { + return context.WithValue(ctx, &profiler, p) +} + +func GetProfiler(ctx context.Context) Profiler { + p, ok := ctx.Value(&profiler).(Profiler) + if !ok { + return DefaultProfiler + } + return p +} diff --git a/libs/databrickscfg/profile/file.go b/libs/databrickscfg/profile/file.go new file mode 100644 index 000000000..1b743014e --- /dev/null +++ b/libs/databrickscfg/profile/file.go @@ -0,0 +1,100 @@ +package profile + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path/filepath" + "strings" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/config" + "github.com/spf13/cobra" +) + +type FileProfilerImpl struct{} + +func (f FileProfilerImpl) getPath(ctx context.Context, replaceHomeDirWithTilde bool) (string, error) { + configFile := env.Get(ctx, "DATABRICKS_CONFIG_FILE") + if configFile == "" { + configFile = "~/.databrickscfg" + } + if !replaceHomeDirWithTilde { + return configFile, nil + } + homedir, err := env.UserHomeDir(ctx) + if err != nil { + return "", err + } + configFile = strings.Replace(configFile, homedir, "~", 1) + return configFile, nil +} + +// Get the path to the .databrickscfg file, falling back to the default in the current user's home directory. +func (f FileProfilerImpl) GetPath(ctx context.Context) (string, error) { + fp, err := f.getPath(ctx, true) + if err != nil { + return "", err + } + return filepath.Clean(fp), nil +} + +var ErrNoConfiguration = errors.New("no configuration file found") + +func (f FileProfilerImpl) Get(ctx context.Context) (*config.File, error) { + path, err := f.getPath(ctx, false) + if err != nil { + return nil, fmt.Errorf("cannot determine Databricks config file path: %w", err) + } + if strings.HasPrefix(path, "~") { + homedir, err := env.UserHomeDir(ctx) + if err != nil { + return nil, err + } + path = filepath.Join(homedir, path[1:]) + } + configFile, err := config.LoadFile(path) + if errors.Is(err, fs.ErrNotExist) { + // downstreams depend on ErrNoConfiguration. TODO: expose this error through SDK + return nil, fmt.Errorf("%w at %s; please create one by running 'databricks configure'", ErrNoConfiguration, path) + } else if err != nil { + return nil, err + } + return configFile, nil +} + +func (f FileProfilerImpl) LoadProfiles(ctx context.Context, fn ProfileMatchFunction) (profiles Profiles, err error) { + file, err := f.Get(ctx) + if err != nil { + return nil, fmt.Errorf("cannot load Databricks config file: %w", err) + } + + // Iterate over sections and collect matching profiles. + for _, v := range file.Sections() { + all := v.KeysHash() + host, ok := all["host"] + if !ok { + // invalid profile + continue + } + profile := Profile{ + Name: v.Name(), + Host: host, + AccountID: all["account_id"], + } + if fn(profile) { + profiles = append(profiles, profile) + } + } + + return +} + +func ProfileCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + profiles, err := DefaultProfiler.LoadProfiles(cmd.Context(), MatchAllProfiles) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + return profiles.Names(), cobra.ShellCompDirectiveNoFileComp +} diff --git a/libs/databrickscfg/profiles_test.go b/libs/databrickscfg/profile/file_test.go similarity index 82% rename from libs/databrickscfg/profiles_test.go rename to libs/databrickscfg/profile/file_test.go index 33a5c9dfd..8e5cfefc0 100644 --- a/libs/databrickscfg/profiles_test.go +++ b/libs/databrickscfg/profile/file_test.go @@ -1,4 +1,4 @@ -package databrickscfg +package profile import ( "context" @@ -32,7 +32,8 @@ func TestLoadProfilesReturnsHomedirAsTilde(t *testing.T) { ctx := context.Background() ctx = env.WithUserHomeDir(ctx, "testdata") ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg") - file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true }) + profiler := FileProfilerImpl{} + file, err := profiler.GetPath(ctx) require.NoError(t, err) require.Equal(t, filepath.Clean("~/databrickscfg"), file) } @@ -41,7 +42,8 @@ func TestLoadProfilesReturnsHomedirAsTildeExoticFile(t *testing.T) { ctx := context.Background() ctx = env.WithUserHomeDir(ctx, "testdata") ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "~/databrickscfg") - file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true }) + profiler := FileProfilerImpl{} + file, err := profiler.GetPath(ctx) require.NoError(t, err) require.Equal(t, filepath.Clean("~/databrickscfg"), file) } @@ -49,7 +51,8 @@ func TestLoadProfilesReturnsHomedirAsTildeExoticFile(t *testing.T) { func TestLoadProfilesReturnsHomedirAsTildeDefaultFile(t *testing.T) { ctx := context.Background() ctx = env.WithUserHomeDir(ctx, "testdata/sample-home") - file, _, err := LoadProfiles(ctx, func(p Profile) bool { return true }) + profiler := FileProfilerImpl{} + file, err := profiler.GetPath(ctx) require.NoError(t, err) require.Equal(t, filepath.Clean("~/.databrickscfg"), file) } @@ -57,14 +60,16 @@ func TestLoadProfilesReturnsHomedirAsTildeDefaultFile(t *testing.T) { func TestLoadProfilesNoConfiguration(t *testing.T) { ctx := context.Background() ctx = env.WithUserHomeDir(ctx, "testdata") - _, _, err := LoadProfiles(ctx, func(p Profile) bool { return true }) + profiler := FileProfilerImpl{} + _, err := profiler.LoadProfiles(ctx, MatchAllProfiles) require.ErrorIs(t, err, ErrNoConfiguration) } func TestLoadProfilesMatchWorkspace(t *testing.T) { ctx := context.Background() ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg") - _, profiles, err := LoadProfiles(ctx, MatchWorkspaceProfiles) + profiler := FileProfilerImpl{} + profiles, err := profiler.LoadProfiles(ctx, MatchWorkspaceProfiles) require.NoError(t, err) assert.Equal(t, []string{"DEFAULT", "query", "foo1", "foo2"}, profiles.Names()) } @@ -72,7 +77,8 @@ func TestLoadProfilesMatchWorkspace(t *testing.T) { func TestLoadProfilesMatchAccount(t *testing.T) { ctx := context.Background() ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./testdata/databrickscfg") - _, profiles, err := LoadProfiles(ctx, MatchAccountProfiles) + profiler := FileProfilerImpl{} + profiles, err := profiler.LoadProfiles(ctx, MatchAccountProfiles) require.NoError(t, err) assert.Equal(t, []string{"acc"}, profiles.Names()) } diff --git a/libs/databrickscfg/profile/in_memory.go b/libs/databrickscfg/profile/in_memory.go new file mode 100644 index 000000000..902ae42e6 --- /dev/null +++ b/libs/databrickscfg/profile/in_memory.go @@ -0,0 +1,25 @@ +package profile + +import "context" + +type InMemoryProfiler struct { + Profiles Profiles +} + +// GetPath implements Profiler. +func (i InMemoryProfiler) GetPath(context.Context) (string, error) { + return "", nil +} + +// LoadProfiles implements Profiler. +func (i InMemoryProfiler) LoadProfiles(ctx context.Context, f ProfileMatchFunction) (Profiles, error) { + res := make(Profiles, 0) + for _, p := range i.Profiles { + if f(p) { + res = append(res, p) + } + } + return res, nil +} + +var _ Profiler = InMemoryProfiler{} diff --git a/libs/databrickscfg/profile/profile.go b/libs/databrickscfg/profile/profile.go new file mode 100644 index 000000000..510e5c9e5 --- /dev/null +++ b/libs/databrickscfg/profile/profile.go @@ -0,0 +1,49 @@ +package profile + +import ( + "strings" + + "github.com/databricks/databricks-sdk-go/config" +) + +// Profile holds a subset of the keys in a databrickscfg profile. +// It should only be used for prompting and filtering. +// Use its name to construct a config.Config. +type Profile struct { + Name string + Host string + AccountID string +} + +func (p Profile) Cloud() string { + cfg := config.Config{Host: p.Host} + switch { + case cfg.IsAws(): + return "AWS" + case cfg.IsAzure(): + return "Azure" + case cfg.IsGcp(): + return "GCP" + default: + return "" + } +} + +type Profiles []Profile + +// SearchCaseInsensitive implements the promptui.Searcher interface. +// This allows the user to immediately starting typing to narrow down the list. +func (p Profiles) SearchCaseInsensitive(input string, index int) bool { + input = strings.ToLower(input) + name := strings.ToLower(p[index].Name) + host := strings.ToLower(p[index].Host) + return strings.Contains(name, input) || strings.Contains(host, input) +} + +func (p Profiles) Names() []string { + names := make([]string, len(p)) + for i, v := range p { + names[i] = v.Name + } + return names +} diff --git a/libs/databrickscfg/profile/profiler.go b/libs/databrickscfg/profile/profiler.go new file mode 100644 index 000000000..c0a549256 --- /dev/null +++ b/libs/databrickscfg/profile/profiler.go @@ -0,0 +1,32 @@ +package profile + +import ( + "context" +) + +type ProfileMatchFunction func(Profile) bool + +func MatchWorkspaceProfiles(p Profile) bool { + return p.AccountID == "" +} + +func MatchAccountProfiles(p Profile) bool { + return p.Host != "" && p.AccountID != "" +} + +func MatchAllProfiles(p Profile) bool { + return true +} + +func WithName(name string) ProfileMatchFunction { + return func(p Profile) bool { + return p.Name == name + } +} + +type Profiler interface { + LoadProfiles(context.Context, ProfileMatchFunction) (Profiles, error) + GetPath(context.Context) (string, error) +} + +var DefaultProfiler = FileProfilerImpl{} diff --git a/libs/databrickscfg/testdata/badcfg b/libs/databrickscfg/profile/testdata/badcfg similarity index 100% rename from libs/databrickscfg/testdata/badcfg rename to libs/databrickscfg/profile/testdata/badcfg diff --git a/libs/databrickscfg/testdata/databrickscfg b/libs/databrickscfg/profile/testdata/databrickscfg similarity index 100% rename from libs/databrickscfg/testdata/databrickscfg rename to libs/databrickscfg/profile/testdata/databrickscfg diff --git a/libs/databrickscfg/testdata/sample-home/.databrickscfg b/libs/databrickscfg/profile/testdata/sample-home/.databrickscfg similarity index 100% rename from libs/databrickscfg/testdata/sample-home/.databrickscfg rename to libs/databrickscfg/profile/testdata/sample-home/.databrickscfg diff --git a/libs/databrickscfg/profiles.go b/libs/databrickscfg/profiles.go deleted file mode 100644 index 200ac9c87..000000000 --- a/libs/databrickscfg/profiles.go +++ /dev/null @@ -1,150 +0,0 @@ -package databrickscfg - -import ( - "context" - "errors" - "fmt" - "io/fs" - "path/filepath" - "strings" - - "github.com/databricks/cli/libs/env" - "github.com/databricks/databricks-sdk-go/config" - "github.com/spf13/cobra" -) - -// Profile holds a subset of the keys in a databrickscfg profile. -// It should only be used for prompting and filtering. -// Use its name to construct a config.Config. -type Profile struct { - Name string - Host string - AccountID string -} - -func (p Profile) Cloud() string { - cfg := config.Config{Host: p.Host} - switch { - case cfg.IsAws(): - return "AWS" - case cfg.IsAzure(): - return "Azure" - case cfg.IsGcp(): - return "GCP" - default: - return "" - } -} - -type Profiles []Profile - -func (p Profiles) Names() []string { - names := make([]string, len(p)) - for i, v := range p { - names[i] = v.Name - } - return names -} - -// SearchCaseInsensitive implements the promptui.Searcher interface. -// This allows the user to immediately starting typing to narrow down the list. -func (p Profiles) SearchCaseInsensitive(input string, index int) bool { - input = strings.ToLower(input) - name := strings.ToLower(p[index].Name) - host := strings.ToLower(p[index].Host) - return strings.Contains(name, input) || strings.Contains(host, input) -} - -type ProfileMatchFunction func(Profile) bool - -func MatchWorkspaceProfiles(p Profile) bool { - return p.AccountID == "" -} - -func MatchAccountProfiles(p Profile) bool { - return p.Host != "" && p.AccountID != "" -} - -func MatchAllProfiles(p Profile) bool { - return true -} - -// Get the path to the .databrickscfg file, falling back to the default in the current user's home directory. -func GetPath(ctx context.Context) (string, error) { - configFile := env.Get(ctx, "DATABRICKS_CONFIG_FILE") - if configFile == "" { - configFile = "~/.databrickscfg" - } - if strings.HasPrefix(configFile, "~") { - homedir, err := env.UserHomeDir(ctx) - if err != nil { - return "", err - } - configFile = filepath.Join(homedir, configFile[1:]) - } - return configFile, nil -} - -var ErrNoConfiguration = errors.New("no configuration file found") - -func Get(ctx context.Context) (*config.File, error) { - path, err := GetPath(ctx) - if err != nil { - return nil, fmt.Errorf("cannot determine Databricks config file path: %w", err) - } - configFile, err := config.LoadFile(path) - if errors.Is(err, fs.ErrNotExist) { - // downstreams depend on ErrNoConfiguration. TODO: expose this error through SDK - return nil, fmt.Errorf("%w at %s; please create one by running 'databricks configure'", ErrNoConfiguration, path) - } else if err != nil { - return nil, err - } - return configFile, nil -} - -func LoadProfiles(ctx context.Context, fn ProfileMatchFunction) (file string, profiles Profiles, err error) { - f, err := Get(ctx) - if err != nil { - return "", nil, fmt.Errorf("cannot load Databricks config file: %w", err) - } - - // Replace homedir with ~ if applicable. - // This is to make the output more readable. - file = filepath.Clean(f.Path()) - home, err := env.UserHomeDir(ctx) - if err != nil { - return "", nil, err - } - homedir := filepath.Clean(home) - if strings.HasPrefix(file, homedir) { - file = "~" + file[len(homedir):] - } - - // Iterate over sections and collect matching profiles. - for _, v := range f.Sections() { - all := v.KeysHash() - host, ok := all["host"] - if !ok { - // invalid profile - continue - } - profile := Profile{ - Name: v.Name(), - Host: host, - AccountID: all["account_id"], - } - if fn(profile) { - profiles = append(profiles, profile) - } - } - - return -} - -func ProfileCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - _, profiles, err := LoadProfiles(cmd.Context(), MatchAllProfiles) - if err != nil { - return nil, cobra.ShellCompDirectiveError - } - return profiles.Names(), cobra.ShellCompDirectiveNoFileComp -} diff --git a/libs/dyn/convert/end_to_end_test.go b/libs/dyn/convert/end_to_end_test.go index 33902bea8..f0e428a69 100644 --- a/libs/dyn/convert/end_to_end_test.go +++ b/libs/dyn/convert/end_to_end_test.go @@ -67,4 +67,49 @@ func TestAdditional(t *testing.T) { SliceOfPointer: []*string{nil}, }) }) + + t.Run("pointer to a empty string", func(t *testing.T) { + s := "" + assertFromTypedToTypedEqual(t, &s) + }) + + t.Run("nil pointer", func(t *testing.T) { + var s *string + assertFromTypedToTypedEqual(t, s) + }) + + t.Run("pointer to struct with scalar values", func(t *testing.T) { + s := "" + type foo struct { + A string `json:"a"` + B int `json:"b"` + C bool `json:"c"` + D *string `json:"d"` + } + assertFromTypedToTypedEqual(t, &foo{ + A: "a", + B: 1, + C: true, + D: &s, + }) + assertFromTypedToTypedEqual(t, &foo{ + A: "", + B: 0, + C: false, + D: nil, + }) + }) + + t.Run("map with scalar values", func(t *testing.T) { + assertFromTypedToTypedEqual(t, map[string]string{ + "a": "a", + "b": "b", + "c": "", + }) + assertFromTypedToTypedEqual(t, map[string]int{ + "a": 1, + "b": 0, + "c": 2, + }) + }) } diff --git a/libs/dyn/convert/from_typed.go b/libs/dyn/convert/from_typed.go index c344d12df..ae491d8ab 100644 --- a/libs/dyn/convert/from_typed.go +++ b/libs/dyn/convert/from_typed.go @@ -12,16 +12,22 @@ import ( type fromTypedOptions int const ( - // Use the zero value instead of setting zero values to nil. This is useful - // for types where the zero values and nil are semantically different. That is - // strings, bools, ints, floats. + // If this flag is set, zero values for scalars (strings, bools, ints, floats) + // would resolve to corresponding zero values in the dynamic representation. + // Otherwise, zero values for scalars resolve to dyn.NilValue. // - // Note: this is not needed for structs because dyn.NilValue is converted back - // to a zero value when using the convert.ToTyped function. + // This flag exists to reconcile the default values for scalars in a Go struct + // being zero values with zero values in a dynamic representation. In a Go struct, + // zero values are the same as the values not being set at all. This is not the case + // in the dynamic representation. // - // Values in maps and slices should be set to zero values, and not nil in the - // dynamic representation. - includeZeroValues fromTypedOptions = 1 << iota + // If a scalar value in a typed Go struct is zero, in the dynamic representation + // we would set it to dyn.NilValue, i.e. equivalent to the value not being set at all. + // + // If a scalar value in a Go map, slice or pointer is set to zero, we will set it + // to the zero value in the dynamic representation, and not dyn.NilValue. This is + // equivalent to the value being intentionally set to zero. + includeZeroValuedScalars fromTypedOptions = 1 << iota ) // FromTyped converts changes made in the typed structure w.r.t. the configuration value @@ -41,6 +47,14 @@ func fromTyped(src any, ref dyn.Value, options ...fromTypedOptions) (dyn.Value, return dyn.NilValue, nil } srcv = srcv.Elem() + + // If a pointer to a scalar type points to a zero value, we should include + // that zero value in the dynamic representation. + // This is because by default a pointer is nil in Go, and it not being nil + // indicates its value was intentionally set to zero. + if !slices.Contains(options, includeZeroValuedScalars) { + options = append(options, includeZeroValuedScalars) + } } switch srcv.Kind() { @@ -129,7 +143,7 @@ func fromTypedMap(src reflect.Value, ref dyn.Value) (dyn.Value, error) { } // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), refv, includeZeroValues) + nv, err := fromTyped(v.Interface(), refv, includeZeroValuedScalars) if err != nil { return dyn.InvalidValue, err } @@ -160,7 +174,7 @@ func fromTypedSlice(src reflect.Value, ref dyn.Value) (dyn.Value, error) { v := src.Index(i) // Convert entry taking into account the reference value (may be equal to dyn.NilValue). - nv, err := fromTyped(v.Interface(), ref.Index(i), includeZeroValues) + nv, err := fromTyped(v.Interface(), ref.Index(i), includeZeroValuedScalars) if err != nil { return dyn.InvalidValue, err } @@ -183,7 +197,7 @@ func fromTypedString(src reflect.Value, ref dyn.Value, options ...fromTypedOptio case dyn.KindNil: // This field is not set in the reference. We set it to nil if it's zero // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { + if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) { return dyn.NilValue, nil } return dyn.V(src.String()), nil @@ -203,7 +217,7 @@ func fromTypedBool(src reflect.Value, ref dyn.Value, options ...fromTypedOptions case dyn.KindNil: // This field is not set in the reference. We set it to nil if it's zero // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { + if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) { return dyn.NilValue, nil } return dyn.V(src.Bool()), nil @@ -228,7 +242,7 @@ func fromTypedInt(src reflect.Value, ref dyn.Value, options ...fromTypedOptions) case dyn.KindNil: // This field is not set in the reference. We set it to nil if it's zero // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { + if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) { return dyn.NilValue, nil } return dyn.V(src.Int()), nil @@ -253,7 +267,7 @@ func fromTypedFloat(src reflect.Value, ref dyn.Value, options ...fromTypedOption case dyn.KindNil: // This field is not set in the reference. We set it to nil if it's zero // valued in the typed representation and the includeZeroValues option is not set. - if src.IsZero() && !slices.Contains(options, includeZeroValues) { + if src.IsZero() && !slices.Contains(options, includeZeroValuedScalars) { return dyn.NilValue, nil } return dyn.V(src.Float()), nil diff --git a/libs/dyn/convert/struct_info.go b/libs/dyn/convert/struct_info.go index dc3ed4da4..595e52edd 100644 --- a/libs/dyn/convert/struct_info.go +++ b/libs/dyn/convert/struct_info.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/textutil" ) // structInfo holds the type information we need to efficiently @@ -84,6 +85,14 @@ func buildStructInfo(typ reflect.Type) structInfo { } name, _, _ := strings.Cut(sf.Tag.Get("json"), ",") + if typ.Name() == "QualityMonitor" && name == "-" { + urlName, _, _ := strings.Cut(sf.Tag.Get("url"), ",") + if urlName == "" || urlName == "-" { + name = textutil.CamelToSnakeCase(sf.Name) + } else { + name = urlName + } + } if name == "" || name == "-" { continue } diff --git a/libs/dyn/dynvar/ref.go b/libs/dyn/dynvar/ref.go index a2047032a..e6340269f 100644 --- a/libs/dyn/dynvar/ref.go +++ b/libs/dyn/dynvar/ref.go @@ -6,7 +6,9 @@ import ( "github.com/databricks/cli/libs/dyn" ) -var re = regexp.MustCompile(`\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}`) +const VariableRegex = `\$\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\}` + +var re = regexp.MustCompile(VariableRegex) // ref represents a variable reference. // It is a string [dyn.Value] contained in a larger [dyn.Value]. diff --git a/libs/dyn/merge/override.go b/libs/dyn/merge/override.go new file mode 100644 index 000000000..97e8f1009 --- /dev/null +++ b/libs/dyn/merge/override.go @@ -0,0 +1,198 @@ +package merge + +import ( + "fmt" + + "github.com/databricks/cli/libs/dyn" +) + +// OverrideVisitor is visiting the changes during the override process +// and allows to control what changes are allowed, or update the effective +// value. +// +// For instance, it can disallow changes outside the specific path(s), or update +// the location of the effective value. +// +// 'VisitDelete' is called when a value is removed from mapping or sequence +// 'VisitInsert' is called when a new value is added to mapping or sequence +// 'VisitUpdate' is called when a leaf value is updated +type OverrideVisitor struct { + VisitDelete func(valuePath dyn.Path, left dyn.Value) error + VisitInsert func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) + VisitUpdate func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) +} + +// Override overrides value 'leftRoot' with 'rightRoot', keeping 'location' if values +// haven't changed. Preserving 'location' is important to preserve the original source of the value +// for error reporting. +func Override(leftRoot dyn.Value, rightRoot dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { + return override(dyn.EmptyPath, leftRoot, rightRoot, visitor) +} + +func override(basePath dyn.Path, left dyn.Value, right dyn.Value, visitor OverrideVisitor) (dyn.Value, error) { + if left == dyn.NilValue && right == dyn.NilValue { + return dyn.NilValue, nil + } + + if left.Kind() != right.Kind() { + return visitor.VisitUpdate(basePath, left, right) + } + + // NB: we only call 'VisitUpdate' on leaf values, and for sequences and mappings + // we don't know if value was updated or not + + switch left.Kind() { + case dyn.KindMap: + merged, err := overrideMapping(basePath, left.MustMap(), right.MustMap(), visitor) + + if err != nil { + return dyn.InvalidValue, err + } + + return dyn.NewValue(merged, left.Location()), nil + + case dyn.KindSequence: + // some sequences are keyed, and we can detect which elements are added/removed/updated, + // but we don't have this information + merged, err := overrideSequence(basePath, left.MustSequence(), right.MustSequence(), visitor) + + if err != nil { + return dyn.InvalidValue, err + } + + return dyn.NewValue(merged, left.Location()), nil + + case dyn.KindString: + if left.MustString() == right.MustString() { + return left, nil + } else { + return visitor.VisitUpdate(basePath, left, right) + } + + case dyn.KindFloat: + // TODO consider comparison with epsilon if normalization doesn't help, where do we use floats? + + if left.MustFloat() == right.MustFloat() { + return left, nil + } else { + return visitor.VisitUpdate(basePath, left, right) + } + + case dyn.KindBool: + if left.MustBool() == right.MustBool() { + return left, nil + } else { + return visitor.VisitUpdate(basePath, left, right) + } + + case dyn.KindTime: + if left.MustTime() == right.MustTime() { + return left, nil + } else { + return visitor.VisitUpdate(basePath, left, right) + } + + case dyn.KindInt: + if left.MustInt() == right.MustInt() { + return left, nil + } else { + return visitor.VisitUpdate(basePath, left, right) + } + } + + return dyn.InvalidValue, fmt.Errorf("unexpected kind %s", left.Kind()) +} + +func overrideMapping(basePath dyn.Path, leftMapping dyn.Mapping, rightMapping dyn.Mapping, visitor OverrideVisitor) (dyn.Mapping, error) { + out := dyn.NewMapping() + + for _, leftPair := range leftMapping.Pairs() { + // detect if key was removed + if _, ok := rightMapping.GetPair(leftPair.Key); !ok { + path := basePath.Append(dyn.Key(leftPair.Key.MustString())) + + err := visitor.VisitDelete(path, leftPair.Value) + + if err != nil { + return dyn.NewMapping(), err + } + } + } + + // iterating only right mapping will remove keys not present anymore + // and insert new keys + + for _, rightPair := range rightMapping.Pairs() { + if leftPair, ok := leftMapping.GetPair(rightPair.Key); ok { + path := basePath.Append(dyn.Key(rightPair.Key.MustString())) + newValue, err := override(path, leftPair.Value, rightPair.Value, visitor) + + if err != nil { + return dyn.NewMapping(), err + } + + // key was there before, so keep its location + err = out.Set(leftPair.Key, newValue) + + if err != nil { + return dyn.NewMapping(), err + } + } else { + path := basePath.Append(dyn.Key(rightPair.Key.MustString())) + + newValue, err := visitor.VisitInsert(path, rightPair.Value) + + if err != nil { + return dyn.NewMapping(), err + } + + err = out.Set(rightPair.Key, newValue) + + if err != nil { + return dyn.NewMapping(), err + } + } + } + + return out, nil +} + +func overrideSequence(basePath dyn.Path, left []dyn.Value, right []dyn.Value, visitor OverrideVisitor) ([]dyn.Value, error) { + minLen := min(len(left), len(right)) + var values []dyn.Value + + for i := 0; i < minLen; i++ { + path := basePath.Append(dyn.Index(i)) + merged, err := override(path, left[i], right[i], visitor) + + if err != nil { + return nil, err + } + + values = append(values, merged) + } + + if len(right) > len(left) { + for i := minLen; i < len(right); i++ { + path := basePath.Append(dyn.Index(i)) + newValue, err := visitor.VisitInsert(path, right[i]) + + if err != nil { + return nil, err + } + + values = append(values, newValue) + } + } else if len(left) > len(right) { + for i := minLen; i < len(left); i++ { + path := basePath.Append(dyn.Index(i)) + err := visitor.VisitDelete(path, left[i]) + + if err != nil { + return nil, err + } + } + } + + return values, nil +} diff --git a/libs/dyn/merge/override_test.go b/libs/dyn/merge/override_test.go new file mode 100644 index 000000000..a34f23424 --- /dev/null +++ b/libs/dyn/merge/override_test.go @@ -0,0 +1,487 @@ +package merge + +import ( + "fmt" + "testing" + "time" + + "github.com/databricks/cli/libs/dyn" + assert "github.com/databricks/cli/libs/dyn/dynassert" +) + +type overrideTestCase struct { + name string + left dyn.Value + right dyn.Value + state visitorState + expected dyn.Value +} + +func TestOverride_Primitive(t *testing.T) { + leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1} + rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1} + + modifiedTestCases := []overrideTestCase{ + { + name: "string (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue("a", leftLocation), + right: dyn.NewValue("b", rightLocation), + expected: dyn.NewValue("b", rightLocation), + }, + { + name: "string (not updated)", + state: visitorState{}, + left: dyn.NewValue("a", leftLocation), + right: dyn.NewValue("a", rightLocation), + expected: dyn.NewValue("a", leftLocation), + }, + { + name: "bool (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue(true, leftLocation), + right: dyn.NewValue(false, rightLocation), + expected: dyn.NewValue(false, rightLocation), + }, + { + name: "bool (not updated)", + state: visitorState{}, + left: dyn.NewValue(true, leftLocation), + right: dyn.NewValue(true, rightLocation), + expected: dyn.NewValue(true, leftLocation), + }, + { + name: "int (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue(1, leftLocation), + right: dyn.NewValue(2, rightLocation), + expected: dyn.NewValue(2, rightLocation), + }, + { + name: "int (not updated)", + state: visitorState{}, + left: dyn.NewValue(int32(1), leftLocation), + right: dyn.NewValue(int64(1), rightLocation), + expected: dyn.NewValue(int32(1), leftLocation), + }, + { + name: "float (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue(1.0, leftLocation), + right: dyn.NewValue(2.0, rightLocation), + expected: dyn.NewValue(2.0, rightLocation), + }, + { + name: "float (not updated)", + state: visitorState{}, + left: dyn.NewValue(float32(1.0), leftLocation), + right: dyn.NewValue(float64(1.0), rightLocation), + expected: dyn.NewValue(float32(1.0), leftLocation), + }, + { + name: "time (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue(time.UnixMilli(10000), leftLocation), + right: dyn.NewValue(time.UnixMilli(10001), rightLocation), + expected: dyn.NewValue(time.UnixMilli(10001), rightLocation), + }, + { + name: "time (not updated)", + state: visitorState{}, + left: dyn.NewValue(time.UnixMilli(10000), leftLocation), + right: dyn.NewValue(time.UnixMilli(10000), rightLocation), + expected: dyn.NewValue(time.UnixMilli(10000), leftLocation), + }, + { + name: "different types (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue("a", leftLocation), + right: dyn.NewValue(42, rightLocation), + expected: dyn.NewValue(42, rightLocation), + }, + { + name: "map - remove 'a', update 'b'", + state: visitorState{ + removed: []string{"root.a"}, + updated: []string{"root.b"}, + }, + left: dyn.NewValue( + map[string]dyn.Value{ + "a": dyn.NewValue(42, leftLocation), + "b": dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + map[string]dyn.Value{ + "b": dyn.NewValue(20, rightLocation), + }, + rightLocation, + ), + expected: dyn.NewValue( + map[string]dyn.Value{ + "b": dyn.NewValue(20, rightLocation), + }, + leftLocation, + ), + }, + { + name: "map - add 'a'", + state: visitorState{ + added: []string{"root.a"}, + }, + left: dyn.NewValue( + map[string]dyn.Value{ + "b": dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + map[string]dyn.Value{ + "a": dyn.NewValue(42, rightLocation), + "b": dyn.NewValue(10, rightLocation), + }, + leftLocation, + ), + expected: dyn.NewValue( + map[string]dyn.Value{ + "a": dyn.NewValue(42, rightLocation), + // location hasn't changed because value hasn't changed + "b": dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + }, + { + name: "map - remove 'a'", + state: visitorState{ + removed: []string{"root.a"}, + }, + left: dyn.NewValue( + map[string]dyn.Value{ + "a": dyn.NewValue(42, leftLocation), + "b": dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + map[string]dyn.Value{ + "b": dyn.NewValue(10, rightLocation), + }, + leftLocation, + ), + expected: dyn.NewValue( + map[string]dyn.Value{ + // location hasn't changed because value hasn't changed + "b": dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + }, + { + name: "map - add 'jobs.job_1'", + state: visitorState{ + added: []string{"root.jobs.job_1"}, + }, + left: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + }, + leftLocation, + ), + right: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, rightLocation), + "job_1": dyn.NewValue(1337, rightLocation), + }, + rightLocation, + ), + }, + rightLocation, + ), + expected: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, leftLocation), + "job_1": dyn.NewValue(1337, rightLocation), + }, + leftLocation, + ), + }, + leftLocation, + ), + }, + { + name: "map - remove nested key", + state: visitorState{removed: []string{"root.jobs.job_1"}}, + left: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, leftLocation), + "job_1": dyn.NewValue(1337, rightLocation), + }, + leftLocation, + ), + }, + leftLocation, + ), + right: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, rightLocation), + }, + rightLocation, + ), + }, + rightLocation, + ), + expected: dyn.NewValue( + map[string]dyn.Value{ + "jobs": dyn.NewValue( + map[string]dyn.Value{ + "job_0": dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + }, + leftLocation, + ), + }, + { + name: "sequence - add", + state: visitorState{added: []string{"root[1]"}}, + left: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, rightLocation), + dyn.NewValue(10, rightLocation), + }, + rightLocation, + ), + expected: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, leftLocation), + dyn.NewValue(10, rightLocation), + }, + leftLocation, + ), + }, + { + name: "sequence - remove", + state: visitorState{removed: []string{"root[1]"}}, + left: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, leftLocation), + dyn.NewValue(10, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, rightLocation), + }, + rightLocation, + ), + expected: dyn.NewValue( + []dyn.Value{ + // location hasn't changed because value hasn't changed + dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + }, + { + name: "sequence (not updated)", + state: visitorState{}, + left: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + right: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, rightLocation), + }, + rightLocation, + ), + expected: dyn.NewValue( + []dyn.Value{ + dyn.NewValue(42, leftLocation), + }, + leftLocation, + ), + }, + { + name: "nil (not updated)", + state: visitorState{}, + left: dyn.NilValue, + right: dyn.NilValue, + expected: dyn.NilValue, + }, + { + name: "nil (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NilValue, + right: dyn.NewValue(42, rightLocation), + expected: dyn.NewValue(42, rightLocation), + }, + { + name: "change kind (updated)", + state: visitorState{updated: []string{"root"}}, + left: dyn.NewValue(42.0, leftLocation), + right: dyn.NewValue(42, rightLocation), + expected: dyn.NewValue(42, rightLocation), + }, + } + + for _, tc := range modifiedTestCases { + t.Run(tc.name, func(t *testing.T) { + s, visitor := createVisitor(visitorOpts{}) + out, err := override(dyn.NewPath(dyn.Key("root")), tc.left, tc.right, visitor) + + assert.NoError(t, err) + assert.Equal(t, tc.state, *s) + assert.Equal(t, tc.expected, out) + }) + + modified := len(tc.state.removed)+len(tc.state.added)+len(tc.state.updated) > 0 + + // visitor is not used unless there is a change + + if modified { + t.Run(tc.name+" - visitor has error", func(t *testing.T) { + _, visitor := createVisitor(visitorOpts{error: fmt.Errorf("unexpected change in test")}) + _, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) + + assert.EqualError(t, err, "unexpected change in test") + }) + + t.Run(tc.name+" - visitor overrides value", func(t *testing.T) { + expected := dyn.NewValue("return value", dyn.Location{}) + s, visitor := createVisitor(visitorOpts{returnValue: &expected}) + out, err := override(dyn.EmptyPath, tc.left, tc.right, visitor) + + assert.NoError(t, err) + + for _, added := range s.added { + actual, err := dyn.GetByPath(out, dyn.MustPathFromString(added)) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) + } + + for _, updated := range s.updated { + actual, err := dyn.GetByPath(out, dyn.MustPathFromString(updated)) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) + } + }) + } + } +} + +func TestOverride_PreserveMappingKeys(t *testing.T) { + leftLocation := dyn.Location{File: "left.yml", Line: 1, Column: 1} + leftKeyLocation := dyn.Location{File: "left.yml", Line: 2, Column: 1} + leftValueLocation := dyn.Location{File: "left.yml", Line: 3, Column: 1} + + rightLocation := dyn.Location{File: "right.yml", Line: 1, Column: 1} + rightKeyLocation := dyn.Location{File: "right.yml", Line: 2, Column: 1} + rightValueLocation := dyn.Location{File: "right.yml", Line: 3, Column: 1} + + left := dyn.NewMapping() + left.Set(dyn.NewValue("a", leftKeyLocation), dyn.NewValue(42, leftValueLocation)) + + right := dyn.NewMapping() + right.Set(dyn.NewValue("a", rightKeyLocation), dyn.NewValue(7, rightValueLocation)) + + state, visitor := createVisitor(visitorOpts{}) + + out, err := override( + dyn.EmptyPath, + dyn.NewValue(left, leftLocation), + dyn.NewValue(right, rightLocation), + visitor, + ) + + assert.NoError(t, err) + + if err != nil { + outPairs := out.MustMap().Pairs() + + assert.Equal(t, visitorState{updated: []string{"a"}}, state) + assert.Equal(t, 1, len(outPairs)) + + // mapping was first defined in left, so it should keep its location + assert.Equal(t, leftLocation, out.Location()) + + // if there is a validation error for key value, it should point + // to where it was initially defined + assert.Equal(t, leftKeyLocation, outPairs[0].Key.Location()) + + // the value should have updated location, because it has changed + assert.Equal(t, rightValueLocation, outPairs[0].Value.Location()) + } +} + +type visitorState struct { + added []string + removed []string + updated []string +} + +type visitorOpts struct { + error error + returnValue *dyn.Value +} + +func createVisitor(opts visitorOpts) (*visitorState, OverrideVisitor) { + s := visitorState{} + + return &s, OverrideVisitor{ + VisitUpdate: func(valuePath dyn.Path, left dyn.Value, right dyn.Value) (dyn.Value, error) { + s.updated = append(s.updated, valuePath.String()) + + if opts.error != nil { + return dyn.NilValue, opts.error + } else if opts.returnValue != nil { + return *opts.returnValue, nil + } else { + return right, nil + } + }, + VisitDelete: func(valuePath dyn.Path, left dyn.Value) error { + s.removed = append(s.removed, valuePath.String()) + + return opts.error + }, + VisitInsert: func(valuePath dyn.Path, right dyn.Value) (dyn.Value, error) { + s.added = append(s.added, valuePath.String()) + + if opts.error != nil { + return dyn.NilValue, opts.error + } else if opts.returnValue != nil { + return *opts.returnValue, nil + } else { + return right, nil + } + }, + } +} diff --git a/libs/filer/local_client.go b/libs/filer/local_client.go index 958b6277d..9398958f5 100644 --- a/libs/filer/local_client.go +++ b/libs/filer/local_client.go @@ -34,7 +34,6 @@ func (w *LocalClient) Write(ctx context.Context, name string, reader io.Reader, flags |= os.O_EXCL } - absPath = filepath.FromSlash(absPath) f, err := os.OpenFile(absPath, flags, 0644) if os.IsNotExist(err) && slices.Contains(mode, CreateParentDirectories) { // Create parent directories if they don't exist. @@ -76,7 +75,6 @@ func (w *LocalClient) Read(ctx context.Context, name string) (io.ReadCloser, err // This stat call serves two purposes: // 1. Checks file at path exists, and throws an error if it does not // 2. Allows us to error out if the path is a directory - absPath = filepath.FromSlash(absPath) stat, err := os.Stat(absPath) if err != nil { if os.IsNotExist(err) { @@ -103,7 +101,6 @@ func (w *LocalClient) Delete(ctx context.Context, name string, mode ...DeleteMod return CannotDeleteRootError{} } - absPath = filepath.FromSlash(absPath) err = os.Remove(absPath) // Return early on success. @@ -131,7 +128,6 @@ func (w *LocalClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, return nil, err } - absPath = filepath.FromSlash(absPath) stat, err := os.Stat(absPath) if err != nil { if os.IsNotExist(err) { @@ -153,7 +149,6 @@ func (w *LocalClient) Mkdir(ctx context.Context, name string) error { return err } - dirPath = filepath.FromSlash(dirPath) return os.MkdirAll(dirPath, 0755) } @@ -163,7 +158,6 @@ func (w *LocalClient) Stat(ctx context.Context, name string) (fs.FileInfo, error return nil, err } - absPath = filepath.FromSlash(absPath) stat, err := os.Stat(absPath) if os.IsNotExist(err) { return nil, FileDoesNotExistError{path: absPath} diff --git a/libs/filer/local_root_path.go b/libs/filer/local_root_path.go index 15a542631..3f8843093 100644 --- a/libs/filer/local_root_path.go +++ b/libs/filer/local_root_path.go @@ -19,7 +19,6 @@ func NewLocalRootPath(root string) localRootPath { func (rp *localRootPath) Join(name string) (string, error) { absPath := filepath.Join(rp.rootPath, name) - if !strings.HasPrefix(absPath, rp.rootPath) { return "", fmt.Errorf("relative path escapes root: %s", name) } diff --git a/libs/filer/workspace_files_extensions_client.go b/libs/filer/workspace_files_extensions_client.go new file mode 100644 index 000000000..bad748b10 --- /dev/null +++ b/libs/filer/workspace_files_extensions_client.go @@ -0,0 +1,345 @@ +package filer + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "path" + "strings" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/notebook" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +type workspaceFilesExtensionsClient struct { + workspaceClient *databricks.WorkspaceClient + apiClient *client.DatabricksClient + + wsfs Filer + root string +} + +var extensionsToLanguages = map[string]workspace.Language{ + ".py": workspace.LanguagePython, + ".r": workspace.LanguageR, + ".scala": workspace.LanguageScala, + ".sql": workspace.LanguageSql, + ".ipynb": workspace.LanguagePython, +} + +// workspaceFileStatus defines a custom response body for the "/api/2.0/workspace/get-status" API. +// The "repos_export_format" field is not exposed by the SDK. +type workspaceFileStatus struct { + *workspace.ObjectInfo + + // The export format of the notebook. This is not exposed by the SDK. + ReposExportFormat workspace.ExportFormat `json:"repos_export_format,omitempty"` + + // Name of the file to be used in any API calls made using the workspace files + // filer. For notebooks this path does not include the extension. + nameForWorkspaceAPI string +} + +// A custom unmarsaller for the workspaceFileStatus struct. This is needed because +// workspaceFileStatus embeds the workspace.ObjectInfo which itself has a custom +// unmarshaller. +// If a custom unmarshaller is not provided extra fields like ReposExportFormat +// will not have values set. +func (s *workspaceFileStatus) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s *workspaceFileStatus) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (w *workspaceFilesExtensionsClient) stat(ctx context.Context, name string) (*workspaceFileStatus, error) { + stat := &workspaceFileStatus{ + nameForWorkspaceAPI: name, + } + + // Perform bespoke API call because "return_export_info" is not exposed by the SDK. + // We need "repos_export_format" to determine if the file is a py or a ipynb notebook. + // This is not exposed by the SDK so we need to make a direct API call. + err := w.apiClient.Do( + ctx, + http.MethodGet, + "/api/2.0/workspace/get-status", + nil, + map[string]string{ + "path": path.Join(w.root, name), + "return_export_info": "true", + }, + stat, + ) + if err != nil { + // If we got an API error we deal with it below. + var aerr *apierr.APIError + if !errors.As(err, &aerr) { + return nil, err + } + + // This API returns a 404 if the specified path does not exist. + if aerr.StatusCode == http.StatusNotFound { + return nil, FileDoesNotExistError{path.Join(w.root, name)} + } + } + return stat, err +} + +// This function returns the stat for the provided notebook. The stat object itself contains the path +// with the extension since it is meant to be used in the context of a fs.FileInfo. +func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithExt(ctx context.Context, name string) (*workspaceFileStatus, error) { + ext := path.Ext(name) + nameWithoutExt := strings.TrimSuffix(name, ext) + + // File name does not have an extension associated with Databricks notebooks, return early. + if _, ok := extensionsToLanguages[ext]; !ok { + return nil, nil + } + + // If the file could be a notebook, check if it is and has the correct language. + stat, err := w.stat(ctx, nameWithoutExt) + if err != nil { + // If the file does not exist, return early. + if errors.As(err, &FileDoesNotExistError{}) { + return nil, nil + } + log.Debugf(ctx, "attempting to determine if %s could be a notebook. Failed to fetch the status of object at %s: %s", name, path.Join(w.root, nameWithoutExt), err) + return nil, err + } + + // Not a notebook. Return early. + if stat.ObjectType != workspace.ObjectTypeNotebook { + log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found an object at %s but it is not a notebook. It is a %s.", name, path.Join(w.root, nameWithoutExt), stat.ObjectType) + return nil, nil + } + + // Not the correct language. Return early. + if stat.Language != extensionsToLanguages[ext] { + log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not of the correct language. Expected %s but found %s.", name, path.Join(w.root, nameWithoutExt), extensionsToLanguages[ext], stat.Language) + return nil, nil + } + + // When the extension is .py we expect the export format to be source. + // If it's not, return early. + if ext == ".py" && stat.ReposExportFormat != workspace.ExportFormatSource { + log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not exported as a source notebook. Its export format is %s.", name, path.Join(w.root, nameWithoutExt), stat.ReposExportFormat) + return nil, nil + } + + // When the extension is .ipynb we expect the export format to be Jupyter. + // If it's not, return early. + if ext == ".ipynb" && stat.ReposExportFormat != workspace.ExportFormatJupyter { + log.Debugf(ctx, "attempting to determine if %s could be a notebook. Found a notebook at %s but it is not exported as a Jupyter notebook. Its export format is %s.", name, path.Join(w.root, nameWithoutExt), stat.ReposExportFormat) + return nil, nil + } + + // Modify the stat object path to include the extension. This stat object will be used + // to return the fs.FileInfo object in the stat method. + stat.Path = stat.Path + ext + return stat, nil +} + +func (w *workspaceFilesExtensionsClient) getNotebookStatByNameWithoutExt(ctx context.Context, name string) (*workspaceFileStatus, error) { + stat, err := w.stat(ctx, name) + if err != nil { + return nil, err + } + + // We expect this internal function to only be called from [ReadDir] when we are sure + // that the object is a notebook. Thus, this should never happen. + if stat.ObjectType != workspace.ObjectTypeNotebook { + return nil, fmt.Errorf("expected object at %s to be a notebook but it is a %s", path.Join(w.root, name), stat.ObjectType) + } + + // Get the extension for the notebook. + ext := notebook.GetExtensionByLanguage(stat.ObjectInfo) + + // If the notebook was exported as a Jupyter notebook, the extension should be .ipynb. + if stat.Language == workspace.LanguagePython && stat.ReposExportFormat == workspace.ExportFormatJupyter { + ext = ".ipynb" + } + + // Modify the stat object path to include the extension. This stat object will be used + // to return the fs.DirEntry object in the ReadDir method. + stat.Path = stat.Path + ext + return stat, nil +} + +type DuplicatePathError struct { + oi1 workspace.ObjectInfo + oi2 workspace.ObjectInfo + + commonName string +} + +func (e DuplicatePathError) Error() string { + return fmt.Sprintf("failed to read files from the workspace file system. Duplicate paths encountered. Both %s at %s and %s at %s resolve to the same name %s. Changing the name of one of these objects will resolve this issue", e.oi1.ObjectType, e.oi1.Path, e.oi2.ObjectType, e.oi2.Path, e.commonName) +} + +// This is a filer for the workspace file system that allows you to pretend the +// workspace file system is a traditional file system. It allows you to list, read, write, +// delete, and stat notebooks (and files in general) in the workspace, using their paths +// with the extension included. +// +// The ReadDir method returns a DuplicatePathError if this traditional file system view is +// not possible. For example, a Python notebook called foo and a Python file called `foo.py` +// would resolve to the same path `foo.py` in a tradition file system. +// +// Users of this filer should be careful when using the Write and Mkdir methods. +// The underlying import API we use to upload notebooks and files returns opaque internal +// errors for namespace clashes (e.g. a file and a notebook or a directory and a notebook). +// Thus users of these methods should be careful to avoid such clashes. +func NewWorkspaceFilesExtensionsClient(w *databricks.WorkspaceClient, root string) (Filer, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, err + } + + filer, err := NewWorkspaceFilesClient(w, root) + if err != nil { + return nil, err + } + + return &workspaceFilesExtensionsClient{ + workspaceClient: w, + apiClient: apiClient, + + wsfs: filer, + root: root, + }, nil +} + +func (w *workspaceFilesExtensionsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + entries, err := w.wsfs.ReadDir(ctx, name) + if err != nil { + return nil, err + } + + seenPaths := make(map[string]workspace.ObjectInfo) + for i := range entries { + info, err := entries[i].Info() + if err != nil { + return nil, err + } + sysInfo := info.Sys().(workspace.ObjectInfo) + + // If the object is a notebook, include an extension in the entry. + if sysInfo.ObjectType == workspace.ObjectTypeNotebook { + stat, err := w.getNotebookStatByNameWithoutExt(ctx, entries[i].Name()) + if err != nil { + return nil, err + } + // Replace the entry with the new entry that includes the extension. + entries[i] = wsfsDirEntry{wsfsFileInfo{oi: *stat.ObjectInfo}} + } + + // Error if we have seen this path before in the current directory. + // If not seen before, add it to the seen paths. + if _, ok := seenPaths[entries[i].Name()]; ok { + return nil, DuplicatePathError{ + oi1: seenPaths[entries[i].Name()], + oi2: sysInfo, + commonName: path.Join(name, entries[i].Name()), + } + } + seenPaths[entries[i].Name()] = sysInfo + } + + return entries, nil +} + +// Note: The import API returns opaque internal errors for namespace clashes +// (e.g. a file and a notebook or a directory and a notebook). Thus users of this +// method should be careful to avoid such clashes. +func (w *workspaceFilesExtensionsClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + return w.wsfs.Write(ctx, name, reader, mode...) +} + +// Try to read the file as a regular file. If the file is not found, try to read it as a notebook. +func (w *workspaceFilesExtensionsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { + r, err := w.wsfs.Read(ctx, name) + + // If the file is not found, it might be a notebook. + if errors.As(err, &FileDoesNotExistError{}) { + stat, serr := w.getNotebookStatByNameWithExt(ctx, name) + if serr != nil { + // Unable to stat. Return the stat error. + return nil, serr + } + if stat == nil { + // Not a notebook. Return the original error. + return nil, err + } + + // The workspace files filer performs an additional stat call to make sure + // the path is not a directory. We can skip this step since we already have + // the stat object and know that the path is a notebook. + return w.workspaceClient.Workspace.Download( + ctx, + path.Join(w.root, stat.nameForWorkspaceAPI), + workspace.DownloadFormat(stat.ReposExportFormat), + ) + } + return r, err +} + +// Try to delete the file as a regular file. If the file is not found, try to delete it as a notebook. +func (w *workspaceFilesExtensionsClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + err := w.wsfs.Delete(ctx, name, mode...) + + // If the file is not found, it might be a notebook. + if errors.As(err, &FileDoesNotExistError{}) { + stat, serr := w.getNotebookStatByNameWithExt(ctx, name) + if serr != nil { + // Unable to stat. Return the stat error. + return serr + } + if stat == nil { + // Not a notebook. Return the original error. + return err + } + + return w.wsfs.Delete(ctx, stat.nameForWorkspaceAPI, mode...) + } + + return err +} + +// Try to stat the file as a regular file. If the file is not found, try to stat it as a notebook. +func (w *workspaceFilesExtensionsClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + info, err := w.wsfs.Stat(ctx, name) + + // If the file is not found, it might be a notebook. + if errors.As(err, &FileDoesNotExistError{}) { + stat, serr := w.getNotebookStatByNameWithExt(ctx, name) + if serr != nil { + // Unable to stat. Return the stat error. + return nil, serr + } + if stat == nil { + // Not a notebook. Return the original error. + return nil, err + } + + return wsfsFileInfo{oi: *stat.ObjectInfo}, nil + } + + return info, err +} + +// Note: The import API returns opaque internal errors for namespace clashes +// (e.g. a file and a notebook or a directory and a notebook). Thus users of this +// method should be careful to avoid such clashes. +func (w *workspaceFilesExtensionsClient) Mkdir(ctx context.Context, name string) error { + return w.wsfs.Mkdir(ctx, name) +} diff --git a/libs/fileset/file.go b/libs/fileset/file.go index 17cae7952..fd846b257 100644 --- a/libs/fileset/file.go +++ b/libs/fileset/file.go @@ -5,6 +5,7 @@ import ( "time" "github.com/databricks/cli/libs/notebook" + "github.com/databricks/cli/libs/vfs" ) type fileType int @@ -16,40 +17,49 @@ const ( ) type File struct { - fs.DirEntry - Absolute, Relative string - fileType fileType + // Root path of the fileset. + root vfs.Path + + // File entry as returned by the [fs.WalkDir] function. + entry fs.DirEntry + + // Type of the file. + fileType fileType + + // Relative path within the fileset. + // Combine with the [vfs.Path] to interact with the underlying file. + Relative string } -func NewNotebookFile(entry fs.DirEntry, absolute string, relative string) File { +func NewNotebookFile(root vfs.Path, entry fs.DirEntry, relative string) File { return File{ - DirEntry: entry, - Absolute: absolute, - Relative: relative, + root: root, + entry: entry, fileType: Notebook, + Relative: relative, } } -func NewFile(entry fs.DirEntry, absolute string, relative string) File { +func NewFile(root vfs.Path, entry fs.DirEntry, relative string) File { return File{ - DirEntry: entry, - Absolute: absolute, - Relative: relative, + root: root, + entry: entry, fileType: Unknown, + Relative: relative, } } -func NewSourceFile(entry fs.DirEntry, absolute string, relative string) File { +func NewSourceFile(root vfs.Path, entry fs.DirEntry, relative string) File { return File{ - DirEntry: entry, - Absolute: absolute, - Relative: relative, + root: root, + entry: entry, fileType: Source, + Relative: relative, } } func (f File) Modified() (ts time.Time) { - info, err := f.Info() + info, err := f.entry.Info() if err != nil { // return default time, beginning of epoch return ts @@ -63,7 +73,7 @@ func (f *File) IsNotebook() (bool, error) { } // Otherwise, detect the notebook type. - isNotebook, _, err := notebook.Detect(f.Absolute) + isNotebook, _, err := notebook.DetectWithFS(f.root, f.Relative) if err != nil { return false, err } diff --git a/libs/fileset/file_test.go b/libs/fileset/file_test.go index cdfc9ba17..1ce1ff59a 100644 --- a/libs/fileset/file_test.go +++ b/libs/fileset/file_test.go @@ -1,22 +1,22 @@ package fileset import ( - "path/filepath" "testing" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) func TestNotebookFileIsNotebook(t *testing.T) { - f := NewNotebookFile(nil, "", "") + f := NewNotebookFile(nil, nil, "") isNotebook, err := f.IsNotebook() require.NoError(t, err) require.True(t, isNotebook) } func TestSourceFileIsNotNotebook(t *testing.T) { - f := NewSourceFile(nil, "", "") + f := NewSourceFile(nil, nil, "") isNotebook, err := f.IsNotebook() require.NoError(t, err) require.False(t, isNotebook) @@ -24,18 +24,19 @@ func TestSourceFileIsNotNotebook(t *testing.T) { func TestUnknownFileDetectsNotebook(t *testing.T) { tmpDir := t.TempDir() + root := vfs.MustNew(tmpDir) t.Run("file", func(t *testing.T) { - path := testutil.Touch(t, tmpDir, "test.py") - f := NewFile(nil, path, filepath.Base(path)) + testutil.Touch(t, tmpDir, "test.py") + f := NewFile(root, nil, "test.py") isNotebook, err := f.IsNotebook() require.NoError(t, err) require.False(t, isNotebook) }) t.Run("notebook", func(t *testing.T) { - path := testutil.TouchNotebook(t, tmpDir, "notebook.py") - f := NewFile(nil, path, filepath.Base(path)) + testutil.TouchNotebook(t, tmpDir, "notebook.py") + f := NewFile(root, nil, "notebook.py") isNotebook, err := f.IsNotebook() require.NoError(t, err) require.True(t, isNotebook) diff --git a/libs/fileset/fileset.go b/libs/fileset/fileset.go index 52463dff3..d0f00f97a 100644 --- a/libs/fileset/fileset.go +++ b/libs/fileset/fileset.go @@ -4,20 +4,24 @@ import ( "fmt" "io/fs" "os" - "path/filepath" + + "github.com/databricks/cli/libs/vfs" ) // FileSet facilitates fast recursive file listing of a path. // It optionally takes into account ignore rules through the [Ignorer] interface. type FileSet struct { - root string + // Root path of the fileset. + root vfs.Path + + // Ignorer interface to check if a file or directory should be ignored. ignore Ignorer } // New returns a [FileSet] for the given root path. -func New(root string) *FileSet { +func New(root vfs.Path) *FileSet { return &FileSet{ - root: filepath.Clean(root), + root: root, ignore: nopIgnorer{}, } } @@ -32,11 +36,6 @@ func (w *FileSet) SetIgnorer(ignore Ignorer) { w.ignore = ignore } -// Return root for fileset. -func (w *FileSet) Root() string { - return w.root -} - // Return all tracked files for Repo func (w *FileSet) All() ([]File, error) { return w.recursiveListFiles() @@ -46,12 +45,7 @@ func (w *FileSet) All() ([]File, error) { // that are being tracked in the FileSet (ie not being ignored for matching one of the // patterns in w.ignore) func (w *FileSet) recursiveListFiles() (fileList []File, err error) { - err = filepath.WalkDir(w.root, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(w.root, path) + err = fs.WalkDir(w.root, ".", func(name string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -66,25 +60,25 @@ func (w *FileSet) recursiveListFiles() (fileList []File, err error) { } if d.IsDir() { - ign, err := w.ignore.IgnoreDirectory(relPath) + ign, err := w.ignore.IgnoreDirectory(name) if err != nil { - return fmt.Errorf("cannot check if %s should be ignored: %w", relPath, err) + return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) } if ign { - return filepath.SkipDir + return fs.SkipDir } return nil } - ign, err := w.ignore.IgnoreFile(relPath) + ign, err := w.ignore.IgnoreFile(name) if err != nil { - return fmt.Errorf("cannot check if %s should be ignored: %w", relPath, err) + return fmt.Errorf("cannot check if %s should be ignored: %w", name, err) } if ign { return nil } - fileList = append(fileList, NewFile(d, path, relPath)) + fileList = append(fileList, NewFile(w.root, d, name)) return nil }) return diff --git a/libs/fileset/glob.go b/libs/fileset/glob.go index 9d8626e54..0a1038472 100644 --- a/libs/fileset/glob.go +++ b/libs/fileset/glob.go @@ -1,22 +1,17 @@ package fileset import ( - "path/filepath" + "path" + + "github.com/databricks/cli/libs/vfs" ) -func NewGlobSet(root string, includes []string) (*FileSet, error) { - absRoot, err := filepath.Abs(root) - if err != nil { - return nil, err - } - +func NewGlobSet(root vfs.Path, includes []string) (*FileSet, error) { for k := range includes { - includes[k] = filepath.ToSlash(filepath.Clean(includes[k])) + includes[k] = path.Clean(includes[k]) } - fs := &FileSet{ - absRoot, - newIncluder(includes), - } + fs := New(root) + fs.SetIgnorer(newIncluder(includes)) return fs, nil } diff --git a/libs/fileset/glob_test.go b/libs/fileset/glob_test.go index e8d3696c4..70b9c444b 100644 --- a/libs/fileset/glob_test.go +++ b/libs/fileset/glob_test.go @@ -2,21 +2,26 @@ package fileset import ( "io/fs" - "os" - "path/filepath" + "path" "slices" "strings" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) -func TestGlobFileset(t *testing.T) { - cwd, err := os.Getwd() - require.NoError(t, err) - root := filepath.Join(cwd, "..", "filer") +func collectRelativePaths(files []File) []string { + relativePaths := make([]string, 0) + for _, f := range files { + relativePaths = append(relativePaths, f.Relative) + } + return relativePaths +} - entries, err := os.ReadDir(root) +func TestGlobFileset(t *testing.T) { + root := vfs.MustNew("../filer") + entries, err := root.ReadDir(".") require.NoError(t, err) g, err := NewGlobSet(root, []string{ @@ -30,7 +35,7 @@ func TestGlobFileset(t *testing.T) { require.Equal(t, len(files), len(entries)) for _, f := range files { exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool { - return de.Name() == f.Name() + return de.Name() == path.Base(f.Relative) }) require.True(t, exists) } @@ -46,9 +51,8 @@ func TestGlobFileset(t *testing.T) { } func TestGlobFilesetWithRelativeRoot(t *testing.T) { - root := filepath.Join("..", "filer") - - entries, err := os.ReadDir(root) + root := vfs.MustNew("../filer") + entries, err := root.ReadDir(".") require.NoError(t, err) g, err := NewGlobSet(root, []string{ @@ -58,21 +62,14 @@ func TestGlobFilesetWithRelativeRoot(t *testing.T) { files, err := g.All() require.NoError(t, err) - require.Equal(t, len(files), len(entries)) - for _, f := range files { - require.True(t, filepath.IsAbs(f.Absolute)) - } } func TestGlobFilesetRecursively(t *testing.T) { - cwd, err := os.Getwd() - require.NoError(t, err) - root := filepath.Join(cwd, "..", "git") - + root := vfs.MustNew("../git") entries := make([]string, 0) - err = filepath.Walk(filepath.Join(root, "testdata"), func(path string, info fs.FileInfo, err error) error { - if !info.IsDir() { + err := fs.WalkDir(root, "testdata", func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { entries = append(entries, path) } return nil @@ -86,24 +83,14 @@ func TestGlobFilesetRecursively(t *testing.T) { files, err := g.All() require.NoError(t, err) - - require.Equal(t, len(files), len(entries)) - for _, f := range files { - exists := slices.ContainsFunc(entries, func(path string) bool { - return path == f.Absolute - }) - require.True(t, exists) - } + require.ElementsMatch(t, entries, collectRelativePaths(files)) } func TestGlobFilesetDir(t *testing.T) { - cwd, err := os.Getwd() - require.NoError(t, err) - root := filepath.Join(cwd, "..", "git") - + root := vfs.MustNew("../git") entries := make([]string, 0) - err = filepath.Walk(filepath.Join(root, "testdata", "a"), func(path string, info fs.FileInfo, err error) error { - if !info.IsDir() { + err := fs.WalkDir(root, "testdata/a", func(path string, d fs.DirEntry, err error) error { + if !d.IsDir() { entries = append(entries, path) } return nil @@ -117,23 +104,13 @@ func TestGlobFilesetDir(t *testing.T) { files, err := g.All() require.NoError(t, err) - - require.Equal(t, len(files), len(entries)) - for _, f := range files { - exists := slices.ContainsFunc(entries, func(path string) bool { - return path == f.Absolute - }) - require.True(t, exists) - } + require.ElementsMatch(t, entries, collectRelativePaths(files)) } func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) { - cwd, err := os.Getwd() - require.NoError(t, err) - root := filepath.Join(cwd, "..", "git") - + root := vfs.MustNew("../git") entries := make([]string, 0) - err = filepath.Walk(filepath.Join(root, "testdata"), func(path string, info fs.FileInfo, err error) error { + err := fs.WalkDir(root, "testdata", func(path string, d fs.DirEntry, err error) error { if strings.HasSuffix(path, ".txt") { entries = append(entries, path) } @@ -148,12 +125,5 @@ func TestGlobFilesetDoubleQuotesWithFilePatterns(t *testing.T) { files, err := g.All() require.NoError(t, err) - - require.Equal(t, len(files), len(entries)) - for _, f := range files { - exists := slices.ContainsFunc(entries, func(path string) bool { - return path == f.Absolute - }) - require.True(t, exists) - } + require.ElementsMatch(t, entries, collectRelativePaths(files)) } diff --git a/libs/git/config.go b/libs/git/config.go index e83c75b7b..424d453bc 100644 --- a/libs/git/config.go +++ b/libs/git/config.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" + "github.com/databricks/cli/libs/vfs" "gopkg.in/ini.v1" ) @@ -87,8 +88,8 @@ func (c config) load(r io.Reader) error { return nil } -func (c config) loadFile(path string) error { - f, err := os.Open(path) +func (c config) loadFile(fs vfs.Path, path string) error { + f, err := fs.Open(path) if err != nil { // If the file doesn't exist it is ignored. // This is the case for both global and repository specific config files. @@ -152,8 +153,8 @@ func globalGitConfig() (*config, error) { // > are missing or unreadable they will be ignored. // // We therefore ignore the error return value for the calls below. - config.loadFile(filepath.Join(xdgConfigHome, "git/config")) - config.loadFile(filepath.Join(config.home, ".gitconfig")) + config.loadFile(vfs.MustNew(xdgConfigHome), "git/config") + config.loadFile(vfs.MustNew(config.home), ".gitconfig") return config, nil } diff --git a/libs/git/fileset.go b/libs/git/fileset.go index c604ac7fa..f1986aa20 100644 --- a/libs/git/fileset.go +++ b/libs/git/fileset.go @@ -2,6 +2,7 @@ package git import ( "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" ) // FileSet is Git repository aware implementation of [fileset.FileSet]. @@ -13,7 +14,7 @@ type FileSet struct { } // NewFileSet returns [FileSet] for the Git repository located at `root`. -func NewFileSet(root string) (*FileSet, error) { +func NewFileSet(root vfs.Path) (*FileSet, error) { fs := fileset.New(root) v, err := NewView(root) if err != nil { @@ -34,10 +35,6 @@ func (f *FileSet) IgnoreDirectory(dir string) (bool, error) { return f.view.IgnoreDirectory(dir) } -func (f *FileSet) Root() string { - return f.fileset.Root() -} - func (f *FileSet) All() ([]fileset.File, error) { f.view.repo.taintIgnoreRules() return f.fileset.All() diff --git a/libs/git/fileset_test.go b/libs/git/fileset_test.go index 74133f525..4e6172bfd 100644 --- a/libs/git/fileset_test.go +++ b/libs/git/fileset_test.go @@ -2,23 +2,25 @@ package git import ( "os" + "path" "path/filepath" "strings" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func testFileSetAll(t *testing.T, path string) { - fileSet, err := NewFileSet(path) +func testFileSetAll(t *testing.T, root string) { + fileSet, err := NewFileSet(vfs.MustNew(root)) require.NoError(t, err) files, err := fileSet.All() require.NoError(t, err) require.Len(t, files, 3) - assert.Equal(t, filepath.Join("a", "b", "world.txt"), files[0].Relative) - assert.Equal(t, filepath.Join("a", "hello.txt"), files[1].Relative) - assert.Equal(t, filepath.Join("databricks.yml"), files[2].Relative) + assert.Equal(t, path.Join("a", "b", "world.txt"), files[0].Relative) + assert.Equal(t, path.Join("a", "hello.txt"), files[1].Relative) + assert.Equal(t, path.Join("databricks.yml"), files[2].Relative) } func TestFileSetListAllInRepo(t *testing.T) { @@ -33,7 +35,7 @@ func TestFileSetNonCleanRoot(t *testing.T) { // Test what happens if the root directory can be simplified. // Path simplification is done by most filepath functions. // This should yield the same result as above test. - fileSet, err := NewFileSet("./testdata/../testdata") + fileSet, err := NewFileSet(vfs.MustNew("./testdata/../testdata")) require.NoError(t, err) files, err := fileSet.All() require.NoError(t, err) @@ -42,7 +44,7 @@ func TestFileSetNonCleanRoot(t *testing.T) { func TestFileSetAddsCacheDirToGitIgnore(t *testing.T) { projectDir := t.TempDir() - fileSet, err := NewFileSet(projectDir) + fileSet, err := NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) fileSet.EnsureValidGitIgnoreExists() @@ -57,7 +59,7 @@ func TestFileSetDoesNotCacheDirToGitIgnoreIfAlreadyPresent(t *testing.T) { projectDir := t.TempDir() gitIgnorePath := filepath.Join(projectDir, ".gitignore") - fileSet, err := NewFileSet(projectDir) + fileSet, err := NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) err = os.WriteFile(gitIgnorePath, []byte(".databricks"), 0o644) require.NoError(t, err) diff --git a/libs/git/ignore.go b/libs/git/ignore.go index ec66a2b23..df3a4e919 100644 --- a/libs/git/ignore.go +++ b/libs/git/ignore.go @@ -1,9 +1,12 @@ package git import ( + "io/fs" "os" + "strings" "time" + "github.com/databricks/cli/libs/vfs" ignore "github.com/sabhiram/go-gitignore" ) @@ -21,7 +24,8 @@ type ignoreRules interface { // ignoreFile represents a gitignore file backed by a path. // If the path doesn't exist (yet), it is treated as an empty file. type ignoreFile struct { - absPath string + root vfs.Path + path string // Signal a reload of this file. // Set this to call [os.Stat] and a potential reload @@ -35,9 +39,10 @@ type ignoreFile struct { patterns *ignore.GitIgnore } -func newIgnoreFile(absPath string) ignoreRules { +func newIgnoreFile(root vfs.Path, path string) ignoreRules { return &ignoreFile{ - absPath: absPath, + root: root, + path: path, checkForReload: true, } } @@ -67,7 +72,7 @@ func (f *ignoreFile) Taint() { func (f *ignoreFile) load() error { // The file must be stat-able. // If it doesn't exist, treat it as an empty file. - stat, err := os.Stat(f.absPath) + stat, err := fs.Stat(f.root, f.path) if err != nil { if os.IsNotExist(err) { return nil @@ -82,7 +87,7 @@ func (f *ignoreFile) load() error { } f.modTime = stat.ModTime() - f.patterns, err = ignore.CompileIgnoreFile(f.absPath) + f.patterns, err = f.loadGitignore() if err != nil { return err } @@ -90,6 +95,16 @@ func (f *ignoreFile) load() error { return nil } +func (f *ignoreFile) loadGitignore() (*ignore.GitIgnore, error) { + data, err := fs.ReadFile(f.root, f.path) + if err != nil { + return nil, err + } + + lines := strings.Split(string(data), "\n") + return ignore.CompileIgnoreLines(lines...), nil +} + // stringIgnoreRules implements the [ignoreRules] interface // for a set of in-memory ignore patterns. type stringIgnoreRules struct { diff --git a/libs/git/ignore_test.go b/libs/git/ignore_test.go index 160f53d7b..057c0cb2e 100644 --- a/libs/git/ignore_test.go +++ b/libs/git/ignore_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -13,7 +14,7 @@ func TestIgnoreFile(t *testing.T) { var ign bool var err error - f := newIgnoreFile("./testdata/.gitignore") + f := newIgnoreFile(vfs.MustNew("testdata"), ".gitignore") ign, err = f.MatchesPath("root.foo") require.NoError(t, err) assert.True(t, ign) @@ -27,7 +28,7 @@ func TestIgnoreFileDoesntExist(t *testing.T) { var err error // Files that don't exist are treated as an empty gitignore file. - f := newIgnoreFile("./testdata/thispathdoesntexist") + f := newIgnoreFile(vfs.MustNew("testdata"), "thispathdoesntexist") ign, err = f.MatchesPath("i'm included") require.NoError(t, err) assert.False(t, ign) @@ -41,7 +42,7 @@ func TestIgnoreFileTaint(t *testing.T) { gitIgnorePath := filepath.Join(tempDir, ".gitignore") // Files that don't exist are treated as an empty gitignore file. - f := newIgnoreFile(gitIgnorePath) + f := newIgnoreFile(vfs.MustNew(tempDir), ".gitignore") ign, err = f.MatchesPath("hello") require.NoError(t, err) assert.False(t, ign) diff --git a/libs/git/reference.go b/libs/git/reference.go index 4021f2e60..2b4bd3e4d 100644 --- a/libs/git/reference.go +++ b/libs/git/reference.go @@ -2,10 +2,12 @@ package git import ( "fmt" + "io/fs" "os" - "path/filepath" "regexp" "strings" + + "github.com/databricks/cli/libs/vfs" ) type ReferenceType string @@ -37,9 +39,9 @@ func isSHA1(s string) bool { return re.MatchString(s) } -func LoadReferenceFile(path string) (*Reference, error) { +func LoadReferenceFile(root vfs.Path, path string) (*Reference, error) { // read reference file content - b, err := os.ReadFile(path) + b, err := fs.ReadFile(root, path) if os.IsNotExist(err) { return nil, nil } @@ -73,8 +75,7 @@ func (ref *Reference) ResolvePath() (string, error) { if ref.Type != ReferenceTypePointer { return "", ErrNotAReferencePointer } - refPath := strings.TrimPrefix(ref.Content, ReferencePrefix) - return filepath.FromSlash(refPath), nil + return strings.TrimPrefix(ref.Content, ReferencePrefix), nil } // resolves the name of the current branch from the reference file content. For example @@ -87,8 +88,6 @@ func (ref *Reference) CurrentBranch() (string, error) { if err != nil { return "", err } - // normalize branch ref path to work accross different operating systems - branchRefPath = filepath.ToSlash(branchRefPath) if !strings.HasPrefix(branchRefPath, HeadPathPrefix) { return "", fmt.Errorf("reference path %s does not have expected prefix %s", branchRefPath, HeadPathPrefix) } diff --git a/libs/git/reference_test.go b/libs/git/reference_test.go index 1b08e989b..194d79333 100644 --- a/libs/git/reference_test.go +++ b/libs/git/reference_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,7 +46,7 @@ func TestReferenceReferencePathForReference(t *testing.T) { } path, err := ref.ResolvePath() assert.NoError(t, err) - assert.Equal(t, filepath.FromSlash("refs/heads/my-branch"), path) + assert.Equal(t, "refs/heads/my-branch", path) } func TestReferenceLoadingForObjectID(t *testing.T) { @@ -55,7 +56,7 @@ func TestReferenceLoadingForObjectID(t *testing.T) { defer f.Close() f.WriteString(strings.Repeat("e", 40) + "\r\n") - ref, err := LoadReferenceFile(filepath.Join(tmp, "HEAD")) + ref, err := LoadReferenceFile(vfs.MustNew(tmp), "HEAD") assert.NoError(t, err) assert.Equal(t, ReferenceTypeSHA1, ref.Type) assert.Equal(t, strings.Repeat("e", 40), ref.Content) @@ -68,7 +69,7 @@ func TestReferenceLoadingForReference(t *testing.T) { defer f.Close() f.WriteString("ref: refs/heads/foo\n") - ref, err := LoadReferenceFile(filepath.Join(tmp, "HEAD")) + ref, err := LoadReferenceFile(vfs.MustNew(tmp), "HEAD") assert.NoError(t, err) assert.Equal(t, ReferenceTypePointer, ref.Type) assert.Equal(t, "ref: refs/heads/foo", ref.Content) @@ -81,7 +82,7 @@ func TestReferenceLoadingFailsForInvalidContent(t *testing.T) { defer f.Close() f.WriteString("abc") - _, err = LoadReferenceFile(filepath.Join(tmp, "HEAD")) + _, err = LoadReferenceFile(vfs.MustNew(tmp), "HEAD") assert.ErrorContains(t, err, "unknown format for git HEAD") } diff --git a/libs/git/repository.go b/libs/git/repository.go index 531fd74e4..6baf26c2e 100644 --- a/libs/git/repository.go +++ b/libs/git/repository.go @@ -7,7 +7,7 @@ import ( "path/filepath" "strings" - "github.com/databricks/cli/libs/folders" + "github.com/databricks/cli/libs/vfs" ) const gitIgnoreFileName = ".gitignore" @@ -21,8 +21,8 @@ type Repository struct { // directory where we process .gitignore files. real bool - // rootPath is the absolute path to the repository root. - rootPath string + // root is the absolute path to the repository root. + root vfs.Path // ignore contains a list of ignore patterns indexed by the // path prefix relative to the repository root. @@ -42,12 +42,12 @@ type Repository struct { // Root returns the absolute path to the repository root. func (r *Repository) Root() string { - return r.rootPath + return r.root.Native() } func (r *Repository) CurrentBranch() (string, error) { // load .git/HEAD - ref, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, "HEAD")) + ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD")) if err != nil { return "", err } @@ -64,7 +64,7 @@ func (r *Repository) CurrentBranch() (string, error) { func (r *Repository) LatestCommit() (string, error) { // load .git/HEAD - ref, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, "HEAD")) + ref, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, "HEAD")) if err != nil { return "", err } @@ -83,7 +83,7 @@ func (r *Repository) LatestCommit() (string, error) { if err != nil { return "", err } - branchHeadRef, err := LoadReferenceFile(filepath.Join(r.rootPath, GitDirectoryName, branchHeadPath)) + branchHeadRef, err := LoadReferenceFile(r.root, path.Join(GitDirectoryName, branchHeadPath)) if err != nil { return "", err } @@ -108,7 +108,7 @@ func (r *Repository) loadConfig() error { if err != nil { return fmt.Errorf("unable to load user specific gitconfig: %w", err) } - err = config.loadFile(filepath.Join(r.rootPath, ".git/config")) + err = config.loadFile(r.root, ".git/config") if err != nil { return fmt.Errorf("unable to load repository specific gitconfig: %w", err) } @@ -119,7 +119,7 @@ func (r *Repository) loadConfig() error { // newIgnoreFile constructs a new [ignoreRules] implementation backed by // a file using the specified path relative to the repository root. func (r *Repository) newIgnoreFile(relativeIgnoreFilePath string) ignoreRules { - return newIgnoreFile(filepath.Join(r.rootPath, relativeIgnoreFilePath)) + return newIgnoreFile(r.root, relativeIgnoreFilePath) } // getIgnoreRules returns a slice of [ignoreRules] that apply @@ -132,7 +132,7 @@ func (r *Repository) getIgnoreRules(prefix string) []ignoreRules { return fs } - r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(filepath.Join(prefix, gitIgnoreFileName))) + r.ignore[prefix] = append(r.ignore[prefix], r.newIgnoreFile(path.Join(prefix, gitIgnoreFileName))) return r.ignore[prefix] } @@ -149,7 +149,7 @@ func (r *Repository) taintIgnoreRules() { // Ignore computes whether to ignore the specified path. // The specified path is relative to the repository root path. func (r *Repository) Ignore(relPath string) (bool, error) { - parts := strings.Split(filepath.ToSlash(relPath), "/") + parts := strings.Split(relPath, "/") // Retain trailing slash for directory patterns. // We know a trailing slash was present if the last element @@ -186,14 +186,9 @@ func (r *Repository) Ignore(relPath string) (bool, error) { return false, nil } -func NewRepository(path string) (*Repository, error) { - path, err := filepath.Abs(path) - if err != nil { - return nil, err - } - +func NewRepository(path vfs.Path) (*Repository, error) { real := true - rootPath, err := folders.FindDirWithLeaf(path, GitDirectoryName) + rootPath, err := vfs.FindLeafInTree(path, GitDirectoryName) if err != nil { if !os.IsNotExist(err) { return nil, err @@ -205,9 +200,9 @@ func NewRepository(path string) (*Repository, error) { } repo := &Repository{ - real: real, - rootPath: rootPath, - ignore: make(map[string][]ignoreRules), + real: real, + root: rootPath, + ignore: make(map[string][]ignoreRules), } err = repo.loadConfig() @@ -221,13 +216,21 @@ func NewRepository(path string) (*Repository, error) { return nil, fmt.Errorf("unable to access core excludes file: %w", err) } + // Load global excludes on this machine. + // This is by definition a local path so we create a new [vfs.Path] instance. + coreExcludes := newStringIgnoreRules([]string{}) + if coreExcludesPath != "" { + dir := filepath.Dir(coreExcludesPath) + base := filepath.Base(coreExcludesPath) + coreExcludes = newIgnoreFile(vfs.MustNew(dir), base) + } + // Initialize root ignore rules. // These are special and not lazily initialized because: // 1) we include a hardcoded ignore pattern // 2) we include a gitignore file at a non-standard path repo.ignore["."] = []ignoreRules{ - // Load global excludes on this machine. - newIgnoreFile(coreExcludesPath), + coreExcludes, // Always ignore root .git directory. newStringIgnoreRules([]string{ ".git", diff --git a/libs/git/repository_test.go b/libs/git/repository_test.go index fb0e38080..7ddc7ea79 100644 --- a/libs/git/repository_test.go +++ b/libs/git/repository_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -43,7 +44,7 @@ func newTestRepository(t *testing.T) *testRepository { _, err = f2.WriteString(`ref: refs/heads/main`) require.NoError(t, err) - repo, err := NewRepository(tmp) + repo, err := NewRepository(vfs.MustNew(tmp)) require.NoError(t, err) return &testRepository{ @@ -53,7 +54,7 @@ func newTestRepository(t *testing.T) *testRepository { } func (testRepo *testRepository) checkoutCommit(commitId string) { - f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm) + f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm) require.NoError(testRepo.t, err) defer f.Close() @@ -63,7 +64,7 @@ func (testRepo *testRepository) checkoutCommit(commitId string) { func (testRepo *testRepository) addBranch(name string, latestCommit string) { // create dir for branch head reference - branchDir := filepath.Join(testRepo.r.rootPath, ".git", "refs", "heads") + branchDir := filepath.Join(testRepo.r.Root(), ".git", "refs", "heads") err := os.MkdirAll(branchDir, os.ModePerm) require.NoError(testRepo.t, err) @@ -78,7 +79,7 @@ func (testRepo *testRepository) addBranch(name string, latestCommit string) { } func (testRepo *testRepository) checkoutBranch(name string) { - f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm) + f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "HEAD"), os.O_WRONLY|os.O_TRUNC, os.ModePerm) require.NoError(testRepo.t, err) defer f.Close() @@ -89,7 +90,7 @@ func (testRepo *testRepository) checkoutBranch(name string) { // add remote origin url to test repo func (testRepo *testRepository) addOriginUrl(url string) { // open config in append mode - f, err := os.OpenFile(filepath.Join(testRepo.r.rootPath, ".git", "config"), os.O_WRONLY|os.O_APPEND, os.ModePerm) + f, err := os.OpenFile(filepath.Join(testRepo.r.Root(), ".git", "config"), os.O_WRONLY|os.O_APPEND, os.ModePerm) require.NoError(testRepo.t, err) defer f.Close() @@ -128,7 +129,7 @@ func (testRepo *testRepository) assertOriginUrl(expected string) { func TestRepository(t *testing.T) { // Load this repository as test. - repo, err := NewRepository("../..") + repo, err := NewRepository(vfs.MustNew("../..")) tr := testRepository{t, repo} require.NoError(t, err) @@ -142,7 +143,7 @@ func TestRepository(t *testing.T) { assert.True(t, tr.Ignore("vendor/")) // Check that ignores under testdata work. - assert.True(t, tr.Ignore(filepath.Join("libs", "git", "testdata", "root.ignoreme"))) + assert.True(t, tr.Ignore("libs/git/testdata/root.ignoreme")) } func TestRepositoryGitConfigForEmptyRepo(t *testing.T) { @@ -192,7 +193,7 @@ func TestRepositoryGitConfigForSshUrl(t *testing.T) { func TestRepositoryGitConfigWhenNotARepo(t *testing.T) { tmp := t.TempDir() - repo, err := NewRepository(tmp) + repo, err := NewRepository(vfs.MustNew(tmp)) require.NoError(t, err) branch, err := repo.CurrentBranch() diff --git a/libs/git/view.go b/libs/git/view.go index 3cb88d8b1..90eed0bb8 100644 --- a/libs/git/view.go +++ b/libs/git/view.go @@ -1,9 +1,13 @@ package git import ( + "fmt" "os" + "path" "path/filepath" "strings" + + "github.com/databricks/cli/libs/vfs" ) // View represents a view on a directory tree that takes into account @@ -29,17 +33,15 @@ type View struct { // Ignore computes whether to ignore the specified path. // The specified path is relative to the view's target path. -func (v *View) Ignore(path string) (bool, error) { - path = filepath.ToSlash(path) - +func (v *View) Ignore(relPath string) (bool, error) { // Retain trailing slash for directory patterns. // Needs special handling because it is removed by path cleaning. trailingSlash := "" - if strings.HasSuffix(path, "/") { + if strings.HasSuffix(relPath, "/") { trailingSlash = "/" } - return v.repo.Ignore(filepath.Join(v.targetPath, path) + trailingSlash) + return v.repo.Ignore(path.Join(v.targetPath, relPath) + trailingSlash) } // IgnoreFile returns if the gitignore rules in this fileset @@ -70,26 +72,27 @@ func (v *View) IgnoreDirectory(dir string) (bool, error) { return v.Ignore(dir + "/") } -func NewView(path string) (*View, error) { - path, err := filepath.Abs(path) - if err != nil { - return nil, err - } - - repo, err := NewRepository(path) +func NewView(root vfs.Path) (*View, error) { + repo, err := NewRepository(root) if err != nil { return nil, err } // Target path must be relative to the repository root path. - targetPath, err := filepath.Rel(repo.rootPath, path) - if err != nil { - return nil, err + target := root.Native() + prefix := repo.root.Native() + if !strings.HasPrefix(target, prefix) { + return nil, fmt.Errorf("path %q is not within repository root %q", root.Native(), prefix) } + // Make target a relative path. + target = strings.TrimPrefix(target, prefix) + target = strings.TrimPrefix(target, string(os.PathSeparator)) + target = path.Clean(filepath.ToSlash(target)) + return &View{ repo: repo, - targetPath: targetPath, + targetPath: target, }, nil } diff --git a/libs/git/view_test.go b/libs/git/view_test.go index 3ecd301b5..76fba3458 100644 --- a/libs/git/view_test.go +++ b/libs/git/view_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -89,19 +90,19 @@ func testViewAtRoot(t *testing.T, tv testView) { } func TestViewRootInBricksRepo(t *testing.T) { - v, err := NewView("./testdata") + v, err := NewView(vfs.MustNew("./testdata")) require.NoError(t, err) testViewAtRoot(t, testView{t, v}) } func TestViewRootInTempRepo(t *testing.T) { - v, err := NewView(createFakeRepo(t, "testdata")) + v, err := NewView(vfs.MustNew(createFakeRepo(t, "testdata"))) require.NoError(t, err) testViewAtRoot(t, testView{t, v}) } func TestViewRootInTempDir(t *testing.T) { - v, err := NewView(copyTestdata(t, "testdata")) + v, err := NewView(vfs.MustNew(copyTestdata(t, "testdata"))) require.NoError(t, err) testViewAtRoot(t, testView{t, v}) } @@ -124,20 +125,20 @@ func testViewAtA(t *testing.T, tv testView) { } func TestViewAInBricksRepo(t *testing.T) { - v, err := NewView("./testdata/a") + v, err := NewView(vfs.MustNew("./testdata/a")) require.NoError(t, err) testViewAtA(t, testView{t, v}) } func TestViewAInTempRepo(t *testing.T) { - v, err := NewView(filepath.Join(createFakeRepo(t, "testdata"), "a")) + v, err := NewView(vfs.MustNew(filepath.Join(createFakeRepo(t, "testdata"), "a"))) require.NoError(t, err) testViewAtA(t, testView{t, v}) } func TestViewAInTempDir(t *testing.T) { // Since this is not a fake repo it should not traverse up the tree. - v, err := NewView(filepath.Join(copyTestdata(t, "testdata"), "a")) + v, err := NewView(vfs.MustNew(filepath.Join(copyTestdata(t, "testdata"), "a"))) require.NoError(t, err) tv := testView{t, v} @@ -174,20 +175,20 @@ func testViewAtAB(t *testing.T, tv testView) { } func TestViewABInBricksRepo(t *testing.T) { - v, err := NewView("./testdata/a/b") + v, err := NewView(vfs.MustNew("./testdata/a/b")) require.NoError(t, err) testViewAtAB(t, testView{t, v}) } func TestViewABInTempRepo(t *testing.T) { - v, err := NewView(filepath.Join(createFakeRepo(t, "testdata"), "a", "b")) + v, err := NewView(vfs.MustNew(filepath.Join(createFakeRepo(t, "testdata"), "a", "b"))) require.NoError(t, err) testViewAtAB(t, testView{t, v}) } func TestViewABInTempDir(t *testing.T) { // Since this is not a fake repo it should not traverse up the tree. - v, err := NewView(filepath.Join(copyTestdata(t, "testdata"), "a", "b")) + v, err := NewView(vfs.MustNew(filepath.Join(copyTestdata(t, "testdata"), "a", "b"))) tv := testView{t, v} require.NoError(t, err) @@ -214,7 +215,7 @@ func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredAtRoot(t *testing.T) // Since root .gitignore already has .databricks, there should be no edits // to root .gitignore - v, err := NewView(repoPath) + v, err := NewView(vfs.MustNew(repoPath)) require.NoError(t, err) err = v.EnsureValidGitIgnoreExists() @@ -234,7 +235,7 @@ func TestViewDoesNotChangeGitignoreIfCacheDirAlreadyIgnoredInSubdir(t *testing.T // Since root .gitignore already has .databricks, there should be no edits // to a/.gitignore - v, err := NewView(filepath.Join(repoPath, "a")) + v, err := NewView(vfs.MustNew(filepath.Join(repoPath, "a"))) require.NoError(t, err) err = v.EnsureValidGitIgnoreExists() @@ -252,7 +253,7 @@ func TestViewAddsGitignoreWithCacheDir(t *testing.T) { assert.NoError(t, err) // Since root .gitignore was deleted, new view adds .databricks to root .gitignore - v, err := NewView(repoPath) + v, err := NewView(vfs.MustNew(repoPath)) require.NoError(t, err) err = v.EnsureValidGitIgnoreExists() @@ -270,7 +271,7 @@ func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) { require.NoError(t, err) // Since root .gitignore was deleted, new view adds .databricks to a/.gitignore - v, err := NewView(filepath.Join(repoPath, "a")) + v, err := NewView(vfs.MustNew(filepath.Join(repoPath, "a"))) require.NoError(t, err) err = v.EnsureValidGitIgnoreExists() @@ -287,7 +288,7 @@ func TestViewAddsGitignoreWithCacheDirAtSubdir(t *testing.T) { func TestViewAlwaysIgnoresCacheDir(t *testing.T) { repoPath := createFakeRepo(t, "testdata") - v, err := NewView(repoPath) + v, err := NewView(vfs.MustNew(repoPath)) require.NoError(t, err) err = v.EnsureValidGitIgnoreExists() diff --git a/libs/jsonschema/schema.go b/libs/jsonschema/schema.go index 967e2e9cd..f1e223ec7 100644 --- a/libs/jsonschema/schema.go +++ b/libs/jsonschema/schema.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "slices" + "strings" "github.com/databricks/cli/internal/build" "golang.org/x/mod/semver" @@ -81,6 +82,41 @@ func (s *Schema) ParseString(v string) (any, error) { return fromString(v, s.Type) } +func (s *Schema) getByPath(path string) (*Schema, error) { + p := strings.Split(path, ".") + + res := s + for _, node := range p { + if node == "*" { + res = res.AdditionalProperties.(*Schema) + continue + } + var ok bool + res, ok = res.Properties[node] + if !ok { + return nil, fmt.Errorf("property %q not found in schema. Query path: %s", node, path) + } + } + return res, nil +} + +func (s *Schema) GetByPath(path string) (Schema, error) { + v, err := s.getByPath(path) + if err != nil { + return Schema{}, err + } + return *v, nil +} + +func (s *Schema) SetByPath(path string, v Schema) error { + dst, err := s.getByPath(path) + if err != nil { + return err + } + *dst = v + return nil +} + type Type string const ( @@ -97,7 +133,7 @@ const ( func (schema *Schema) validateSchemaPropertyTypes() error { for _, v := range schema.Properties { switch v.Type { - case NumberType, BooleanType, StringType, IntegerType: + case NumberType, BooleanType, StringType, IntegerType, ObjectType, ArrayType: continue case "int", "int32", "int64": return fmt.Errorf("type %s is not a recognized json schema type. Please use \"integer\" instead", v.Type) diff --git a/libs/jsonschema/schema_test.go b/libs/jsonschema/schema_test.go index cf1f12767..c365cf235 100644 --- a/libs/jsonschema/schema_test.go +++ b/libs/jsonschema/schema_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSchemaValidateTypeNames(t *testing.T) { @@ -305,3 +306,92 @@ func TestValidateSchemaSkippedPropertiesHaveDefaults(t *testing.T) { err = s.validate() assert.NoError(t, err) } + +func testSchema() *Schema { + return &Schema{ + Type: "object", + Properties: map[string]*Schema{ + "int_val": { + Type: "integer", + Default: int64(123), + }, + "string_val": { + Type: "string", + }, + "object_val": { + Type: "object", + Properties: map[string]*Schema{ + "bar": { + Type: "string", + Default: "baz", + }, + }, + AdditionalProperties: &Schema{ + Type: "object", + Properties: map[string]*Schema{ + "foo": { + Type: "string", + Default: "zab", + }, + }, + }, + }, + }, + } + +} + +func TestSchemaGetByPath(t *testing.T) { + s := testSchema() + + ss, err := s.GetByPath("int_val") + require.NoError(t, err) + assert.Equal(t, Schema{ + Type: IntegerType, + Default: int64(123), + }, ss) + + ss, err = s.GetByPath("string_val") + require.NoError(t, err) + assert.Equal(t, Schema{ + Type: StringType, + }, ss) + + ss, err = s.GetByPath("object_val.bar") + require.NoError(t, err) + assert.Equal(t, Schema{ + Type: StringType, + Default: "baz", + }, ss) + + ss, err = s.GetByPath("object_val.*.foo") + require.NoError(t, err) + assert.Equal(t, Schema{ + Type: StringType, + Default: "zab", + }, ss) +} + +func TestSchemaSetByPath(t *testing.T) { + s := testSchema() + + err := s.SetByPath("int_val", Schema{ + Type: IntegerType, + Default: int64(456), + }) + require.NoError(t, err) + assert.Equal(t, int64(456), s.Properties["int_val"].Default) + + err = s.SetByPath("object_val.*.foo", Schema{ + Type: StringType, + Default: "zooby", + }) + require.NoError(t, err) + + ns, err := s.GetByPath("object_val.*.foo") + require.NoError(t, err) + assert.Equal(t, Schema{ + Type: StringType, + Default: "zooby", + }, ns) +} diff --git a/libs/notebook/detect.go b/libs/notebook/detect.go index 17685f3bf..0b7c04d6d 100644 --- a/libs/notebook/detect.go +++ b/libs/notebook/detect.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "io" + "io/fs" "os" "path/filepath" "strings" @@ -15,8 +16,8 @@ import ( const headerLength = 32 // readHeader reads the first N bytes from a file. -func readHeader(path string) ([]byte, error) { - f, err := os.Open(path) +func readHeader(fsys fs.FS, name string) ([]byte, error) { + f, err := fsys.Open(name) if err != nil { return nil, err } @@ -36,10 +37,10 @@ func readHeader(path string) ([]byte, error) { // Detect returns whether the file at path is a Databricks notebook. // If it is, it returns the notebook language. -func Detect(path string) (notebook bool, language workspace.Language, err error) { +func DetectWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { header := "" - buf, err := readHeader(path) + buf, err := readHeader(fsys, name) if err != nil { return false, "", err } @@ -48,7 +49,7 @@ func Detect(path string) (notebook bool, language workspace.Language, err error) fileHeader := scanner.Text() // Determine which header to expect based on filename extension. - ext := strings.ToLower(filepath.Ext(path)) + ext := strings.ToLower(filepath.Ext(name)) switch ext { case ".py": header = `# Databricks notebook source` @@ -63,7 +64,7 @@ func Detect(path string) (notebook bool, language workspace.Language, err error) header = "-- Databricks notebook source" language = workspace.LanguageSql case ".ipynb": - return DetectJupyter(path) + return DetectJupyterWithFS(fsys, name) default: return false, "", nil } @@ -74,3 +75,11 @@ func Detect(path string) (notebook bool, language workspace.Language, err error) return true, language, nil } + +// Detect calls DetectWithFS with the local filesystem. +// The name argument may be a local relative path or a local absolute path. +func Detect(name string) (notebook bool, language workspace.Language, err error) { + d := filepath.ToSlash(filepath.Dir(name)) + b := filepath.Base(name) + return DetectWithFS(os.DirFS(d), b) +} diff --git a/libs/notebook/detect_jupyter.go b/libs/notebook/detect_jupyter.go index 7d96763cd..f631b5812 100644 --- a/libs/notebook/detect_jupyter.go +++ b/libs/notebook/detect_jupyter.go @@ -3,7 +3,9 @@ package notebook import ( "encoding/json" "fmt" + "io/fs" "os" + "path/filepath" "github.com/databricks/databricks-sdk-go/service/workspace" ) @@ -56,8 +58,8 @@ func resolveLanguage(nb *jupyter) workspace.Language { // DetectJupyter returns whether the file at path is a valid Jupyter notebook. // We assume it is valid if we can read it as JSON and see a couple expected fields. // If we cannot, importing into the workspace will always fail, so we also return an error. -func DetectJupyter(path string) (notebook bool, language workspace.Language, err error) { - f, err := os.Open(path) +func DetectJupyterWithFS(fsys fs.FS, name string) (notebook bool, language workspace.Language, err error) { + f, err := fsys.Open(name) if err != nil { return false, "", err } @@ -68,18 +70,26 @@ func DetectJupyter(path string) (notebook bool, language workspace.Language, err dec := json.NewDecoder(f) err = dec.Decode(&nb) if err != nil { - return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", path, err) + return false, "", fmt.Errorf("%s: error loading Jupyter notebook file: %w", name, err) } // Not a Jupyter notebook if the cells or metadata fields aren't defined. if nb.Cells == nil || nb.Metadata == nil { - return false, "", fmt.Errorf("%s: invalid Jupyter notebook file", path) + return false, "", fmt.Errorf("%s: invalid Jupyter notebook file", name) } // Major version must be at least 4. if nb.NbFormatMajor < 4 { - return false, "", fmt.Errorf("%s: unsupported Jupyter notebook version: %d", path, nb.NbFormatMajor) + return false, "", fmt.Errorf("%s: unsupported Jupyter notebook version: %d", name, nb.NbFormatMajor) } return true, resolveLanguage(&nb), nil } + +// DetectJupyter calls DetectJupyterWithFS with the local filesystem. +// The name argument may be a local relative path or a local absolute path. +func DetectJupyter(name string) (notebook bool, language workspace.Language, err error) { + d := filepath.ToSlash(filepath.Dir(name)) + b := filepath.Base(name) + return DetectJupyterWithFS(os.DirFS(d), b) +} diff --git a/libs/sync/diff.go b/libs/sync/diff.go index 074bfc56c..e91f7277e 100644 --- a/libs/sync/diff.go +++ b/libs/sync/diff.go @@ -2,7 +2,6 @@ package sync import ( "path" - "path/filepath" "golang.org/x/exp/maps" ) @@ -64,7 +63,7 @@ func (d *diff) addFilesWithRemoteNameChanged(after *SnapshotState, before *Snaps func (d *diff) addNewFiles(after *SnapshotState, before *SnapshotState) { for localName := range after.LastModifiedTimes { if _, ok := before.LastModifiedTimes[localName]; !ok { - d.put = append(d.put, filepath.ToSlash(localName)) + d.put = append(d.put, localName) } } @@ -79,7 +78,7 @@ func (d *diff) addUpdatedFiles(after *SnapshotState, before *SnapshotState) { for localName, modTime := range after.LastModifiedTimes { prevModTime, ok := before.LastModifiedTimes[localName] if ok && modTime.After(prevModTime) { - d.put = append(d.put, filepath.ToSlash(localName)) + d.put = append(d.put, localName) } } } diff --git a/libs/sync/dirset.go b/libs/sync/dirset.go index 3c37c97cf..33b85cb8e 100644 --- a/libs/sync/dirset.go +++ b/libs/sync/dirset.go @@ -2,7 +2,6 @@ package sync import ( "path" - "path/filepath" "sort" ) @@ -16,8 +15,8 @@ func MakeDirSet(files []string) DirSet { // Iterate over all files. for _, f := range files { - // Get the directory of the file in /-separated form. - dir := filepath.ToSlash(filepath.Dir(f)) + // Get the directory of the file. + dir := path.Dir(f) // Add this directory and its parents until it is either "." or already in the set. for dir != "." { diff --git a/libs/sync/snapshot.go b/libs/sync/snapshot.go index a27a8c84f..392e274d4 100644 --- a/libs/sync/snapshot.go +++ b/libs/sync/snapshot.go @@ -172,6 +172,11 @@ func loadOrNewSnapshot(ctx context.Context, opts *SyncOptions) (*Snapshot, error return nil, fmt.Errorf("failed to json unmarshal persisted snapshot: %s", err) } + // Ensure that all paths are slash-separated upon loading + // an existing snapshot file. If it was created by an older + // CLI version (<= v0.220.0), it may contain backslashes. + snapshot.SnapshotState = snapshot.SnapshotState.ToSlash() + snapshot.New = false return snapshot, nil } diff --git a/libs/sync/snapshot_state.go b/libs/sync/snapshot_state.go index 10cd34e6d..09bb5b63e 100644 --- a/libs/sync/snapshot_state.go +++ b/libs/sync/snapshot_state.go @@ -2,6 +2,7 @@ package sync import ( "fmt" + "path" "path/filepath" "strings" "time" @@ -48,7 +49,7 @@ func NewSnapshotState(localFiles []fileset.File) (*SnapshotState, error) { for k := range localFiles { f := &localFiles[k] // Compute the remote name the file will have in WSFS - remoteName := filepath.ToSlash(f.Relative) + remoteName := f.Relative isNotebook, err := f.IsNotebook() if err != nil { @@ -57,7 +58,7 @@ func NewSnapshotState(localFiles []fileset.File) (*SnapshotState, error) { continue } if isNotebook { - ext := filepath.Ext(remoteName) + ext := path.Ext(remoteName) remoteName = strings.TrimSuffix(remoteName, ext) } @@ -119,3 +120,30 @@ func (fs *SnapshotState) validate() error { } return nil } + +// ToSlash ensures all local paths in the snapshot state +// are slash-separated. Returns a new snapshot state. +func (old SnapshotState) ToSlash() *SnapshotState { + new := SnapshotState{ + LastModifiedTimes: make(map[string]time.Time), + LocalToRemoteNames: make(map[string]string), + RemoteToLocalNames: make(map[string]string), + } + + // Keys are local paths. + for k, v := range old.LastModifiedTimes { + new.LastModifiedTimes[filepath.ToSlash(k)] = v + } + + // Keys are local paths. + for k, v := range old.LocalToRemoteNames { + new.LocalToRemoteNames[filepath.ToSlash(k)] = v + } + + // Values are remote paths. + for k, v := range old.RemoteToLocalNames { + new.RemoteToLocalNames[k] = filepath.ToSlash(v) + } + + return &new +} diff --git a/libs/sync/snapshot_state_test.go b/libs/sync/snapshot_state_test.go index bfcdbef65..92c14e8e0 100644 --- a/libs/sync/snapshot_state_test.go +++ b/libs/sync/snapshot_state_test.go @@ -1,25 +1,27 @@ package sync import ( + "runtime" "testing" "time" "github.com/databricks/cli/libs/fileset" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestSnapshotState(t *testing.T) { - fileSet := fileset.New("./testdata/sync-fileset") + fileSet := fileset.New(vfs.MustNew("./testdata/sync-fileset")) files, err := fileSet.All() require.NoError(t, err) // Assert initial contents of the fileset assert.Len(t, files, 4) - assert.Equal(t, "invalid-nb.ipynb", files[0].Name()) - assert.Equal(t, "my-nb.py", files[1].Name()) - assert.Equal(t, "my-script.py", files[2].Name()) - assert.Equal(t, "valid-nb.ipynb", files[3].Name()) + assert.Equal(t, "invalid-nb.ipynb", files[0].Relative) + assert.Equal(t, "my-nb.py", files[1].Relative) + assert.Equal(t, "my-script.py", files[2].Relative) + assert.Equal(t, "valid-nb.ipynb", files[3].Relative) // Assert snapshot state generated from the fileset. Note that the invalid notebook // has been ignored. @@ -114,3 +116,30 @@ func TestSnapshotStateValidationErrors(t *testing.T) { } assert.EqualError(t, s.validate(), "invalid sync state representation. Inconsistent values found. Remote file c points to a. Local file a points to b") } + +func TestSnapshotStateWithBackslashes(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Skipping test on non-Windows platform") + } + + now := time.Now() + s1 := &SnapshotState{ + LastModifiedTimes: map[string]time.Time{ + "foo\\bar.py": now, + }, + LocalToRemoteNames: map[string]string{ + "foo\\bar.py": "foo/bar", + }, + RemoteToLocalNames: map[string]string{ + "foo/bar": "foo\\bar.py", + }, + } + + assert.NoError(t, s1.validate()) + + s2 := s1.ToSlash() + assert.NoError(t, s1.validate()) + assert.Equal(t, map[string]time.Time{"foo/bar.py": now}, s2.LastModifiedTimes) + assert.Equal(t, map[string]string{"foo/bar.py": "foo/bar"}, s2.LocalToRemoteNames) + assert.Equal(t, map[string]string{"foo/bar": "foo/bar.py"}, s2.RemoteToLocalNames) +} diff --git a/libs/sync/snapshot_test.go b/libs/sync/snapshot_test.go index d6358d4a1..050b5d965 100644 --- a/libs/sync/snapshot_test.go +++ b/libs/sync/snapshot_test.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/testfile" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -29,7 +30,7 @@ func TestDiff(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ @@ -93,7 +94,7 @@ func TestSymlinkDiff(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ @@ -124,7 +125,7 @@ func TestFolderDiff(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ @@ -169,7 +170,7 @@ func TestPythonNotebookDiff(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ @@ -244,7 +245,7 @@ func TestErrorWhenIdenticalRemoteName(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ @@ -281,7 +282,7 @@ func TestNoErrorRenameWithIdenticalRemoteName(t *testing.T) { // Create temp project dir projectDir := t.TempDir() - fileSet, err := git.NewFileSet(projectDir) + fileSet, err := git.NewFileSet(vfs.MustNew(projectDir)) require.NoError(t, err) state := Snapshot{ SnapshotState: &SnapshotState{ diff --git a/libs/sync/sync.go b/libs/sync/sync.go index 30b68ccf3..585e8a887 100644 --- a/libs/sync/sync.go +++ b/libs/sync/sync.go @@ -10,12 +10,13 @@ import ( "github.com/databricks/cli/libs/git" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/set" + "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/iam" ) type SyncOptions struct { - LocalPath string + LocalPath vfs.Path RemotePath string Include []string Exclude []string @@ -54,6 +55,7 @@ func New(ctx context.Context, opts SyncOptions) (*Sync, error) { if err != nil { return nil, err } + err = fileSet.EnsureValidGitIgnoreExists() if err != nil { return nil, err @@ -186,7 +188,7 @@ func (s *Sync) GetFileList(ctx context.Context) ([]fileset.File, error) { // tradeoff: doing portable monitoring only due to macOS max descriptor manual ulimit setting requirement // https://github.com/gorakhargosh/watchdog/blob/master/src/watchdog/observers/kqueue.py#L394-L418 all := set.NewSetF(func(f fileset.File) string { - return f.Absolute + return f.Relative }) gitFiles, err := s.fileSet.All() if err != nil { diff --git a/libs/sync/sync_test.go b/libs/sync/sync_test.go index dc220dbf7..292586e8d 100644 --- a/libs/sync/sync_test.go +++ b/libs/sync/sync_test.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/fileset" "github.com/databricks/cli/libs/git" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/require" ) @@ -73,16 +74,17 @@ func TestGetFileSet(t *testing.T) { ctx := context.Background() dir := setupFiles(t) - fileSet, err := git.NewFileSet(dir) + root := vfs.MustNew(dir) + fileSet, err := git.NewFileSet(root) require.NoError(t, err) err = fileSet.EnsureValidGitIgnoreExists() require.NoError(t, err) - inc, err := fileset.NewGlobSet(dir, []string{}) + inc, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) - excl, err := fileset.NewGlobSet(dir, []string{}) + excl, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) s := &Sync{ @@ -97,10 +99,10 @@ func TestGetFileSet(t *testing.T) { require.NoError(t, err) require.Equal(t, len(fileList), 9) - inc, err = fileset.NewGlobSet(dir, []string{}) + inc, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) - excl, err = fileset.NewGlobSet(dir, []string{"*.go"}) + excl, err = fileset.NewGlobSet(root, []string{"*.go"}) require.NoError(t, err) s = &Sync{ @@ -115,10 +117,10 @@ func TestGetFileSet(t *testing.T) { require.NoError(t, err) require.Equal(t, len(fileList), 1) - inc, err = fileset.NewGlobSet(dir, []string{".databricks/*"}) + inc, err = fileset.NewGlobSet(root, []string{".databricks/*"}) require.NoError(t, err) - excl, err = fileset.NewGlobSet(dir, []string{}) + excl, err = fileset.NewGlobSet(root, []string{}) require.NoError(t, err) s = &Sync{ @@ -138,16 +140,17 @@ func TestRecursiveExclude(t *testing.T) { ctx := context.Background() dir := setupFiles(t) - fileSet, err := git.NewFileSet(dir) + root := vfs.MustNew(dir) + fileSet, err := git.NewFileSet(root) require.NoError(t, err) err = fileSet.EnsureValidGitIgnoreExists() require.NoError(t, err) - inc, err := fileset.NewGlobSet(dir, []string{}) + inc, err := fileset.NewGlobSet(root, []string{}) require.NoError(t, err) - excl, err := fileset.NewGlobSet(dir, []string{"test/**"}) + excl, err := fileset.NewGlobSet(root, []string{"test/**"}) require.NoError(t, err) s := &Sync{ diff --git a/libs/sync/watchdog.go b/libs/sync/watchdog.go index b0c96e01c..ca7ec46e9 100644 --- a/libs/sync/watchdog.go +++ b/libs/sync/watchdog.go @@ -4,8 +4,6 @@ import ( "context" "errors" "io/fs" - "os" - "path/filepath" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" @@ -59,7 +57,7 @@ func (s *Sync) applyMkdir(ctx context.Context, localName string) error { func (s *Sync) applyPut(ctx context.Context, localName string) error { s.notifyProgress(ctx, EventActionPut, localName, 0.0) - localFile, err := os.Open(filepath.Join(s.LocalPath, localName)) + localFile, err := s.LocalPath.Open(localName) if err != nil { return err } diff --git a/libs/textutil/case.go b/libs/textutil/case.go new file mode 100644 index 000000000..a8c780591 --- /dev/null +++ b/libs/textutil/case.go @@ -0,0 +1,14 @@ +package textutil + +import "unicode" + +func CamelToSnakeCase(name string) string { + var out []rune = make([]rune, 0, len(name)*2) + for i, r := range name { + if i > 0 && unicode.IsUpper(r) { + out = append(out, '_') + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} diff --git a/libs/textutil/case_test.go b/libs/textutil/case_test.go new file mode 100644 index 000000000..77b3e0679 --- /dev/null +++ b/libs/textutil/case_test.go @@ -0,0 +1,40 @@ +package textutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCamelToSnakeCase(t *testing.T) { + cases := []struct { + input string + expected string + }{ + { + input: "test", + expected: "test", + }, + { + input: "testTest", + expected: "test_test", + }, + { + input: "testTestTest", + expected: "test_test_test", + }, + { + input: "TestTest", + expected: "test_test", + }, + { + input: "TestTestTest", + expected: "test_test_test", + }, + } + + for _, c := range cases { + output := CamelToSnakeCase(c.input) + assert.Equal(t, c.expected, output) + } +} diff --git a/libs/vfs/leaf.go b/libs/vfs/leaf.go new file mode 100644 index 000000000..8c11f9039 --- /dev/null +++ b/libs/vfs/leaf.go @@ -0,0 +1,29 @@ +package vfs + +import ( + "errors" + "io/fs" +) + +// FindLeafInTree returns the first path that holds `name`, +// traversing up to the root of the filesystem, starting at `p`. +func FindLeafInTree(p Path, name string) (Path, error) { + for p != nil { + _, err := fs.Stat(p, name) + + // No error means we found the leaf in p. + if err == nil { + return p, nil + } + + // ErrNotExist means we continue traversal up the tree. + if errors.Is(err, fs.ErrNotExist) { + p = p.Parent() + continue + } + + return nil, err + } + + return nil, fs.ErrNotExist +} diff --git a/libs/vfs/leaf_test.go b/libs/vfs/leaf_test.go new file mode 100644 index 000000000..da9412ec0 --- /dev/null +++ b/libs/vfs/leaf_test.go @@ -0,0 +1,38 @@ +package vfs + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindLeafInTree(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + root := filepath.Join(wd, "..", "..") + + // Find from working directory should work. + { + out, err := FindLeafInTree(MustNew(wd), ".git") + assert.NoError(t, err) + assert.Equal(t, root, out.Native()) + } + + // Find from project root itself should work. + { + out, err := FindLeafInTree(MustNew(root), ".git") + assert.NoError(t, err) + assert.Equal(t, root, out.Native()) + } + + // Find for something that doesn't exist should work. + { + out, err := FindLeafInTree(MustNew(root), "this-leaf-doesnt-exist-anywhere") + assert.ErrorIs(t, err, os.ErrNotExist) + assert.Equal(t, nil, out) + } +} diff --git a/libs/vfs/os.go b/libs/vfs/os.go new file mode 100644 index 000000000..26447d830 --- /dev/null +++ b/libs/vfs/os.go @@ -0,0 +1,82 @@ +package vfs + +import ( + "io/fs" + "os" + "path/filepath" +) + +type osPath struct { + path string + + openFn func(name string) (fs.File, error) + statFn func(name string) (fs.FileInfo, error) + readDirFn func(name string) ([]fs.DirEntry, error) + readFileFn func(name string) ([]byte, error) +} + +func New(name string) (Path, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + + return newOsPath(abs), nil +} + +func MustNew(name string) Path { + p, err := New(name) + if err != nil { + panic(err) + } + + return p +} + +func newOsPath(name string) Path { + if !filepath.IsAbs(name) { + panic("vfs: abs path must be absolute") + } + + // [os.DirFS] implements all required interfaces. + // We used type assertion below to get the underlying types. + dirfs := os.DirFS(name) + + return &osPath{ + path: name, + + openFn: dirfs.Open, + statFn: dirfs.(fs.StatFS).Stat, + readDirFn: dirfs.(fs.ReadDirFS).ReadDir, + readFileFn: dirfs.(fs.ReadFileFS).ReadFile, + } +} + +func (o osPath) Open(name string) (fs.File, error) { + return o.openFn(name) +} + +func (o osPath) Stat(name string) (fs.FileInfo, error) { + return o.statFn(name) +} + +func (o osPath) ReadDir(name string) ([]fs.DirEntry, error) { + return o.readDirFn(name) +} + +func (o osPath) ReadFile(name string) ([]byte, error) { + return o.readFileFn(name) +} + +func (o osPath) Parent() Path { + dir := filepath.Dir(o.path) + if dir == o.path { + return nil + } + + return newOsPath(dir) +} + +func (o osPath) Native() string { + return o.path +} diff --git a/libs/vfs/os_test.go b/libs/vfs/os_test.go new file mode 100644 index 000000000..6199bdc71 --- /dev/null +++ b/libs/vfs/os_test.go @@ -0,0 +1,54 @@ +package vfs + +import ( + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOsNewWithRelativePath(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + p, err := New(".") + require.NoError(t, err) + require.Equal(t, wd, p.Native()) +} + +func TestOsPathParent(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + p := MustNew(wd) + require.NotNil(t, p) + + // Traverse all the way to the root. + for { + q := p.Parent() + if q == nil { + // Parent returns nil when it is the root. + break + } + + p = q + } + + // We should have reached the root. + if runtime.GOOS == "windows" { + require.Equal(t, filepath.VolumeName(wd)+`\`, p.Native()) + } else { + require.Equal(t, "/", p.Native()) + } +} + +func TestOsPathNative(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + + p := MustNew(wd) + require.NotNil(t, p) + require.Equal(t, wd, p.Native()) +} diff --git a/libs/vfs/path.go b/libs/vfs/path.go new file mode 100644 index 000000000..19f119d50 --- /dev/null +++ b/libs/vfs/path.go @@ -0,0 +1,29 @@ +package vfs + +import "io/fs" + +// FS combines the fs.FS, fs.StatFS, fs.ReadDirFS, and fs.ReadFileFS interfaces. +// It mandates that Path implementations must support all these interfaces. +type FS interface { + fs.FS + fs.StatFS + fs.ReadDirFS + fs.ReadFileFS +} + +// Path defines a read-only virtual file system interface for: +// +// 1. Intercepting file operations to inject custom logic (e.g., logging, access control). +// 2. Traversing directories to find specific leaf directories (e.g., .git). +// 3. Converting virtual paths to OS-native paths. +// +// Options 2 and 3 are not possible with the standard fs.FS interface. +// They are needed such that we can provide an instance to the sync package +// and still detect the containing .git directory and convert paths to native paths. +type Path interface { + FS + + Parent() Path + + Native() string +} diff --git a/libs/vfs/path_test.go b/libs/vfs/path_test.go new file mode 100644 index 000000000..54c60940e --- /dev/null +++ b/libs/vfs/path_test.go @@ -0,0 +1 @@ +package vfs