steve/databricks-cli
steve
/
databricks-cli
mirror of https://github.com/databricks/cli.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'databricks/main' into cp-better-errors

This commit is contained in:
Lennart Kats 2024-06-02 13:17:52 +02:00
parent 7cdfc8d8ff a33d0c8bf9
commit fc07725307
No known key found for this signature in database
GPG Key ID: 1EB8B57673197023
213 changed files with 7155 additions and 1892 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.codegen/_openapi_sha
View File

@ -1 +1 @@
94684175b8bd65f8701f89729351f8069e8309c9
7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92

7
.codegen/service.go.tmpl
View File

@ -39,6 +39,7 @@ import (
{{define "service"}}
{{- $excludeMethods := list "put-secret" -}}
{{- $hideService := .IsPrivatePreview }}
// Slice with functions to override default command behavior.
// Functions can be added from the `init()` function in manually curated files in this directory.
@ -57,7 +58,7 @@ func New() *cobra.Command {
"package": "{{ .Package.Name }}",
},
{{- end }}
{{- if .IsPrivatePreview }}
{{- if $hideService }}
// This service is being previewed; hide from help output.
Hidden: true,
@ -151,6 +152,7 @@ func new{{.PascalName}}() *cobra.Command {
"provider-exchanges delete"
"provider-exchanges delete-listing-from-exchange"
"provider-exchanges list-exchanges-for-listing"
"provider-exchanges list-listings-for-exchange"
-}}
{{- $fullCommandName := (print $serviceName " " .KebabName) -}}
{{- $noPrompt := or .IsCrudCreate (in $excludeFromPrompts $fullCommandName) }}
@ -189,7 +191,8 @@ func new{{.PascalName}}() *cobra.Command {
{{- end -}}
`
{{- end }}
{{- if .IsPrivatePreview }}
{{/* Don't hide commands if the service itself is already hidden. */}}
{{- if and (not $hideService) .IsPrivatePreview }}
// This command is being previewed; hide from help output.
cmd.Hidden = true

6
.gitattributes vendored
View File

@ -37,6 +37,7 @@ cmd/workspace/clean-rooms/clean-rooms.go linguist-generated=true
cmd/workspace/cluster-policies/cluster-policies.go linguist-generated=true
cmd/workspace/clusters/clusters.go linguist-generated=true
cmd/workspace/cmd.go linguist-generated=true
cmd/workspace/compliance-security-profile/compliance-security-profile.go linguist-generated=true
cmd/workspace/connections/connections.go linguist-generated=true
cmd/workspace/consumer-fulfillments/consumer-fulfillments.go linguist-generated=true
cmd/workspace/consumer-installations/consumer-installations.go linguist-generated=true
@ -44,13 +45,12 @@ cmd/workspace/consumer-listings/consumer-listings.go linguist-generated=true
cmd/workspace/consumer-personalization-requests/consumer-personalization-requests.go linguist-generated=true
cmd/workspace/consumer-providers/consumer-providers.go linguist-generated=true
cmd/workspace/credentials-manager/credentials-manager.go linguist-generated=true
cmd/workspace/csp-enablement/csp-enablement.go linguist-generated=true
cmd/workspace/current-user/current-user.go linguist-generated=true
cmd/workspace/dashboard-widgets/dashboard-widgets.go linguist-generated=true
cmd/workspace/dashboards/dashboards.go linguist-generated=true
cmd/workspace/data-sources/data-sources.go linguist-generated=true
cmd/workspace/default-namespace/default-namespace.go linguist-generated=true
cmd/workspace/esm-enablement/esm-enablement.go linguist-generated=true
cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true
cmd/workspace/experiments/experiments.go linguist-generated=true
cmd/workspace/external-locations/external-locations.go linguist-generated=true
cmd/workspace/functions/functions.go linguist-generated=true
@ -62,7 +62,6 @@ cmd/workspace/instance-pools/instance-pools.go linguist-generated=true
cmd/workspace/instance-profiles/instance-profiles.go linguist-generated=true
cmd/workspace/ip-access-lists/ip-access-lists.go linguist-generated=true
cmd/workspace/jobs/jobs.go linguist-generated=true
cmd/workspace/lakehouse-monitors/lakehouse-monitors.go linguist-generated=true
cmd/workspace/lakeview/lakeview.go linguist-generated=true
cmd/workspace/libraries/libraries.go linguist-generated=true
cmd/workspace/metastores/metastores.go linguist-generated=true
@ -81,6 +80,7 @@ cmd/workspace/provider-personalization-requests/provider-personalization-request
cmd/workspace/provider-provider-analytics-dashboards/provider-provider-analytics-dashboards.go linguist-generated=true
cmd/workspace/provider-providers/provider-providers.go linguist-generated=true
cmd/workspace/providers/providers.go linguist-generated=true
cmd/workspace/quality-monitors/quality-monitors.go linguist-generated=true
cmd/workspace/queries/queries.go linguist-generated=true
cmd/workspace/query-history/query-history.go linguist-generated=true
cmd/workspace/query-visualizations/query-visualizations.go linguist-generated=true

76
CHANGELOG.md
View File

@ -1,5 +1,81 @@
# Version changelog
## 0.220.0
CLI:
* Add line about Docker installation to README.md ([#1363](https://github.com/databricks/cli/pull/1363)).
* Improve token refresh flow ([#1434](https://github.com/databricks/cli/pull/1434)).
Bundles:
* Upgrade Terraform provider to v1.42.0 ([#1418](https://github.com/databricks/cli/pull/1418)).
* Upgrade Terraform provider to v1.43.0 ([#1429](https://github.com/databricks/cli/pull/1429)).
* Don't merge-in remote resources during deployments ([#1432](https://github.com/databricks/cli/pull/1432)).
* Remove dependency on `ConfigFilePath` from path translation mutator ([#1437](https://github.com/databricks/cli/pull/1437)).
* Add `merge.Override` transform ([#1428](https://github.com/databricks/cli/pull/1428)).
* Fixed panic when loading incorrectly defined jobs ([#1402](https://github.com/databricks/cli/pull/1402)).
* Add more tests for `merge.Override` ([#1439](https://github.com/databricks/cli/pull/1439)).
* Fixed seg fault when specifying environment key for tasks ([#1443](https://github.com/databricks/cli/pull/1443)).
* Fix conversion of zero valued scalar pointers to a dynamic value ([#1433](https://github.com/databricks/cli/pull/1433)).
Internal:
* Don't hide commands of services that are already hidden ([#1438](https://github.com/databricks/cli/pull/1438)).
API Changes:
* Renamed `lakehouse-monitors` command group to `quality-monitors`.
* Added `apps` command group.
* Renamed `csp-enablement` command group to `compliance-security-profile`.
* Renamed `esm-enablement` command group to `enhanced-security-monitoring`.
* Added `databricks vector-search-indexes scan-index` command.
OpenAPI commit 7eb5ad9a2ed3e3f1055968a2d1014ac92c06fe92 (2024-05-21)
Dependency updates:
* Bump golang.org/x/text from 0.14.0 to 0.15.0 ([#1419](https://github.com/databricks/cli/pull/1419)).
* Bump golang.org/x/oauth2 from 0.19.0 to 0.20.0 ([#1421](https://github.com/databricks/cli/pull/1421)).
* Bump golang.org/x/term from 0.19.0 to 0.20.0 ([#1422](https://github.com/databricks/cli/pull/1422)).
* Bump github.com/databricks/databricks-sdk-go from 0.39.0 to 0.40.1 ([#1431](https://github.com/databricks/cli/pull/1431)).
* Bump github.com/fatih/color from 1.16.0 to 1.17.0 ([#1441](https://github.com/databricks/cli/pull/1441)).
* Bump github.com/hashicorp/terraform-json from 0.21.0 to 0.22.1 ([#1440](https://github.com/databricks/cli/pull/1440)).
* Bump github.com/hashicorp/terraform-exec from 0.20.0 to 0.21.0 ([#1442](https://github.com/databricks/cli/pull/1442)).
* Update Go SDK to v0.41.0 ([#1445](https://github.com/databricks/cli/pull/1445)).
## 0.219.0
Bundles:
* Don't fail while parsing outdated terraform state ([#1404](https://github.com/databricks/cli/pull/1404)).
* Annotate DLT pipelines when deployed using DABs ([#1410](https://github.com/databricks/cli/pull/1410)).
API Changes:
* Changed `databricks libraries cluster-status` command. New request type is compute.ClusterStatus.
* Changed `databricks libraries cluster-status` command to return .
* Added `databricks serving-endpoints get-open-api` command.
OpenAPI commit 21f9f1482f9d0d15228da59f2cd9f0863d2a6d55 (2024-04-23)
Dependency updates:
* Bump github.com/databricks/databricks-sdk-go from 0.38.0 to 0.39.0 ([#1405](https://github.com/databricks/cli/pull/1405)).
## 0.218.1
This is a bugfix release.
CLI:
* Pass `DATABRICKS_CONFIG_FILE` for `auth profiles` ([#1394](https://github.com/databricks/cli/pull/1394)).
Bundles:
* Show a better error message for using wheel tasks with older DBR versions ([#1373](https://github.com/databricks/cli/pull/1373)).
* Allow variable references in non-string fields in the JSON schema ([#1398](https://github.com/databricks/cli/pull/1398)).
* Fix variable overrides in targets for non-string variables ([#1397](https://github.com/databricks/cli/pull/1397)).
* Fix bundle schema for variables ([#1396](https://github.com/databricks/cli/pull/1396)).
* Fix bundle documentation URL ([#1399](https://github.com/databricks/cli/pull/1399)).
Internal:
* Removed autogenerated docs for the CLI commands ([#1392](https://github.com/databricks/cli/pull/1392)).
* Remove `JSON.parse` call from homebrew-tap action ([#1393](https://github.com/databricks/cli/pull/1393)).
* Ensure that Python dependencies are installed during upgrade ([#1390](https://github.com/databricks/cli/pull/1390)).
## 0.218.0
This release marks the general availability of Databricks Asset Bundles.

12
README.md
View File

@ -15,6 +15,18 @@ See https://github.com/databricks/cli/releases for releases and
[the docs pages](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) for
installation instructions.
------
You can use the CLI via a Docker image by pulling the image from `ghcr.io`. You can find all available versions
at: https://github.com/databricks/cli/pkgs/container/cli.
```
docker pull ghcr.io/databricks/cli:latest
```
Example of how to run the CLI using the Docker image. More documentation is available at https://docs.databricks.com/dev-tools/bundles/airgapped-environment.html.
```
docker run -e DATABRICKS_HOST=$YOUR_HOST_URL -e DATABRICKS_TOKEN=$YOUR_TOKEN ghcr.io/databricks/cli:latest current-user me
```
## Authentication
This CLI follows the Databricks Unified Authentication principles.

4
bundle/artifacts/artifacts.go
View File

@ -150,6 +150,10 @@ func uploadArtifact(ctx context.Context, b *bundle.Bundle, a *config.Artifact, u
for i := range job.Environments {
env := &job.Environments[i]
if env.Spec == nil {
continue
}
for j := range env.Spec.Dependencies {
lib := env.Spec.Dependencies[j]
if isArtifactMatchLibrary(f, lib, b) {

3
bundle/bundle.go
View File

@ -22,6 +22,7 @@ import (
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/tags"
"github.com/databricks/cli/libs/terraform"
"github.com/databricks/cli/libs/vfs"
"github.com/databricks/databricks-sdk-go"
sdkconfig "github.com/databricks/databricks-sdk-go/config"
"github.com/hashicorp/terraform-exec/tfexec"
@ -208,7 +209,7 @@ func (b *Bundle) GitRepository() (*git.Repository, error) {
return nil, fmt.Errorf("unable to locate repository root: %w", err)
}
return git.NewRepository(rootPath)
return git.NewRepository(vfs.MustNew(rootPath))
}
// AuthEnv returns a map with environment variables and their values

12
bundle/config/mutator/default_queueing_test.go
View File

@ -56,7 +56,11 @@ func TestDefaultQueueingApplyEnableQueueing(t *testing.T) {
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job": {},
"job": {
JobSettings: &jobs.JobSettings{
Name: "job",
},
},
},
},
},
@ -77,7 +81,11 @@ func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) {
Queue: &jobs.QueueSettings{Enabled: false},
},
},
"job2": {},
"job2": {
JobSettings: &jobs.JobSettings{
Name: "job",
},
},
"job3": {
JobSettings: &jobs.JobSettings{
Queue: &jobs.QueueSettings{Enabled: true},

3
bundle/config/mutator/load_git_details.go
View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/git"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/vfs"
)
type loadGitDetails struct{}
@ -22,7 +23,7 @@ func (m *loadGitDetails) Name() string {
func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
// Load relevant git repository
repo, err := git.NewRepository(b.RootPath)
repo, err := git.NewRepository(vfs.MustNew(b.RootPath))
if err != nil {
return diag.FromErr(err)
}

8
bundle/config/mutator/process_target_mode_test.go
View File

@ -97,6 +97,9 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
RegisteredModels: map[string]*resources.RegisteredModel{
"registeredmodel1": {CreateRegisteredModelRequest: &catalog.CreateRegisteredModelRequest{Name: "registeredmodel1"}},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"qualityMonitor1": {CreateMonitor: &catalog.CreateMonitor{TableName: "qualityMonitor1"}},
},
},
},
// Use AWS implementation for testing.
@ -145,6 +148,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) {
// Registered model 1
assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
// Quality Monitor 1
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
}
func TestProcessTargetModeDevelopmentTagNormalizationForAws(t *testing.T) {
@ -200,6 +206,7 @@ func TestProcessTargetModeDefault(t *testing.T) {
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
}
func TestProcessTargetModeProduction(t *testing.T) {
@ -240,6 +247,7 @@ func TestProcessTargetModeProduction(t *testing.T) {
assert.False(t, b.Config.Resources.Pipelines["pipeline1"].PipelineSpec.Development)
assert.Equal(t, "servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name)
assert.Equal(t, "registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name)
assert.Equal(t, "qualityMonitor1", b.Config.Resources.QualityMonitors["qualityMonitor1"].TableName)
}
func TestProcessTargetModeProductionOkForPrincipal(t *testing.T) {

10
bundle/config/mutator/run_as.go
View File

@ -86,6 +86,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
)
}
// Monitors do not support run_as in the API.
if len(b.Config.Resources.QualityMonitors) > 0 {
return errUnsupportedResourceTypeForRunAs{
resourceType: "quality_monitors",
resourceLocation: b.Config.GetLocation("resources.quality_monitors"),
currentUser: b.Config.Workspace.CurrentUser.UserName,
runAsUser: identity,
}
}
return nil
}

1
bundle/config/mutator/run_as_test.go
View File

@ -37,6 +37,7 @@ func allResourceTypes(t *testing.T) []string {
"model_serving_endpoints",
"models",
"pipelines",
"quality_monitors",
"registered_models",
},
resourceTypes,

2
bundle/config/mutator/set_variables.go
View File

@ -53,8 +53,6 @@ func setVariable(ctx context.Context, v *variable.Variable, name string) diag.Di
}
// We should have had a value to set for the variable at this point.
// TODO: use cmdio to request values for unassigned variables if current
// terminal is a tty. Tracked in https://github.com/databricks/cli/issues/379
return diag.Errorf(`no value assigned to required variable %s. Assignment can be done through the "--var" flag or by setting the %s environment variable`, name, bundleVarPrefix+name)
}

28
bundle/config/mutator/translate_paths.go
View File

@ -213,3 +213,31 @@ func (m *translatePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos
return diag.FromErr(err)
}
func gatherFallbackPaths(v dyn.Value, typ string) (map[string]string, error) {
var fallback = make(map[string]string)
var pattern = dyn.NewPattern(dyn.Key("resources"), dyn.Key(typ), dyn.AnyKey())
// Previous behavior was to use a resource's location as the base path to resolve
// relative paths in its definition. With the introduction of [dyn.Value] throughout,
// we can use the location of the [dyn.Value] of the relative path itself.
//
// This is more flexible, as resources may have overrides that are not
// located in the same directory as the resource configuration file.
//
// To maintain backwards compatibility, we allow relative paths to be resolved using
// the original approach as fallback if the [dyn.Value] location cannot be resolved.
_, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
key := p[2].Key()
dir, err := v.Location().Directory()
if err != nil {
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for %s: %w", p, err)
}
fallback[key] = dir
return v, nil
})
if err != nil {
return nil, err
}
return fallback, nil
}

19
bundle/config/mutator/translate_paths_jobs.go
View File

@ -55,21 +55,14 @@ func rewritePatterns(base dyn.Pattern) []jobRewritePattern {
}
func (m *translatePaths) applyJobTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) {
var fallback = make(map[string]string)
fallback, err := gatherFallbackPaths(v, "jobs")
if err != nil {
return dyn.InvalidValue, err
}
// Do not translate job task paths if using Git source
var ignore []string
var err error
for key, job := range b.Config.Resources.Jobs {
dir, err := job.ConfigFileDirectory()
if err != nil {
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for job %s: %w", key, err)
}
// If we cannot resolve the relative path using the [dyn.Value] location itself,
// use the job's location as fallback. This is necessary for backwards compatibility.
fallback[key] = dir
// Do not translate job task paths if using git source
if job.GitSource != nil {
ignore = append(ignore, key)
}

15
bundle/config/mutator/translate_paths_pipelines.go
View File

@ -8,18 +8,9 @@ import (
)
func (m *translatePaths) applyPipelineTranslations(b *bundle.Bundle, v dyn.Value) (dyn.Value, error) {
var fallback = make(map[string]string)
var err error
for key, pipeline := range b.Config.Resources.Pipelines {
dir, err := pipeline.ConfigFileDirectory()
if err != nil {
return dyn.InvalidValue, fmt.Errorf("unable to determine directory for pipeline %s: %w", key, err)
}
// If we cannot resolve the relative path using the [dyn.Value] location itself,
// use the pipeline's location as fallback. This is necessary for backwards compatibility.
fallback[key] = dir
fallback, err := gatherFallbackPaths(v, "pipelines")
if err != nil {
return dyn.InvalidValue, err
}
// Base pattern to match all libraries in all pipelines.

10
bundle/config/paths/paths.go
View File

@ -1,9 +1,6 @@
package paths
import (
"fmt"
"path/filepath"
"github.com/databricks/cli/libs/dyn"
)
@ -23,10 +20,3 @@ func (p *Paths) ConfigureConfigFilePath() {
}
p.ConfigFilePath = p.DynamicValue.Location().File
}
func (p *Paths) ConfigFileDirectory() (string, error) {
if p.ConfigFilePath == "" {
return "", fmt.Errorf("config file path not configured")
}
return filepath.Dir(p.ConfigFilePath), nil
}

62
bundle/config/resources.go
View File

@ -18,6 +18,7 @@ type Resources struct {
Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"`
ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"`
RegisteredModels map[string]*resources.RegisteredModel `json:"registered_models,omitempty"`
QualityMonitors map[string]*resources.QualityMonitor `json:"quality_monitors,omitempty"`
}
type UniqueResourceIdTracker struct {
@ -28,6 +29,7 @@ type UniqueResourceIdTracker struct {
type ConfigResource interface {
Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error)
TerraformResourceName() string
Validate() error
json.Marshaler
json.Unmarshaler
@ -132,9 +134,66 @@ func (r *Resources) VerifyUniqueResourceIdentifiers() (*UniqueResourceIdTracker,
tracker.Type[k] = "registered_model"
tracker.ConfigPath[k] = r.RegisteredModels[k].ConfigFilePath
}
for k := range r.QualityMonitors {
if _, ok := tracker.Type[k]; ok {
return tracker, fmt.Errorf("multiple resources named %s (%s at %s, %s at %s)",
k,
tracker.Type[k],
tracker.ConfigPath[k],
"quality_monitor",
r.QualityMonitors[k].ConfigFilePath,
)
}
tracker.Type[k] = "quality_monitor"
tracker.ConfigPath[k] = r.QualityMonitors[k].ConfigFilePath
}
return tracker, nil
}
type resource struct {
resource ConfigResource
resource_type string
key string
}
func (r *Resources) allResources() []resource {
all := make([]resource, 0)
for k, e := range r.Jobs {
all = append(all, resource{resource_type: "job", resource: e, key: k})
}
for k, e := range r.Pipelines {
all = append(all, resource{resource_type: "pipeline", resource: e, key: k})
}
for k, e := range r.Models {
all = append(all, resource{resource_type: "model", resource: e, key: k})
}
for k, e := range r.Experiments {
all = append(all, resource{resource_type: "experiment", resource: e, key: k})
}
for k, e := range r.ModelServingEndpoints {
all = append(all, resource{resource_type: "serving endpoint", resource: e, key: k})
}
for k, e := range r.RegisteredModels {
all = append(all, resource{resource_type: "registered model", resource: e, key: k})
}
for k, e := range r.QualityMonitors {
all = append(all, resource{resource_type: "quality monitor", resource: e, key: k})
}
return all
}
func (r *Resources) VerifyAllResourcesDefined() error {
all := r.allResources()
for _, e := range all {
err := e.resource.Validate()
if err != nil {
return fmt.Errorf("%s %s is not defined", e.resource_type, e.key)
}
}
return nil
}
// ConfigureConfigFilePath sets the specified path for all resources contained in this instance.
// This property is used to correctly resolve paths relative to the path
// of the configuration file they were defined in.
@ -157,6 +216,9 @@ func (r *Resources) ConfigureConfigFilePath() {
for _, e := range r.RegisteredModels {
e.ConfigureConfigFilePath()
}
for _, e := range r.QualityMonitors {
e.ConfigureConfigFilePath()
}
}
func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) {

9
bundle/config/resources/job.go
View File

@ -2,6 +2,7 @@ package resources
import (
"context"
"fmt"
"strconv"
"github.com/databricks/cli/bundle/config/paths"
@ -47,3 +48,11 @@ func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id stri
func (j *Job) TerraformResourceName() string {
return "databricks_job"
}
func (j *Job) Validate() error {
if j == nil || !j.DynamicValue.IsValid() || j.JobSettings == nil {
return fmt.Errorf("job is not defined")
}
return nil
}

28
bundle/config/resources/mlflow_experiment.go
View File

@ -1,7 +1,12 @@
package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/ml"
)
@ -23,3 +28,26 @@ func (s *MlflowExperiment) UnmarshalJSON(b []byte) error {
func (s MlflowExperiment) MarshalJSON() ([]byte, error) {
return marshal.Marshal(s)
}
func (s *MlflowExperiment) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.Experiments.GetExperiment(ctx, ml.GetExperimentRequest{
ExperimentId: id,
})
if err != nil {
log.Debugf(ctx, "experiment %s does not exist", id)
return false, err
}
return true, nil
}
func (s *MlflowExperiment) TerraformResourceName() string {
return "databricks_mlflow_experiment"
}
func (s *MlflowExperiment) Validate() error {
if s == nil || !s.DynamicValue.IsValid() {
return fmt.Errorf("experiment is not defined")
}
return nil
}

28
bundle/config/resources/mlflow_model.go
View File

@ -1,7 +1,12 @@
package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/ml"
)
@ -23,3 +28,26 @@ func (s *MlflowModel) UnmarshalJSON(b []byte) error {
func (s MlflowModel) MarshalJSON() ([]byte, error) {
return marshal.Marshal(s)
}
func (s *MlflowModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.ModelRegistry.GetModel(ctx, ml.GetModelRequest{
Name: id,
})
if err != nil {
log.Debugf(ctx, "model %s does not exist", id)
return false, err
}
return true, nil
}
func (s *MlflowModel) TerraformResourceName() string {
return "databricks_mlflow_model"
}
func (s *MlflowModel) Validate() error {
if s == nil || !s.DynamicValue.IsValid() {
return fmt.Errorf("model is not defined")
}
return nil
}

28
bundle/config/resources/model_serving_endpoint.go
View File

@ -1,7 +1,12 @@
package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/serving"
)
@ -33,3 +38,26 @@ func (s *ModelServingEndpoint) UnmarshalJSON(b []byte) error {
func (s ModelServingEndpoint) MarshalJSON() ([]byte, error) {
return marshal.Marshal(s)
}
func (s *ModelServingEndpoint) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.ServingEndpoints.Get(ctx, serving.GetServingEndpointRequest{
Name: id,
})
if err != nil {
log.Debugf(ctx, "serving endpoint %s does not exist", id)
return false, err
}
return true, nil
}
func (s *ModelServingEndpoint) TerraformResourceName() string {
return "databricks_model_serving"
}
func (s *ModelServingEndpoint) Validate() error {
if s == nil || !s.DynamicValue.IsValid() {
return fmt.Errorf("serving endpoint is not defined")
}
return nil
}

9
bundle/config/resources/pipeline.go
View File

@ -2,6 +2,7 @@ package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
@ -42,3 +43,11 @@ func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id
func (p *Pipeline) TerraformResourceName() string {
return "databricks_pipeline"
}
func (p *Pipeline) Validate() error {
if p == nil || !p.DynamicValue.IsValid() {
return fmt.Errorf("pipeline is not defined")
}
return nil
}

60
bundle/config/resources/quality_monitor.go Normal file
View File

@ -0,0 +1,60 @@
package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/catalog"
)
type QualityMonitor struct {
// Represents the Input Arguments for Terraform and will get
// converted to a HCL representation for CRUD
*catalog.CreateMonitor
// This represents the id which is the full name of the monitor
// (catalog_name.schema_name.table_name) that can be used
// as a reference in other resources. This value is returned by terraform.
ID string `json:"id,omitempty" bundle:"readonly"`
// Path to config file where the resource is defined. All bundle resources
// include this for interpolation purposes.
paths.Paths
ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
}
func (s *QualityMonitor) UnmarshalJSON(b []byte) error {
return marshal.Unmarshal(b, s)
}
func (s QualityMonitor) MarshalJSON() ([]byte, error) {
return marshal.Marshal(s)
}
func (s *QualityMonitor) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.QualityMonitors.Get(ctx, catalog.GetQualityMonitorRequest{
TableName: id,
})
if err != nil {
log.Debugf(ctx, "quality monitor %s does not exist", id)
return false, err
}
return true, nil
}
func (s *QualityMonitor) TerraformResourceName() string {
return "databricks_quality_monitor"
}
func (s *QualityMonitor) Validate() error {
if s == nil || !s.DynamicValue.IsValid() {
return fmt.Errorf("quality monitor is not defined")
}
return nil
}

28
bundle/config/resources/registered_model.go
View File

@ -1,7 +1,12 @@
package resources
import (
"context"
"fmt"
"github.com/databricks/cli/bundle/config/paths"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/marshal"
"github.com/databricks/databricks-sdk-go/service/catalog"
)
@ -34,3 +39,26 @@ func (s *RegisteredModel) UnmarshalJSON(b []byte) error {
func (s RegisteredModel) MarshalJSON() ([]byte, error) {
return marshal.Marshal(s)
}
func (s *RegisteredModel) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
_, err := w.RegisteredModels.Get(ctx, catalog.GetRegisteredModelRequest{
FullName: id,
})
if err != nil {
log.Debugf(ctx, "registered model %s does not exist", id)
return false, err
}
return true, nil
}
func (s *RegisteredModel) TerraformResourceName() string {
return "databricks_registered_model"
}
func (s *RegisteredModel) Validate() error {
if s == nil || !s.DynamicValue.IsValid() {
return fmt.Errorf("registered model is not defined")
}
return nil
}

56
bundle/config/resources_test.go
View File

@ -1,6 +1,8 @@
package config
import (
"encoding/json"
"reflect"
"testing"
"github.com/databricks/cli/bundle/config/paths"
@ -125,3 +127,57 @@ func TestVerifySafeMergeForRegisteredModels(t *testing.T) {
err := r.VerifySafeMerge(&other)
assert.ErrorContains(t, err, "multiple resources named bar (registered_model at bar.yml, registered_model at bar2.yml)")
}
// This test ensures that all resources have a custom marshaller and unmarshaller.
// This is required because DABs resources map to Databricks APIs, and they do so
// by embedding the corresponding Go SDK structs.
//
// Go SDK structs often implement custom marshalling and unmarshalling methods (based on the API specifics).
// If the Go SDK struct implements custom marshalling and unmarshalling and we do not
// for the resources at the top level, marshalling and unmarshalling operations will panic.
// Thus we will be overly cautious and ensure that all resources need a custom marshaller and unmarshaller.
//
// Why do we not assert this using an interface to assert MarshalJSON and UnmarshalJSON
// are implemented at the top level?
// If a method is implemented for an embedded struct, the top level struct will
// also have that method and satisfy the interface. This is why we cannot assert
// that the methods are implemented at the top level using an interface.
//
// Why don't we use reflection to assert that the methods are implemented at the
// top level?
// Same problem as above, the golang reflection package does not seem to provide
// a way to directly assert that MarshalJSON and UnmarshalJSON are implemented
// at the top level.
func TestCustomMarshallerIsImplemented(t *testing.T) {
r := Resources{}
rt := reflect.TypeOf(r)
for i := 0; i < rt.NumField(); i++ {
field := rt.Field(i)
// Fields in Resources are expected be of the form map[string]*resourceStruct
assert.Equal(t, field.Type.Kind(), reflect.Map, "Resource %s is not a map", field.Name)
kt := field.Type.Key()
assert.Equal(t, kt.Kind(), reflect.String, "Resource %s is not a map with string keys", field.Name)
vt := field.Type.Elem()
assert.Equal(t, vt.Kind(), reflect.Ptr, "Resource %s is not a map with pointer values", field.Name)
// Marshalling a resourceStruct will panic if resourceStruct does not have a custom marshaller
// This is because resourceStruct embeds a Go SDK struct that implements
// a custom marshaller.
// Eg: resource.Job implements MarshalJSON
v := reflect.Zero(vt.Elem()).Interface()
assert.NotPanics(t, func() {
json.Marshal(v)
}, "Resource %s does not have a custom marshaller", field.Name)
// Unmarshalling a *resourceStruct will panic if the resource does not have a custom unmarshaller
// This is because resourceStruct embeds a Go SDK struct that implements
// a custom unmarshaller.
// Eg: *resource.Job implements UnmarshalJSON
v = reflect.New(vt.Elem()).Interface()
assert.NotPanics(t, func() {
json.Unmarshal([]byte("{}"), v)
}, "Resource %s does not have a custom unmarshaller", field.Name)
}
}

26
bundle/config/root.go
View File

@ -138,6 +138,14 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error {
// Assign the normalized configuration tree.
r.value = nv
// At the moment the check has to be done as part of updateWithDynamicValue
// because otherwise ConfigureConfigFilePath will fail with a panic.
// In the future, we should move this check to a separate mutator in initialise phase.
err = r.Resources.VerifyAllResourcesDefined()
if err != nil {
return err
}
// Assign config file paths after converting to typed configuration.
r.ConfigureConfigFilePath()
return nil
@ -408,15 +416,19 @@ func rewriteShorthands(v dyn.Value) (dyn.Value, error) {
// For each variable, normalize its contents if it is a single string.
return dyn.Map(target, "variables", dyn.Foreach(func(_ dyn.Path, variable dyn.Value) (dyn.Value, error) {
if variable.Kind() != dyn.KindString {
switch variable.Kind() {
case dyn.KindString, dyn.KindBool, dyn.KindFloat, dyn.KindInt:
// Rewrite the variable to a map with a single key called "default".
// This conforms to the variable type. Normalization back to the typed
// configuration will convert this to a string if necessary.
return dyn.NewValue(map[string]dyn.Value{
"default": variable,
}, variable.Location()), nil
default:
return variable, nil
}
// Rewrite the variable to a map with a single key called "default".
// This conforms to the variable type.
return dyn.NewValue(map[string]dyn.Value{
"default": variable,
}, variable.Location()), nil
}))
}))
}

3
bundle/config/validate/validate_sync_patterns.go
View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/fileset"
"github.com/databricks/cli/libs/vfs"
"golang.org/x/sync/errgroup"
)
@ -50,7 +51,7 @@ func checkPatterns(patterns []string, path string, rb bundle.ReadOnlyBundle) (di
index := i
p := pattern
errs.Go(func() error {
fs, err := fileset.NewGlobSet(rb.RootPath(), []string{p})
fs, err := fileset.NewGlobSet(vfs.MustNew(rb.RootPath()), []string{p})
if err != nil {
return err
}

3
bundle/deploy/files/sync.go
View File

@ -6,6 +6,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/sync"
"github.com/databricks/cli/libs/vfs"
)
func GetSync(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.Sync, error) {
@ -28,7 +29,7 @@ func GetSyncOptions(ctx context.Context, rb bundle.ReadOnlyBundle) (*sync.SyncOp
}
opts := &sync.SyncOptions{
LocalPath: rb.RootPath(),
LocalPath: vfs.MustNew(rb.RootPath()),
RemotePath: rb.Config().Workspace.FilePath,
Include: includes,
Exclude: rb.Config().Sync.Exclude,

3
bundle/deploy/metadata/annotate_jobs.go
View File

@ -2,7 +2,6 @@ package metadata
import (
"context"
"path"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
@ -27,7 +26,7 @@ func (m *annotateJobs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti
job.JobSettings.Deployment = &jobs.JobDeployment{
Kind: jobs.JobDeploymentKindBundle,
MetadataFilePath: path.Join(b.Config.Workspace.StatePath, MetadataFileName),
MetadataFilePath: metadataFilePath(b),
}
job.JobSettings.EditMode = jobs.JobEditModeUiLocked
job.JobSettings.Format = jobs.FormatMultiTask

34
bundle/deploy/metadata/annotate_pipelines.go Normal file
View File

@ -0,0 +1,34 @@
package metadata
import (
"context"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/service/pipelines"
)
type annotatePipelines struct{}
func AnnotatePipelines() bundle.Mutator {
return &annotatePipelines{}
}
func (m *annotatePipelines) Name() string {
return "metadata.AnnotatePipelines"
}
func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
for _, pipeline := range b.Config.Resources.Pipelines {
if pipeline.PipelineSpec == nil {
continue
}
pipeline.PipelineSpec.Deployment = &pipelines.PipelineDeployment{
Kind: pipelines.DeploymentKindBundle,
MetadataFilePath: metadataFilePath(b),
}
}
return nil
}

72
bundle/deploy/metadata/annotate_pipelines_test.go Normal file
View File

@ -0,0 +1,72 @@
package metadata
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAnnotatePipelinesMutator(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
StatePath: "/a/b/c",
},
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
"my-pipeline-1": {
PipelineSpec: &pipelines.PipelineSpec{
Name: "My Pipeline One",
},
},
"my-pipeline-2": {
PipelineSpec: &pipelines.PipelineSpec{
Name: "My Pipeline Two",
},
},
},
},
},
}
diags := bundle.Apply(context.Background(), b, AnnotatePipelines())
require.NoError(t, diags.Error())
assert.Equal(t,
&pipelines.PipelineDeployment{
Kind: pipelines.DeploymentKindBundle,
MetadataFilePath: "/a/b/c/metadata.json",
},
b.Config.Resources.Pipelines["my-pipeline-1"].PipelineSpec.Deployment)
assert.Equal(t,
&pipelines.PipelineDeployment{
Kind: pipelines.DeploymentKindBundle,
MetadataFilePath: "/a/b/c/metadata.json",
},
b.Config.Resources.Pipelines["my-pipeline-2"].PipelineSpec.Deployment)
}
func TestAnnotatePipelinesMutatorPipelineWithoutASpec(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Workspace: config.Workspace{
StatePath: "/a/b/c",
},
Resources: config.Resources{
Pipelines: map[string]*resources.Pipeline{
"my-pipeline-1": {},
},
},
},
}
diags := bundle.Apply(context.Background(), b, AnnotatePipelines())
require.NoError(t, diags.Error())
}

9
bundle/deploy/metadata/upload.go
View File

@ -4,13 +4,18 @@ import (
"bytes"
"context"
"encoding/json"
"path"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/filer"
)
const MetadataFileName = "metadata.json"
const metadataFileName = "metadata.json"
func metadataFilePath(b *bundle.Bundle) string {
return path.Join(b.Config.Workspace.StatePath, metadataFileName)
}
type upload struct{}
@ -33,5 +38,5 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
return diag.FromErr(err)
}
return diag.FromErr(f.Write(ctx, MetadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists))
return diag.FromErr(f.Write(ctx, metadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists))
}

13
bundle/deploy/state.go
View File

@ -12,6 +12,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/fileset"
"github.com/databricks/cli/libs/vfs"
)
const DeploymentStateFileName = "deployment.json"
@ -112,12 +113,18 @@ func FromSlice(files []fileset.File) (Filelist, error) {
func (f Filelist) ToSlice(basePath string) []fileset.File {
var files []fileset.File
root := vfs.MustNew(basePath)
for _, file := range f {
absPath := filepath.Join(basePath, file.LocalPath)
entry := newEntry(filepath.Join(basePath, file.LocalPath))
// Snapshots created with versions <= v0.220.0 use platform-specific
// paths (i.e. with backslashes). Files returned by [libs/fileset] always
// contain forward slashes after this version. Normalize before using.
relative := filepath.ToSlash(file.LocalPath)
if file.IsNotebook {
files = append(files, fileset.NewNotebookFile(newEntry(absPath), absPath, file.LocalPath))
files = append(files, fileset.NewNotebookFile(root, entry, relative))
} else {
files = append(files, fileset.NewSourceFile(newEntry(absPath), absPath, file.LocalPath))
files = append(files, fileset.NewSourceFile(root, entry, relative))
}
}
return files

23
bundle/deploy/state_test.go
View File

@ -3,17 +3,17 @@ package deploy
import (
"bytes"
"encoding/json"
"path/filepath"
"testing"
"github.com/databricks/cli/internal/testutil"
"github.com/databricks/cli/libs/fileset"
"github.com/databricks/cli/libs/vfs"
"github.com/stretchr/testify/require"
)
func TestFromSlice(t *testing.T) {
tmpDir := t.TempDir()
fileset := fileset.New(tmpDir)
fileset := fileset.New(vfs.MustNew(tmpDir))
testutil.Touch(t, tmpDir, "test1.py")
testutil.Touch(t, tmpDir, "test2.py")
testutil.Touch(t, tmpDir, "test3.py")
@ -32,7 +32,7 @@ func TestFromSlice(t *testing.T) {
func TestToSlice(t *testing.T) {
tmpDir := t.TempDir()
fileset := fileset.New(tmpDir)
fileset := fileset.New(vfs.MustNew(tmpDir))
testutil.Touch(t, tmpDir, "test1.py")
testutil.Touch(t, tmpDir, "test2.py")
testutil.Touch(t, tmpDir, "test3.py")
@ -48,18 +48,11 @@ func TestToSlice(t *testing.T) {
require.Len(t, s, 3)
for _, file := range s {
require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Name())
require.Contains(t, []string{
filepath.Join(tmpDir, "test1.py"),
filepath.Join(tmpDir, "test2.py"),
filepath.Join(tmpDir, "test3.py"),
}, file.Absolute)
require.False(t, file.IsDir())
require.NotZero(t, file.Type())
info, err := file.Info()
require.NoError(t, err)
require.NotNil(t, info)
require.Equal(t, file.Name(), info.Name())
require.Contains(t, []string{"test1.py", "test2.py", "test3.py"}, file.Relative)
// If the mtime is not zero we know we produced a valid fs.DirEntry.
ts := file.Modified()
require.NotZero(t, ts)
}
}

93
bundle/deploy/check_running_resources.go → bundle/deploy/terraform/check_running_resources.go
View File

@ -1,4 +1,4 @@
package deploy
package terraform
import (
"context"
@ -10,7 +10,6 @@ import (
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
"golang.org/x/sync/errgroup"
)
@ -36,26 +35,16 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia
return nil
}
tf := b.Terraform
if tf == nil {
return diag.Errorf("terraform not initialized")
}
err := tf.Init(ctx, tfexec.Upgrade(true))
if err != nil {
return diag.Errorf("terraform init: %v", err)
}
state, err := b.Terraform.Show(ctx)
if err != nil {
state, err := ParseResourcesState(ctx, b)
if err != nil && state == nil {
return diag.FromErr(err)
}
err = checkAnyResourceRunning(ctx, b.WorkspaceClient(), state)
w := b.WorkspaceClient()
err = checkAnyResourceRunning(ctx, w, state)
if err != nil {
return diag.Errorf("deployment aborted, err: %v", err)
return diag.FromErr(err)
}
return nil
}
@ -63,53 +52,49 @@ func CheckRunningResource() *checkRunningResources {
return &checkRunningResources{}
}
func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *tfjson.State) error {
if state.Values == nil || state.Values.RootModule == nil {
func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *resourcesState) error {
if state == nil {
return nil
}
errs, errCtx := errgroup.WithContext(ctx)
for _, resource := range state.Values.RootModule.Resources {
// Limit to resources.
for _, resource := range state.Resources {
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
for _, instance := range resource.Instances {
id := instance.Attributes.ID
if id == "" {
continue
}
value, ok := resource.AttributeValues["id"]
if !ok {
continue
}
id, ok := value.(string)
if !ok {
continue
}
switch resource.Type {
case "databricks_job":
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
return nil
})
case "databricks_pipeline":
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
switch resource.Type {
case "databricks_job":
errs.Go(func() error {
isRunning, err := IsJobRunning(errCtx, w, id)
// If there's an error retrieving the job, we assume it's not running
if err != nil {
return err
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
}
return nil
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
})
case "databricks_pipeline":
errs.Go(func() error {
isRunning, err := IsPipelineRunning(errCtx, w, id)
// If there's an error retrieving the pipeline, we assume it's not running
if err != nil {
return nil
}
if isRunning {
return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
}
return nil
})
}
}
}

73
bundle/deploy/check_running_resources_test.go → bundle/deploy/terraform/check_running_resources_test.go
View File

@ -1,4 +1,4 @@
package deploy
package terraform
import (
"context"
@ -8,31 +8,26 @@ import (
"github.com/databricks/databricks-sdk-go/experimental/mocks"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/databricks/databricks-sdk-go/service/pipelines"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
mock := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{}
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &resourcesState{})
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithJob(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
resources := &resourcesState{
Resources: []stateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "job1",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "123"}},
},
},
},
@ -46,7 +41,7 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
{RunId: 1234},
}, nil).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "job 123 is running")
jobsApi.EXPECT().ListRunsAll(mock.Anything, jobs.ListRunsRequest{
@ -54,23 +49,20 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
ActiveOnly: true,
}).Return([]jobs.BaseRun{}, nil).Once()
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
resources := &resourcesState{
Resources: []stateResource{
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "pipeline1",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "123"}},
},
},
},
@ -84,7 +76,7 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
State: pipelines.PipelineStateRunning,
}, nil).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.ErrorContains(t, err, "pipeline 123 is running")
pipelineApi.EXPECT().Get(mock.Anything, pipelines.GetPipelineRequest{
@ -93,23 +85,20 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
PipelineId: "123",
State: pipelines.PipelineStateIdle,
}, nil).Once()
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err = checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}
func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
m := mocks.NewMockWorkspaceClient(t)
state := &tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_pipeline",
AttributeValues: map[string]interface{}{
"id": "123",
},
Mode: tfjson.ManagedResourceMode,
},
resources := &resourcesState{
Resources: []stateResource{
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "pipeline1",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "123"}},
},
},
},
@ -120,6 +109,6 @@ func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
PipelineId: "123",
}).Return(nil, errors.New("API failure")).Once()
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, state)
err := checkAnyResourceRunning(context.Background(), m.WorkspaceClient, resources)
require.NoError(t, err)
}

110
bundle/deploy/terraform/convert.go
View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"reflect"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
@ -19,15 +18,6 @@ func conv(from any, to any) {
json.Unmarshal(buf, &to)
}
func convRemoteToLocal(remote any, local any) resources.ModifiedStatus {
var modifiedStatus resources.ModifiedStatus
if reflect.ValueOf(local).Elem().IsNil() {
modifiedStatus = resources.ModifiedStatusDeleted
}
conv(remote, local)
return modifiedStatus
}
func convPermissions(acl []resources.Permission) *schema.ResourcePermissions {
if len(acl) == 0 {
return nil
@ -232,6 +222,13 @@ func BundleToTerraform(config *config.Root) *schema.Root {
}
}
for k, src := range config.Resources.QualityMonitors {
noResources = false
var dst schema.ResourceQualityMonitor
conv(src, &dst)
tfroot.Resource.QualityMonitor[k] = &dst
}
// We explicitly set "resource" to nil to omit it from a JSON encoding.
// This is required because the terraform CLI requires >= 1 resources defined
// if the "resource" property is used in a .tf.json file.
@ -248,7 +245,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
tfroot.Provider = schema.NewProviders()
// Convert each resource in the bundle to the equivalent Terraform representation.
resources, err := dyn.Get(root, "resources")
dynResources, err := dyn.Get(root, "resources")
if err != nil {
// If the resources key is missing, return an empty root.
if dyn.IsNoSuchKeyError(err) {
@ -260,11 +257,20 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
tfroot.Resource = schema.NewResources()
numResources := 0
_, err = dyn.Walk(resources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
_, err = dyn.Walk(dynResources, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
if len(p) < 2 {
return v, nil
}
// Skip resources that have been deleted locally.
modifiedStatus, err := dyn.Get(v, "modified_status")
if err == nil {
modifiedStatusStr, ok := modifiedStatus.AsString()
if ok && modifiedStatusStr == resources.ModifiedStatusDeleted {
return v, dyn.ErrSkip
}
}
typ := p[0].Key()
key := p[1].Key()
@ -275,7 +281,7 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
}
// Convert resource to Terraform representation.
err := c.Convert(ctx, key, v, tfroot.Resource)
err = c.Convert(ctx, key, v, tfroot.Resource)
if err != nil {
return dyn.InvalidValue, err
}
@ -299,76 +305,83 @@ func BundleToTerraformWithDynValue(ctx context.Context, root dyn.Value) (*schema
return tfroot, nil
}
func TerraformToBundle(state *tfjson.State, config *config.Root) error {
if state.Values != nil && state.Values.RootModule != nil {
for _, resource := range state.Values.RootModule.Resources {
// Limit to resources.
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
func TerraformToBundle(state *resourcesState, config *config.Root) error {
for _, resource := range state.Resources {
if resource.Mode != tfjson.ManagedResourceMode {
continue
}
for _, instance := range resource.Instances {
switch resource.Type {
case "databricks_job":
var tmp schema.ResourceJob
conv(resource.AttributeValues, &tmp)
if config.Resources.Jobs == nil {
config.Resources.Jobs = make(map[string]*resources.Job)
}
cur := config.Resources.Jobs[resource.Name]
// TODO: make sure we can unmarshall tf state properly and don't swallow errors
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.Job{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Jobs[resource.Name] = cur
case "databricks_pipeline":
var tmp schema.ResourcePipeline
conv(resource.AttributeValues, &tmp)
if config.Resources.Pipelines == nil {
config.Resources.Pipelines = make(map[string]*resources.Pipeline)
}
cur := config.Resources.Pipelines[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.Pipeline{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Pipelines[resource.Name] = cur
case "databricks_mlflow_model":
var tmp schema.ResourceMlflowModel
conv(resource.AttributeValues, &tmp)
if config.Resources.Models == nil {
config.Resources.Models = make(map[string]*resources.MlflowModel)
}
cur := config.Resources.Models[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.MlflowModel{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Models[resource.Name] = cur
case "databricks_mlflow_experiment":
var tmp schema.ResourceMlflowExperiment
conv(resource.AttributeValues, &tmp)
if config.Resources.Experiments == nil {
config.Resources.Experiments = make(map[string]*resources.MlflowExperiment)
}
cur := config.Resources.Experiments[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.MlflowExperiment{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.Experiments[resource.Name] = cur
case "databricks_model_serving":
var tmp schema.ResourceModelServing
conv(resource.AttributeValues, &tmp)
if config.Resources.ModelServingEndpoints == nil {
config.Resources.ModelServingEndpoints = make(map[string]*resources.ModelServingEndpoint)
}
cur := config.Resources.ModelServingEndpoints[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.ModelServingEndpoint{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.ModelServingEndpoints[resource.Name] = cur
case "databricks_registered_model":
var tmp schema.ResourceRegisteredModel
conv(resource.AttributeValues, &tmp)
if config.Resources.RegisteredModels == nil {
config.Resources.RegisteredModels = make(map[string]*resources.RegisteredModel)
}
cur := config.Resources.RegisteredModels[resource.Name]
modifiedStatus := convRemoteToLocal(tmp, &cur)
cur.ModifiedStatus = modifiedStatus
if cur == nil {
cur = &resources.RegisteredModel{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.RegisteredModels[resource.Name] = cur
case "databricks_quality_monitor":
if config.Resources.QualityMonitors == nil {
config.Resources.QualityMonitors = make(map[string]*resources.QualityMonitor)
}
cur := config.Resources.QualityMonitors[resource.Name]
if cur == nil {
cur = &resources.QualityMonitor{ModifiedStatus: resources.ModifiedStatusDeleted}
}
cur.ID = instance.Attributes.ID
config.Resources.QualityMonitors[resource.Name] = cur
case "databricks_permissions":
case "databricks_grants":
// Ignore; no need to pull these back into the configuration.
@ -408,6 +421,11 @@ func TerraformToBundle(state *tfjson.State, config *config.Root) error {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
for _, src := range config.Resources.QualityMonitors {
if src.ModifiedStatus == "" && src.ID == "" {
src.ModifiedStatus = resources.ModifiedStatusCreated
}
}
return nil
}

348
bundle/deploy/terraform/convert_test.go
View File

@ -17,7 +17,6 @@ import (
"github.com/databricks/databricks-sdk-go/service/ml"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/databricks/databricks-sdk-go/service/serving"
tfjson "github.com/hashicorp/terraform-json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -548,50 +547,94 @@ func TestBundleToTerraformRegisteredModelGrants(t *testing.T) {
bundleToTerraformEquivalenceTest(t, &config)
}
func TestBundleToTerraformDeletedResources(t *testing.T) {
var job1 = resources.Job{
JobSettings: &jobs.JobSettings{},
}
var job2 = resources.Job{
ModifiedStatus: resources.ModifiedStatusDeleted,
JobSettings: &jobs.JobSettings{},
}
var config = config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"my_job1": &job1,
"my_job2": &job2,
},
},
}
vin, err := convert.FromTyped(config, dyn.NilValue)
require.NoError(t, err)
out, err := BundleToTerraformWithDynValue(context.Background(), vin)
require.NoError(t, err)
_, ok := out.Resource.Job["my_job1"]
assert.True(t, ok)
_, ok = out.Resource.Job["my_job2"]
assert.False(t, ok)
}
func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
var config = config.Root{
Resources: config.Resources{},
}
var tfState = tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
var tfState = resourcesState{
Resources: []stateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_quality_monitor",
Mode: "managed",
Name: "test_monitor",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
},
@ -617,6 +660,9 @@ func TestTerraformToBundleEmptyLocalResources(t *testing.T) {
assert.Equal(t, "1", config.Resources.RegisteredModels["test_registered_model"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus)
assert.Equal(t, "1", config.Resources.QualityMonitors["test_monitor"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -665,10 +711,17 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
},
},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"test_monitor": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "test_monitor",
},
},
},
},
}
var tfState = tfjson.State{
Values: nil,
var tfState = resourcesState{
Resources: nil,
}
err := TerraformToBundle(&tfState, &config)
assert.NoError(t, err)
@ -691,6 +744,9 @@ func TestTerraformToBundleEmptyRemoteResources(t *testing.T) {
assert.Equal(t, "", config.Resources.RegisteredModels["test_registered_model"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.RegisteredModels["test_registered_model"].ModifiedStatus)
assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}
@ -769,84 +825,132 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
},
},
},
QualityMonitors: map[string]*resources.QualityMonitor{
"test_monitor": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "test_monitor",
},
},
"test_monitor_new": {
CreateMonitor: &catalog.CreateMonitor{
TableName: "test_monitor_new",
},
},
},
},
}
var tfState = tfjson.State{
Values: &tfjson.StateValues{
RootModule: &tfjson.StateModule{
Resources: []*tfjson.StateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
AttributeValues: map[string]interface{}{"id": "1"},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model_old",
AttributeValues: map[string]interface{}{"id": "2"},
},
var tfState = resourcesState{
Resources: []stateResource{
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_job",
Mode: "managed",
Name: "test_job_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_pipeline",
Mode: "managed",
Name: "test_pipeline_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_model",
Mode: "managed",
Name: "test_mlflow_model_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_mlflow_experiment",
Mode: "managed",
Name: "test_mlflow_experiment_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_model_serving",
Mode: "managed",
Name: "test_model_serving_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "1"}},
},
},
{
Type: "databricks_registered_model",
Mode: "managed",
Name: "test_registered_model_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "2"}},
},
},
{
Type: "databricks_quality_monitor",
Mode: "managed",
Name: "test_monitor",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "test_monitor"}},
},
},
{
Type: "databricks_quality_monitor",
Mode: "managed",
Name: "test_monitor_old",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "test_monitor_old"}},
},
},
},
@ -896,6 +1000,12 @@ func TestTerraformToBundleModifiedResources(t *testing.T) {
assert.Equal(t, "", config.Resources.ModelServingEndpoints["test_model_serving_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.ModelServingEndpoints["test_model_serving_new"].ModifiedStatus)
assert.Equal(t, "test_monitor", config.Resources.QualityMonitors["test_monitor"].ID)
assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor"].ModifiedStatus)
assert.Equal(t, "test_monitor_old", config.Resources.QualityMonitors["test_monitor_old"].ID)
assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.QualityMonitors["test_monitor_old"].ModifiedStatus)
assert.Equal(t, "", config.Resources.QualityMonitors["test_monitor_new"].ID)
assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.QualityMonitors["test_monitor_new"].ModifiedStatus)
AssertFullResourceCoverage(t, &config)
}

2
bundle/deploy/terraform/interpolate.go
View File

@ -54,6 +54,8 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D
path = dyn.NewPath(dyn.Key("databricks_model_serving")).Append(path[2:]...)
case dyn.Key("registered_models"):
path = dyn.NewPath(dyn.Key("databricks_registered_model")).Append(path[2:]...)
case dyn.Key("quality_monitors"):
path = dyn.NewPath(dyn.Key("databricks_quality_monitor")).Append(path[2:]...)
default:
// Trigger "key not found" for unknown resource types.
return dyn.GetByPath(root, path)

16
bundle/deploy/terraform/load.go
View File

@ -8,7 +8,6 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/hashicorp/terraform-exec/tfexec"
tfjson "github.com/hashicorp/terraform-json"
)
type loadMode int
@ -34,7 +33,7 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
return diag.Errorf("terraform init: %v", err)
}
state, err := b.Terraform.Show(ctx)
state, err := ParseResourcesState(ctx, b)
if err != nil {
return diag.FromErr(err)
}
@ -53,16 +52,13 @@ func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
return nil
}
func (l *load) validateState(state *tfjson.State) error {
if state.Values == nil {
if slices.Contains(l.modes, ErrorOnEmptyState) {
return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
}
return nil
func (l *load) validateState(state *resourcesState) error {
if state.Version != SupportedStateVersion {
return fmt.Errorf("unsupported deployment state version: %d. Try re-deploying the bundle", state.Version)
}
if state.Values.RootModule == nil {
return fmt.Errorf("malformed terraform state: RootModule not set")
if len(state.Resources) == 0 && slices.Contains(l.modes, ErrorOnEmptyState) {
return fmt.Errorf("no deployment state. Did you forget to run 'databricks bundle deploy'?")
}
return nil

37
bundle/deploy/terraform/tfdyn/convert_quality_monitor.go Normal file
View File

@ -0,0 +1,37 @@
package tfdyn
import (
"context"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/cli/libs/log"
)
func convertQualityMonitorResource(ctx context.Context, vin dyn.Value) (dyn.Value, error) {
// Normalize the output value to the target schema.
vout, diags := convert.Normalize(schema.ResourceQualityMonitor{}, vin)
for _, diag := range diags {
log.Debugf(ctx, "monitor normalization diagnostic: %s", diag.Summary)
}
return vout, nil
}
type qualityMonitorConverter struct{}
func (qualityMonitorConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
vout, err := convertQualityMonitorResource(ctx, vin)
if err != nil {
return err
}
// Add the converted resource to the output.
out.QualityMonitor[key] = vout.AsAny()
return nil
}
func init() {
registerConverter("quality_monitors", qualityMonitorConverter{})
}

46
bundle/deploy/terraform/tfdyn/convert_quality_monitor_test.go Normal file
View File

@ -0,0 +1,46 @@
package tfdyn
import (
"context"
"testing"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/bundle/internal/tf/schema"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestConvertQualityMonitor(t *testing.T) {
var src = resources.QualityMonitor{
CreateMonitor: &catalog.CreateMonitor{
TableName: "test_table_name",
AssetsDir: "assets_dir",
OutputSchemaName: "output_schema_name",
InferenceLog: &catalog.MonitorInferenceLog{
ModelIdCol: "model_id",
PredictionCol: "test_prediction_col",
ProblemType: "PROBLEM_TYPE_CLASSIFICATION",
},
},
}
vin, err := convert.FromTyped(src, dyn.NilValue)
require.NoError(t, err)
ctx := context.Background()
out := schema.NewResources()
err = qualityMonitorConverter{}.Convert(ctx, "my_monitor", vin, out)
require.NoError(t, err)
assert.Equal(t, map[string]any{
"assets_dir": "assets_dir",
"output_schema_name": "output_schema_name",
"table_name": "test_table_name",
"inference_log": map[string]any{
"model_id_col": "model_id",
"prediction_col": "test_prediction_col",
"problem_type": "PROBLEM_TYPE_CLASSIFICATION",
},
}, out.QualityMonitor["my_monitor"])
}

55
bundle/deploy/terraform/util.go
View File

@ -1,14 +1,46 @@
package terraform
import (
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"github.com/databricks/cli/bundle"
tfjson "github.com/hashicorp/terraform-json"
)
type state struct {
// Partial representation of the Terraform state file format.
// We are only interested global version and serial numbers,
// plus resource types, names, modes, and ids.
type resourcesState struct {
Version int `json:"version"`
Resources []stateResource `json:"resources"`
}
const SupportedStateVersion = 4
type serialState struct {
Serial int `json:"serial"`
}
type stateResource struct {
Type string `json:"type"`
Name string `json:"name"`
Mode tfjson.ResourceMode `json:"mode"`
Instances []stateResourceInstance `json:"instances"`
}
type stateResourceInstance struct {
Attributes stateInstanceAttributes `json:"attributes"`
}
type stateInstanceAttributes struct {
ID string `json:"id"`
}
func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
localState, err := loadState(local)
if err != nil {
@ -23,12 +55,12 @@ func IsLocalStateStale(local io.Reader, remote io.Reader) bool {
return localState.Serial < remoteState.Serial
}
func loadState(input io.Reader) (*state, error) {
func loadState(input io.Reader) (*serialState, error) {
content, err := io.ReadAll(input)
if err != nil {
return nil, err
}
var s state
var s serialState
err = json.Unmarshal(content, &s)
if err != nil {
return nil, err
@ -36,3 +68,20 @@ func loadState(input io.Reader) (*state, error) {
return &s, nil
}
func ParseResourcesState(ctx context.Context, b *bundle.Bundle) (*resourcesState, error) {
cacheDir, err := Dir(ctx, b)
if err != nil {
return nil, err
}
rawState, err := os.ReadFile(filepath.Join(cacheDir, TerraformStateFileName))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return &resourcesState{Version: SupportedStateVersion}, nil
}
return nil, err
}
var state resourcesState
err = json.Unmarshal(rawState, &state)
return &state, err
}

99
bundle/deploy/terraform/util_test.go
View File

@ -1,11 +1,16 @@
package terraform
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"testing/iotest"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/stretchr/testify/assert"
)
@ -38,3 +43,97 @@ func TestLocalStateMarkNonStaleWhenRemoteFailsToLoad(t *testing.T) {
remote := iotest.ErrReader(fmt.Errorf("Random error"))
assert.False(t, IsLocalStateStale(local, remote))
}
func TestParseResourcesStateWithNoFile(t *testing.T) {
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},
},
},
}
state, err := ParseResourcesState(context.Background(), b)
assert.NoError(t, err)
assert.Equal(t, &resourcesState{Version: SupportedStateVersion}, state)
}
func TestParseResourcesStateWithExistingStateFile(t *testing.T) {
ctx := context.Background()
b := &bundle.Bundle{
RootPath: t.TempDir(),
Config: config.Root{
Bundle: config.Bundle{
Target: "whatever",
Terraform: &config.Terraform{
ExecPath: "terraform",
},
},
},
}
cacheDir, err := Dir(ctx, b)
assert.NoError(t, err)
data := []byte(`{
"version": 4,
"unknown_field": "hello",
"resources": [
{
"mode": "managed",
"type": "databricks_pipeline",
"name": "test_pipeline",
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"allow_duplicate_names": false,
"catalog": null,
"channel": "CURRENT",
"cluster": [],
"random_field": "random_value",
"configuration": {
"bundle.sourcePath": "/Workspace//Users/user/.bundle/test/dev/files/src"
},
"continuous": false,
"development": true,
"edition": "ADVANCED",
"filters": [],
"id": "123",
"library": [],
"name": "test_pipeline",
"notification": [],
"photon": false,
"serverless": false,
"storage": "dbfs:/123456",
"target": "test_dev",
"timeouts": null,
"url": "https://test.com"
},
"sensitive_attributes": []
}
]
}
]
}`)
err = os.WriteFile(filepath.Join(cacheDir, TerraformStateFileName), data, os.ModePerm)
assert.NoError(t, err)
state, err := ParseResourcesState(ctx, b)
assert.NoError(t, err)
expected := &resourcesState{
Version: 4,
Resources: []stateResource{
{
Mode: "managed",
Type: "databricks_pipeline",
Name: "test_pipeline",
Instances: []stateResourceInstance{
{Attributes: stateInstanceAttributes{ID: "123"}},
},
},
},
}
assert.Equal(t, expected, state)
}

2
bundle/internal/tf/codegen/schema/version.go
View File

@ -1,3 +1,3 @@
package schema
const ProviderVersion = "1.40.0"
const ProviderVersion = "1.46.0"

46
bundle/internal/tf/schema/data_source_catalog.go Normal file
View File

@ -0,0 +1,46 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag struct {
InheritedFromName string `json:"inherited_from_name,omitempty"`
InheritedFromType string `json:"inherited_from_type,omitempty"`
Value string `json:"value"`
}
type DataSourceCatalogCatalogInfoProvisioningInfo struct {
State string `json:"state,omitempty"`
}
type DataSourceCatalogCatalogInfo struct {
BrowseOnly bool `json:"browse_only,omitempty"`
CatalogType string `json:"catalog_type,omitempty"`
Comment string `json:"comment,omitempty"`
ConnectionName string `json:"connection_name,omitempty"`
CreatedAt int `json:"created_at,omitempty"`
CreatedBy string `json:"created_by,omitempty"`
EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"`
FullName string `json:"full_name,omitempty"`
IsolationMode string `json:"isolation_mode,omitempty"`
MetastoreId string `json:"metastore_id,omitempty"`
Name string `json:"name,omitempty"`
Options map[string]string `json:"options,omitempty"`
Owner string `json:"owner,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
ProviderName string `json:"provider_name,omitempty"`
SecurableKind string `json:"securable_kind,omitempty"`
SecurableType string `json:"securable_type,omitempty"`
ShareName string `json:"share_name,omitempty"`
StorageLocation string `json:"storage_location,omitempty"`
StorageRoot string `json:"storage_root,omitempty"`
UpdatedAt int `json:"updated_at,omitempty"`
UpdatedBy string `json:"updated_by,omitempty"`
EffectivePredictiveOptimizationFlag *DataSourceCatalogCatalogInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"`
ProvisioningInfo *DataSourceCatalogCatalogInfoProvisioningInfo `json:"provisioning_info,omitempty"`
}
type DataSourceCatalog struct {
Id string `json:"id,omitempty"`
Name string `json:"name"`
CatalogInfo *DataSourceCatalogCatalogInfo `json:"catalog_info,omitempty"`
}

67
bundle/internal/tf/schema/data_source_job.go
View File

@ -55,9 +55,9 @@ type DataSourceJobJobSettingsSettingsGitSource struct {
}
type DataSourceJobJobSettingsSettingsHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type DataSourceJobJobSettingsSettingsHealth struct {
@ -222,7 +222,7 @@ type DataSourceJobJobSettingsSettingsJobClusterNewCluster struct {
}
type DataSourceJobJobSettingsSettingsJobCluster struct {
JobClusterKey string `json:"job_cluster_key,omitempty"`
JobClusterKey string `json:"job_cluster_key"`
NewCluster *DataSourceJobJobSettingsSettingsJobClusterNewCluster `json:"new_cluster,omitempty"`
}
@ -243,12 +243,13 @@ type DataSourceJobJobSettingsSettingsLibraryPypi struct {
}
type DataSourceJobJobSettingsSettingsLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsLibraryPypi `json:"pypi,omitempty"`
}
type DataSourceJobJobSettingsSettingsNewClusterAutoscale struct {
@ -532,9 +533,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskEmailNotifications struc
}
type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskHealth struct {
@ -558,12 +559,13 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi struct {
}
type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"`
}
type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskNewClusterAutoscale struct {
@ -803,7 +805,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskQuery struct {
type DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTask struct {
Parameters map[string]string `json:"parameters,omitempty"`
WarehouseId string `json:"warehouse_id,omitempty"`
WarehouseId string `json:"warehouse_id"`
Alert *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"`
Dashboard *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"`
File *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"`
@ -842,7 +844,7 @@ type DataSourceJobJobSettingsSettingsTaskForEachTaskTask struct {
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key,omitempty"`
TaskKey string `json:"task_key"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *DataSourceJobJobSettingsSettingsTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"`
@ -870,9 +872,9 @@ type DataSourceJobJobSettingsSettingsTaskForEachTask struct {
}
type DataSourceJobJobSettingsSettingsTaskHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type DataSourceJobJobSettingsSettingsTaskHealth struct {
@ -896,12 +898,13 @@ type DataSourceJobJobSettingsSettingsTaskLibraryPypi struct {
}
type DataSourceJobJobSettingsSettingsTaskLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsTaskLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsTaskLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *DataSourceJobJobSettingsSettingsTaskLibraryCran `json:"cran,omitempty"`
Maven *DataSourceJobJobSettingsSettingsTaskLibraryMaven `json:"maven,omitempty"`
Pypi *DataSourceJobJobSettingsSettingsTaskLibraryPypi `json:"pypi,omitempty"`
}
type DataSourceJobJobSettingsSettingsTaskNewClusterAutoscale struct {
@ -1141,7 +1144,7 @@ type DataSourceJobJobSettingsSettingsTaskSqlTaskQuery struct {
type DataSourceJobJobSettingsSettingsTaskSqlTask struct {
Parameters map[string]string `json:"parameters,omitempty"`
WarehouseId string `json:"warehouse_id,omitempty"`
WarehouseId string `json:"warehouse_id"`
Alert *DataSourceJobJobSettingsSettingsTaskSqlTaskAlert `json:"alert,omitempty"`
Dashboard *DataSourceJobJobSettingsSettingsTaskSqlTaskDashboard `json:"dashboard,omitempty"`
File *DataSourceJobJobSettingsSettingsTaskSqlTaskFile `json:"file,omitempty"`
@ -1180,7 +1183,7 @@ type DataSourceJobJobSettingsSettingsTask struct {
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key,omitempty"`
TaskKey string `json:"task_key"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *DataSourceJobJobSettingsSettingsTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *DataSourceJobJobSettingsSettingsTaskDbtTask `json:"dbt_task,omitempty"`

19
bundle/internal/tf/schema/data_source_mlflow_experiment.go Normal file
View File

@ -0,0 +1,19 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type DataSourceMlflowExperimentTags struct {
Key string `json:"key,omitempty"`
Value string `json:"value,omitempty"`
}
type DataSourceMlflowExperiment struct {
ArtifactLocation string `json:"artifact_location,omitempty"`
CreationTime int `json:"creation_time,omitempty"`
ExperimentId string `json:"experiment_id,omitempty"`
Id string `json:"id,omitempty"`
LastUpdateTime int `json:"last_update_time,omitempty"`
LifecycleStage string `json:"lifecycle_stage,omitempty"`
Name string `json:"name,omitempty"`
Tags []DataSourceMlflowExperimentTags `json:"tags,omitempty"`
}

127
bundle/internal/tf/schema/data_source_table.go Normal file
View File

@ -0,0 +1,127 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type DataSourceTableTableInfoColumnsMask struct {
FunctionName string `json:"function_name,omitempty"`
UsingColumnNames []string `json:"using_column_names,omitempty"`
}
type DataSourceTableTableInfoColumns struct {
Comment string `json:"comment,omitempty"`
Name string `json:"name,omitempty"`
Nullable bool `json:"nullable,omitempty"`
PartitionIndex int `json:"partition_index,omitempty"`
Position int `json:"position,omitempty"`
TypeIntervalType string `json:"type_interval_type,omitempty"`
TypeJson string `json:"type_json,omitempty"`
TypeName string `json:"type_name,omitempty"`
TypePrecision int `json:"type_precision,omitempty"`
TypeScale int `json:"type_scale,omitempty"`
TypeText string `json:"type_text,omitempty"`
Mask *DataSourceTableTableInfoColumnsMask `json:"mask,omitempty"`
}
type DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs struct {
DeltaRuntimeProperties map[string]string `json:"delta_runtime_properties"`
}
type DataSourceTableTableInfoEffectivePredictiveOptimizationFlag struct {
InheritedFromName string `json:"inherited_from_name,omitempty"`
InheritedFromType string `json:"inherited_from_type,omitempty"`
Value string `json:"value"`
}
type DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails struct {
Algorithm string `json:"algorithm,omitempty"`
AwsKmsKeyArn string `json:"aws_kms_key_arn,omitempty"`
}
type DataSourceTableTableInfoEncryptionDetails struct {
SseEncryptionDetails *DataSourceTableTableInfoEncryptionDetailsSseEncryptionDetails `json:"sse_encryption_details,omitempty"`
}
type DataSourceTableTableInfoRowFilter struct {
FunctionName string `json:"function_name"`
InputColumnNames []string `json:"input_column_names"`
}
type DataSourceTableTableInfoTableConstraintsForeignKeyConstraint struct {
ChildColumns []string `json:"child_columns"`
Name string `json:"name"`
ParentColumns []string `json:"parent_columns"`
ParentTable string `json:"parent_table"`
}
type DataSourceTableTableInfoTableConstraintsNamedTableConstraint struct {
Name string `json:"name"`
}
type DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint struct {
ChildColumns []string `json:"child_columns"`
Name string `json:"name"`
}
type DataSourceTableTableInfoTableConstraints struct {
ForeignKeyConstraint *DataSourceTableTableInfoTableConstraintsForeignKeyConstraint `json:"foreign_key_constraint,omitempty"`
NamedTableConstraint *DataSourceTableTableInfoTableConstraintsNamedTableConstraint `json:"named_table_constraint,omitempty"`
PrimaryKeyConstraint *DataSourceTableTableInfoTableConstraintsPrimaryKeyConstraint `json:"primary_key_constraint,omitempty"`
}
type DataSourceTableTableInfoViewDependenciesDependenciesFunction struct {
FunctionFullName string `json:"function_full_name"`
}
type DataSourceTableTableInfoViewDependenciesDependenciesTable struct {
TableFullName string `json:"table_full_name"`
}
type DataSourceTableTableInfoViewDependenciesDependencies struct {
Function *DataSourceTableTableInfoViewDependenciesDependenciesFunction `json:"function,omitempty"`
Table *DataSourceTableTableInfoViewDependenciesDependenciesTable `json:"table,omitempty"`
}
type DataSourceTableTableInfoViewDependencies struct {
Dependencies []DataSourceTableTableInfoViewDependenciesDependencies `json:"dependencies,omitempty"`
}
type DataSourceTableTableInfo struct {
AccessPoint string `json:"access_point,omitempty"`
BrowseOnly bool `json:"browse_only,omitempty"`
CatalogName string `json:"catalog_name,omitempty"`
Comment string `json:"comment,omitempty"`
CreatedAt int `json:"created_at,omitempty"`
CreatedBy string `json:"created_by,omitempty"`
DataAccessConfigurationId string `json:"data_access_configuration_id,omitempty"`
DataSourceFormat string `json:"data_source_format,omitempty"`
DeletedAt int `json:"deleted_at,omitempty"`
EnablePredictiveOptimization string `json:"enable_predictive_optimization,omitempty"`
FullName string `json:"full_name,omitempty"`
MetastoreId string `json:"metastore_id,omitempty"`
Name string `json:"name,omitempty"`
Owner string `json:"owner,omitempty"`
PipelineId string `json:"pipeline_id,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
SchemaName string `json:"schema_name,omitempty"`
SqlPath string `json:"sql_path,omitempty"`
StorageCredentialName string `json:"storage_credential_name,omitempty"`
StorageLocation string `json:"storage_location,omitempty"`
TableId string `json:"table_id,omitempty"`
TableType string `json:"table_type,omitempty"`
UpdatedAt int `json:"updated_at,omitempty"`
UpdatedBy string `json:"updated_by,omitempty"`
ViewDefinition string `json:"view_definition,omitempty"`
Columns []DataSourceTableTableInfoColumns `json:"columns,omitempty"`
DeltaRuntimePropertiesKvpairs *DataSourceTableTableInfoDeltaRuntimePropertiesKvpairs `json:"delta_runtime_properties_kvpairs,omitempty"`
EffectivePredictiveOptimizationFlag *DataSourceTableTableInfoEffectivePredictiveOptimizationFlag `json:"effective_predictive_optimization_flag,omitempty"`
EncryptionDetails *DataSourceTableTableInfoEncryptionDetails `json:"encryption_details,omitempty"`
RowFilter *DataSourceTableTableInfoRowFilter `json:"row_filter,omitempty"`
TableConstraints []DataSourceTableTableInfoTableConstraints `json:"table_constraints,omitempty"`
ViewDependencies *DataSourceTableTableInfoViewDependencies `json:"view_dependencies,omitempty"`
}
type DataSourceTable struct {
Id string `json:"id,omitempty"`
Name string `json:"name"`
TableInfo *DataSourceTableTableInfo `json:"table_info,omitempty"`
}

6
bundle/internal/tf/schema/data_sources.go
View File

@ -7,6 +7,7 @@ type DataSources struct {
AwsBucketPolicy map[string]any `json:"databricks_aws_bucket_policy,omitempty"`
AwsCrossaccountPolicy map[string]any `json:"databricks_aws_crossaccount_policy,omitempty"`
AwsUnityCatalogPolicy map[string]any `json:"databricks_aws_unity_catalog_policy,omitempty"`
Catalog map[string]any `json:"databricks_catalog,omitempty"`
Catalogs map[string]any `json:"databricks_catalogs,omitempty"`
Cluster map[string]any `json:"databricks_cluster,omitempty"`
ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"`
@ -26,6 +27,7 @@ type DataSources struct {
Jobs map[string]any `json:"databricks_jobs,omitempty"`
Metastore map[string]any `json:"databricks_metastore,omitempty"`
Metastores map[string]any `json:"databricks_metastores,omitempty"`
MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"`
MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"`
MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"`
MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"`
@ -43,6 +45,7 @@ type DataSources struct {
SqlWarehouses map[string]any `json:"databricks_sql_warehouses,omitempty"`
StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"`
StorageCredentials map[string]any `json:"databricks_storage_credentials,omitempty"`
Table map[string]any `json:"databricks_table,omitempty"`
Tables map[string]any `json:"databricks_tables,omitempty"`
User map[string]any `json:"databricks_user,omitempty"`
Views map[string]any `json:"databricks_views,omitempty"`
@ -56,6 +59,7 @@ func NewDataSources() *DataSources {
AwsBucketPolicy: make(map[string]any),
AwsCrossaccountPolicy: make(map[string]any),
AwsUnityCatalogPolicy: make(map[string]any),
Catalog: make(map[string]any),
Catalogs: make(map[string]any),
Cluster: make(map[string]any),
ClusterPolicy: make(map[string]any),
@ -75,6 +79,7 @@ func NewDataSources() *DataSources {
Jobs: make(map[string]any),
Metastore: make(map[string]any),
Metastores: make(map[string]any),
MlflowExperiment: make(map[string]any),
MlflowModel: make(map[string]any),
MwsCredentials: make(map[string]any),
MwsWorkspaces: make(map[string]any),
@ -92,6 +97,7 @@ func NewDataSources() *DataSources {
SqlWarehouses: make(map[string]any),
StorageCredential: make(map[string]any),
StorageCredentials: make(map[string]any),
Table: make(map[string]any),
Tables: make(map[string]any),
User: make(map[string]any),
Views: make(map[string]any),

39
bundle/internal/tf/schema/resource_automatic_cluster_update_workspace_setting.go Normal file
View File

@ -0,0 +1,39 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails struct {
ForcedForComplianceMode bool `json:"forced_for_compliance_mode,omitempty"`
UnavailableForDisabledEntitlement bool `json:"unavailable_for_disabled_entitlement,omitempty"`
UnavailableForNonEnterpriseTier bool `json:"unavailable_for_non_enterprise_tier,omitempty"`
}
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime struct {
Hours int `json:"hours,omitempty"`
Minutes int `json:"minutes,omitempty"`
}
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule struct {
DayOfWeek string `json:"day_of_week,omitempty"`
Frequency string `json:"frequency,omitempty"`
WindowStartTime *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedScheduleWindowStartTime `json:"window_start_time,omitempty"`
}
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow struct {
WeekDayBasedSchedule *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindowWeekDayBasedSchedule `json:"week_day_based_schedule,omitempty"`
}
type ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace struct {
CanToggle bool `json:"can_toggle,omitempty"`
Enabled bool `json:"enabled,omitempty"`
RestartEvenIfNoUpdatesAvailable bool `json:"restart_even_if_no_updates_available,omitempty"`
EnablementDetails *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceEnablementDetails `json:"enablement_details,omitempty"`
MaintenanceWindow *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspaceMaintenanceWindow `json:"maintenance_window,omitempty"`
}
type ResourceAutomaticClusterUpdateWorkspaceSetting struct {
Etag string `json:"etag,omitempty"`
Id string `json:"id,omitempty"`
SettingName string `json:"setting_name,omitempty"`
AutomaticClusterUpdateWorkspace *ResourceAutomaticClusterUpdateWorkspaceSettingAutomaticClusterUpdateWorkspace `json:"automatic_cluster_update_workspace,omitempty"`
}

19
bundle/internal/tf/schema/resource_cluster.go
View File

@ -32,10 +32,6 @@ type ResourceClusterAzureAttributes struct {
LogAnalyticsInfo *ResourceClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
}
type ResourceClusterCloneFrom struct {
SourceClusterId string `json:"source_cluster_id"`
}
type ResourceClusterClusterLogConfDbfs struct {
Destination string `json:"destination"`
}
@ -146,12 +142,13 @@ type ResourceClusterLibraryPypi struct {
}
type ResourceClusterLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceClusterLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceClusterLibraryPypi `json:"pypi,omitempty"`
}
type ResourceClusterWorkloadTypeClients struct {
@ -168,7 +165,6 @@ type ResourceCluster struct {
AutoterminationMinutes int `json:"autotermination_minutes,omitempty"`
ClusterId string `json:"cluster_id,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
ClusterSource string `json:"cluster_source,omitempty"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
DataSecurityMode string `json:"data_security_mode,omitempty"`
DefaultTags map[string]string `json:"default_tags,omitempty"`
@ -194,7 +190,6 @@ type ResourceCluster struct {
Autoscale *ResourceClusterAutoscale `json:"autoscale,omitempty"`
AwsAttributes *ResourceClusterAwsAttributes `json:"aws_attributes,omitempty"`
AzureAttributes *ResourceClusterAzureAttributes `json:"azure_attributes,omitempty"`
CloneFrom *ResourceClusterCloneFrom `json:"clone_from,omitempty"`
ClusterLogConf *ResourceClusterClusterLogConf `json:"cluster_log_conf,omitempty"`
ClusterMountInfo []ResourceClusterClusterMountInfo `json:"cluster_mount_info,omitempty"`
DockerImage *ResourceClusterDockerImage `json:"docker_image,omitempty"`

13
bundle/internal/tf/schema/resource_cluster_policy.go
View File

@ -19,12 +19,13 @@ type ResourceClusterPolicyLibrariesPypi struct {
}
type ResourceClusterPolicyLibraries struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceClusterPolicyLibrariesCran `json:"cran,omitempty"`
Maven *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"`
Pypi *ResourceClusterPolicyLibrariesPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceClusterPolicyLibrariesCran `json:"cran,omitempty"`
Maven *ResourceClusterPolicyLibrariesMaven `json:"maven,omitempty"`
Pypi *ResourceClusterPolicyLibrariesPypi `json:"pypi,omitempty"`
}
type ResourceClusterPolicy struct {

15
bundle/internal/tf/schema/resource_compliance_security_profile_workspace_setting.go Normal file
View File

@ -0,0 +1,15 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace struct {
ComplianceStandards []string `json:"compliance_standards,omitempty"`
IsEnabled bool `json:"is_enabled,omitempty"`
}
type ResourceComplianceSecurityProfileWorkspaceSetting struct {
Etag string `json:"etag,omitempty"`
Id string `json:"id,omitempty"`
SettingName string `json:"setting_name,omitempty"`
ComplianceSecurityProfileWorkspace *ResourceComplianceSecurityProfileWorkspaceSettingComplianceSecurityProfileWorkspace `json:"compliance_security_profile_workspace,omitempty"`
}

14
bundle/internal/tf/schema/resource_enhanced_security_monitoring_workspace_setting.go Normal file
View File

@ -0,0 +1,14 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace struct {
IsEnabled bool `json:"is_enabled,omitempty"`
}
type ResourceEnhancedSecurityMonitoringWorkspaceSetting struct {
Etag string `json:"etag,omitempty"`
Id string `json:"id,omitempty"`
SettingName string `json:"setting_name,omitempty"`
EnhancedSecurityMonitoringWorkspace *ResourceEnhancedSecurityMonitoringWorkspaceSettingEnhancedSecurityMonitoringWorkspace `json:"enhanced_security_monitoring_workspace,omitempty"`
}

402
bundle/internal/tf/schema/resource_job.go
View File

@ -39,6 +39,10 @@ type ResourceJobEnvironment struct {
Spec *ResourceJobEnvironmentSpec `json:"spec,omitempty"`
}
type ResourceJobGitSourceGitSnapshot struct {
UsedCommit string `json:"used_commit,omitempty"`
}
type ResourceJobGitSourceJobSource struct {
DirtyState string `json:"dirty_state,omitempty"`
ImportFromGitBranch string `json:"import_from_git_branch"`
@ -46,18 +50,19 @@ type ResourceJobGitSourceJobSource struct {
}
type ResourceJobGitSource struct {
Branch string `json:"branch,omitempty"`
Commit string `json:"commit,omitempty"`
Provider string `json:"provider,omitempty"`
Tag string `json:"tag,omitempty"`
Url string `json:"url"`
JobSource *ResourceJobGitSourceJobSource `json:"job_source,omitempty"`
Branch string `json:"branch,omitempty"`
Commit string `json:"commit,omitempty"`
Provider string `json:"provider,omitempty"`
Tag string `json:"tag,omitempty"`
Url string `json:"url"`
GitSnapshot *ResourceJobGitSourceGitSnapshot `json:"git_snapshot,omitempty"`
JobSource *ResourceJobGitSourceJobSource `json:"job_source,omitempty"`
}
type ResourceJobHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type ResourceJobHealth struct {
@ -72,7 +77,9 @@ type ResourceJobJobClusterNewClusterAutoscale struct {
type ResourceJobJobClusterNewClusterAwsAttributes struct {
Availability string `json:"availability,omitempty"`
EbsVolumeCount int `json:"ebs_volume_count,omitempty"`
EbsVolumeIops int `json:"ebs_volume_iops,omitempty"`
EbsVolumeSize int `json:"ebs_volume_size,omitempty"`
EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"`
EbsVolumeType string `json:"ebs_volume_type,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
@ -80,10 +87,16 @@ type ResourceJobJobClusterNewClusterAwsAttributes struct {
ZoneId string `json:"zone_id,omitempty"`
}
type ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo struct {
LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"`
LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
}
type ResourceJobJobClusterNewClusterAzureAttributes struct {
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
LogAnalyticsInfo *ResourceJobJobClusterNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
}
type ResourceJobJobClusterNewClusterClusterLogConfDbfs struct {
@ -179,6 +192,32 @@ type ResourceJobJobClusterNewClusterInitScripts struct {
Workspace *ResourceJobJobClusterNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
}
type ResourceJobJobClusterNewClusterLibraryCran struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobJobClusterNewClusterLibraryMaven struct {
Coordinates string `json:"coordinates"`
Exclusions []string `json:"exclusions,omitempty"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobJobClusterNewClusterLibraryPypi struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobJobClusterNewClusterLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobJobClusterNewClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobJobClusterNewClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobJobClusterNewClusterLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobJobClusterNewClusterWorkloadTypeClients struct {
Jobs bool `json:"jobs,omitempty"`
Notebooks bool `json:"notebooks,omitempty"`
@ -190,7 +229,6 @@ type ResourceJobJobClusterNewClusterWorkloadType struct {
type ResourceJobJobClusterNewCluster struct {
ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"`
AutoterminationMinutes int `json:"autotermination_minutes,omitempty"`
ClusterId string `json:"cluster_id,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
@ -218,11 +256,12 @@ type ResourceJobJobClusterNewCluster struct {
DockerImage *ResourceJobJobClusterNewClusterDockerImage `json:"docker_image,omitempty"`
GcpAttributes *ResourceJobJobClusterNewClusterGcpAttributes `json:"gcp_attributes,omitempty"`
InitScripts []ResourceJobJobClusterNewClusterInitScripts `json:"init_scripts,omitempty"`
Library []ResourceJobJobClusterNewClusterLibrary `json:"library,omitempty"`
WorkloadType *ResourceJobJobClusterNewClusterWorkloadType `json:"workload_type,omitempty"`
}
type ResourceJobJobCluster struct {
JobClusterKey string `json:"job_cluster_key,omitempty"`
JobClusterKey string `json:"job_cluster_key"`
NewCluster *ResourceJobJobClusterNewCluster `json:"new_cluster,omitempty"`
}
@ -243,12 +282,13 @@ type ResourceJobLibraryPypi struct {
}
type ResourceJobLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobNewClusterAutoscale struct {
@ -259,7 +299,9 @@ type ResourceJobNewClusterAutoscale struct {
type ResourceJobNewClusterAwsAttributes struct {
Availability string `json:"availability,omitempty"`
EbsVolumeCount int `json:"ebs_volume_count,omitempty"`
EbsVolumeIops int `json:"ebs_volume_iops,omitempty"`
EbsVolumeSize int `json:"ebs_volume_size,omitempty"`
EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"`
EbsVolumeType string `json:"ebs_volume_type,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
@ -267,10 +309,16 @@ type ResourceJobNewClusterAwsAttributes struct {
ZoneId string `json:"zone_id,omitempty"`
}
type ResourceJobNewClusterAzureAttributesLogAnalyticsInfo struct {
LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"`
LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
}
type ResourceJobNewClusterAzureAttributes struct {
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
LogAnalyticsInfo *ResourceJobNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
}
type ResourceJobNewClusterClusterLogConfDbfs struct {
@ -366,6 +414,32 @@ type ResourceJobNewClusterInitScripts struct {
Workspace *ResourceJobNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
}
type ResourceJobNewClusterLibraryCran struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobNewClusterLibraryMaven struct {
Coordinates string `json:"coordinates"`
Exclusions []string `json:"exclusions,omitempty"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobNewClusterLibraryPypi struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobNewClusterLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobNewClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobNewClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobNewClusterLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobNewClusterWorkloadTypeClients struct {
Jobs bool `json:"jobs,omitempty"`
Notebooks bool `json:"notebooks,omitempty"`
@ -377,7 +451,6 @@ type ResourceJobNewClusterWorkloadType struct {
type ResourceJobNewCluster struct {
ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"`
AutoterminationMinutes int `json:"autotermination_minutes,omitempty"`
ClusterId string `json:"cluster_id,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
@ -405,6 +478,7 @@ type ResourceJobNewCluster struct {
DockerImage *ResourceJobNewClusterDockerImage `json:"docker_image,omitempty"`
GcpAttributes *ResourceJobNewClusterGcpAttributes `json:"gcp_attributes,omitempty"`
InitScripts []ResourceJobNewClusterInitScripts `json:"init_scripts,omitempty"`
Library []ResourceJobNewClusterLibrary `json:"library,omitempty"`
WorkloadType *ResourceJobNewClusterWorkloadType `json:"workload_type,omitempty"`
}
@ -532,9 +606,9 @@ type ResourceJobTaskForEachTaskTaskEmailNotifications struct {
}
type ResourceJobTaskForEachTaskTaskHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type ResourceJobTaskForEachTaskTaskHealth struct {
@ -558,12 +632,13 @@ type ResourceJobTaskForEachTaskTaskLibraryPypi struct {
}
type ResourceJobTaskForEachTaskTaskLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskForEachTaskTaskLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskForEachTaskTaskLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskForEachTaskTaskLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct {
@ -574,7 +649,9 @@ type ResourceJobTaskForEachTaskTaskNewClusterAutoscale struct {
type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct {
Availability string `json:"availability,omitempty"`
EbsVolumeCount int `json:"ebs_volume_count,omitempty"`
EbsVolumeIops int `json:"ebs_volume_iops,omitempty"`
EbsVolumeSize int `json:"ebs_volume_size,omitempty"`
EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"`
EbsVolumeType string `json:"ebs_volume_type,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
@ -582,10 +659,16 @@ type ResourceJobTaskForEachTaskTaskNewClusterAwsAttributes struct {
ZoneId string `json:"zone_id,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo struct {
LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"`
LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterAzureAttributes struct {
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
LogAnalyticsInfo *ResourceJobTaskForEachTaskTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterClusterLogConfDbfs struct {
@ -681,6 +764,32 @@ type ResourceJobTaskForEachTaskTaskNewClusterInitScripts struct {
Workspace *ResourceJobTaskForEachTaskTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterLibraryCran struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven struct {
Coordinates string `json:"coordinates"`
Exclusions []string `json:"exclusions,omitempty"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskForEachTaskTaskNewClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskForEachTaskTaskNewClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskForEachTaskTaskNewClusterLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobTaskForEachTaskTaskNewClusterWorkloadTypeClients struct {
Jobs bool `json:"jobs,omitempty"`
Notebooks bool `json:"notebooks,omitempty"`
@ -692,7 +801,6 @@ type ResourceJobTaskForEachTaskTaskNewClusterWorkloadType struct {
type ResourceJobTaskForEachTaskTaskNewCluster struct {
ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"`
AutoterminationMinutes int `json:"autotermination_minutes,omitempty"`
ClusterId string `json:"cluster_id,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
@ -704,7 +812,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct {
IdempotencyToken string `json:"idempotency_token,omitempty"`
InstancePoolId string `json:"instance_pool_id,omitempty"`
NodeTypeId string `json:"node_type_id,omitempty"`
NumWorkers int `json:"num_workers"`
NumWorkers int `json:"num_workers,omitempty"`
PolicyId string `json:"policy_id,omitempty"`
RuntimeEngine string `json:"runtime_engine,omitempty"`
SingleUserName string `json:"single_user_name,omitempty"`
@ -720,6 +828,7 @@ type ResourceJobTaskForEachTaskTaskNewCluster struct {
DockerImage *ResourceJobTaskForEachTaskTaskNewClusterDockerImage `json:"docker_image,omitempty"`
GcpAttributes *ResourceJobTaskForEachTaskTaskNewClusterGcpAttributes `json:"gcp_attributes,omitempty"`
InitScripts []ResourceJobTaskForEachTaskTaskNewClusterInitScripts `json:"init_scripts,omitempty"`
Library []ResourceJobTaskForEachTaskTaskNewClusterLibrary `json:"library,omitempty"`
WorkloadType *ResourceJobTaskForEachTaskTaskNewClusterWorkloadType `json:"workload_type,omitempty"`
}
@ -748,9 +857,21 @@ type ResourceJobTaskForEachTaskTaskPythonWheelTask struct {
Parameters []string `json:"parameters,omitempty"`
}
type ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams struct {
FullRefresh bool `json:"full_refresh,omitempty"`
}
type ResourceJobTaskForEachTaskTaskRunJobTask struct {
JobId int `json:"job_id"`
JobParameters map[string]string `json:"job_parameters,omitempty"`
DbtCommands []string `json:"dbt_commands,omitempty"`
JarParams []string `json:"jar_params,omitempty"`
JobId int `json:"job_id"`
JobParameters map[string]string `json:"job_parameters,omitempty"`
NotebookParams map[string]string `json:"notebook_params,omitempty"`
PythonNamedParams map[string]string `json:"python_named_params,omitempty"`
PythonParams []string `json:"python_params,omitempty"`
SparkSubmitParams []string `json:"spark_submit_params,omitempty"`
SqlParams map[string]string `json:"sql_params,omitempty"`
PipelineParams *ResourceJobTaskForEachTaskTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"`
}
type ResourceJobTaskForEachTaskTaskSparkJarTask struct {
@ -803,7 +924,7 @@ type ResourceJobTaskForEachTaskTaskSqlTaskQuery struct {
type ResourceJobTaskForEachTaskTaskSqlTask struct {
Parameters map[string]string `json:"parameters,omitempty"`
WarehouseId string `json:"warehouse_id,omitempty"`
WarehouseId string `json:"warehouse_id"`
Alert *ResourceJobTaskForEachTaskTaskSqlTaskAlert `json:"alert,omitempty"`
Dashboard *ResourceJobTaskForEachTaskTaskSqlTaskDashboard `json:"dashboard,omitempty"`
File *ResourceJobTaskForEachTaskTaskSqlTaskFile `json:"file,omitempty"`
@ -834,33 +955,34 @@ type ResourceJobTaskForEachTaskTaskWebhookNotifications struct {
}
type ResourceJobTaskForEachTaskTask struct {
Description string `json:"description,omitempty"`
EnvironmentKey string `json:"environment_key,omitempty"`
ExistingClusterId string `json:"existing_cluster_id,omitempty"`
JobClusterKey string `json:"job_cluster_key,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"`
DependsOn []ResourceJobTaskForEachTaskTaskDependsOn `json:"depends_on,omitempty"`
EmailNotifications *ResourceJobTaskForEachTaskTaskEmailNotifications `json:"email_notifications,omitempty"`
Health *ResourceJobTaskForEachTaskTaskHealth `json:"health,omitempty"`
Library []ResourceJobTaskForEachTaskTaskLibrary `json:"library,omitempty"`
NewCluster *ResourceJobTaskForEachTaskTaskNewCluster `json:"new_cluster,omitempty"`
NotebookTask *ResourceJobTaskForEachTaskTaskNotebookTask `json:"notebook_task,omitempty"`
NotificationSettings *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"`
PipelineTask *ResourceJobTaskForEachTaskTaskPipelineTask `json:"pipeline_task,omitempty"`
PythonWheelTask *ResourceJobTaskForEachTaskTaskPythonWheelTask `json:"python_wheel_task,omitempty"`
RunJobTask *ResourceJobTaskForEachTaskTaskRunJobTask `json:"run_job_task,omitempty"`
SparkJarTask *ResourceJobTaskForEachTaskTaskSparkJarTask `json:"spark_jar_task,omitempty"`
SparkPythonTask *ResourceJobTaskForEachTaskTaskSparkPythonTask `json:"spark_python_task,omitempty"`
SparkSubmitTask *ResourceJobTaskForEachTaskTaskSparkSubmitTask `json:"spark_submit_task,omitempty"`
SqlTask *ResourceJobTaskForEachTaskTaskSqlTask `json:"sql_task,omitempty"`
WebhookNotifications *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
Description string `json:"description,omitempty"`
DisableAutoOptimization bool `json:"disable_auto_optimization,omitempty"`
EnvironmentKey string `json:"environment_key,omitempty"`
ExistingClusterId string `json:"existing_cluster_id,omitempty"`
JobClusterKey string `json:"job_cluster_key,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *ResourceJobTaskForEachTaskTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *ResourceJobTaskForEachTaskTaskDbtTask `json:"dbt_task,omitempty"`
DependsOn []ResourceJobTaskForEachTaskTaskDependsOn `json:"depends_on,omitempty"`
EmailNotifications *ResourceJobTaskForEachTaskTaskEmailNotifications `json:"email_notifications,omitempty"`
Health *ResourceJobTaskForEachTaskTaskHealth `json:"health,omitempty"`
Library []ResourceJobTaskForEachTaskTaskLibrary `json:"library,omitempty"`
NewCluster *ResourceJobTaskForEachTaskTaskNewCluster `json:"new_cluster,omitempty"`
NotebookTask *ResourceJobTaskForEachTaskTaskNotebookTask `json:"notebook_task,omitempty"`
NotificationSettings *ResourceJobTaskForEachTaskTaskNotificationSettings `json:"notification_settings,omitempty"`
PipelineTask *ResourceJobTaskForEachTaskTaskPipelineTask `json:"pipeline_task,omitempty"`
PythonWheelTask *ResourceJobTaskForEachTaskTaskPythonWheelTask `json:"python_wheel_task,omitempty"`
RunJobTask *ResourceJobTaskForEachTaskTaskRunJobTask `json:"run_job_task,omitempty"`
SparkJarTask *ResourceJobTaskForEachTaskTaskSparkJarTask `json:"spark_jar_task,omitempty"`
SparkPythonTask *ResourceJobTaskForEachTaskTaskSparkPythonTask `json:"spark_python_task,omitempty"`
SparkSubmitTask *ResourceJobTaskForEachTaskTaskSparkSubmitTask `json:"spark_submit_task,omitempty"`
SqlTask *ResourceJobTaskForEachTaskTaskSqlTask `json:"sql_task,omitempty"`
WebhookNotifications *ResourceJobTaskForEachTaskTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
}
type ResourceJobTaskForEachTask struct {
@ -870,9 +992,9 @@ type ResourceJobTaskForEachTask struct {
}
type ResourceJobTaskHealthRules struct {
Metric string `json:"metric,omitempty"`
Op string `json:"op,omitempty"`
Value int `json:"value,omitempty"`
Metric string `json:"metric"`
Op string `json:"op"`
Value int `json:"value"`
}
type ResourceJobTaskHealth struct {
@ -896,12 +1018,13 @@ type ResourceJobTaskLibraryPypi struct {
}
type ResourceJobTaskLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskLibraryPypi `json:"pypi,omitempty"`
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobTaskNewClusterAutoscale struct {
@ -912,7 +1035,9 @@ type ResourceJobTaskNewClusterAutoscale struct {
type ResourceJobTaskNewClusterAwsAttributes struct {
Availability string `json:"availability,omitempty"`
EbsVolumeCount int `json:"ebs_volume_count,omitempty"`
EbsVolumeIops int `json:"ebs_volume_iops,omitempty"`
EbsVolumeSize int `json:"ebs_volume_size,omitempty"`
EbsVolumeThroughput int `json:"ebs_volume_throughput,omitempty"`
EbsVolumeType string `json:"ebs_volume_type,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
InstanceProfileArn string `json:"instance_profile_arn,omitempty"`
@ -920,10 +1045,16 @@ type ResourceJobTaskNewClusterAwsAttributes struct {
ZoneId string `json:"zone_id,omitempty"`
}
type ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo struct {
LogAnalyticsPrimaryKey string `json:"log_analytics_primary_key,omitempty"`
LogAnalyticsWorkspaceId string `json:"log_analytics_workspace_id,omitempty"`
}
type ResourceJobTaskNewClusterAzureAttributes struct {
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
Availability string `json:"availability,omitempty"`
FirstOnDemand int `json:"first_on_demand,omitempty"`
SpotBidMaxPrice int `json:"spot_bid_max_price,omitempty"`
LogAnalyticsInfo *ResourceJobTaskNewClusterAzureAttributesLogAnalyticsInfo `json:"log_analytics_info,omitempty"`
}
type ResourceJobTaskNewClusterClusterLogConfDbfs struct {
@ -1019,6 +1150,32 @@ type ResourceJobTaskNewClusterInitScripts struct {
Workspace *ResourceJobTaskNewClusterInitScriptsWorkspace `json:"workspace,omitempty"`
}
type ResourceJobTaskNewClusterLibraryCran struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskNewClusterLibraryMaven struct {
Coordinates string `json:"coordinates"`
Exclusions []string `json:"exclusions,omitempty"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskNewClusterLibraryPypi struct {
Package string `json:"package"`
Repo string `json:"repo,omitempty"`
}
type ResourceJobTaskNewClusterLibrary struct {
Egg string `json:"egg,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceJobTaskNewClusterLibraryCran `json:"cran,omitempty"`
Maven *ResourceJobTaskNewClusterLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceJobTaskNewClusterLibraryPypi `json:"pypi,omitempty"`
}
type ResourceJobTaskNewClusterWorkloadTypeClients struct {
Jobs bool `json:"jobs,omitempty"`
Notebooks bool `json:"notebooks,omitempty"`
@ -1030,7 +1187,6 @@ type ResourceJobTaskNewClusterWorkloadType struct {
type ResourceJobTaskNewCluster struct {
ApplyPolicyDefaultValues bool `json:"apply_policy_default_values,omitempty"`
AutoterminationMinutes int `json:"autotermination_minutes,omitempty"`
ClusterId string `json:"cluster_id,omitempty"`
ClusterName string `json:"cluster_name,omitempty"`
CustomTags map[string]string `json:"custom_tags,omitempty"`
@ -1058,6 +1214,7 @@ type ResourceJobTaskNewCluster struct {
DockerImage *ResourceJobTaskNewClusterDockerImage `json:"docker_image,omitempty"`
GcpAttributes *ResourceJobTaskNewClusterGcpAttributes `json:"gcp_attributes,omitempty"`
InitScripts []ResourceJobTaskNewClusterInitScripts `json:"init_scripts,omitempty"`
Library []ResourceJobTaskNewClusterLibrary `json:"library,omitempty"`
WorkloadType *ResourceJobTaskNewClusterWorkloadType `json:"workload_type,omitempty"`
}
@ -1086,9 +1243,21 @@ type ResourceJobTaskPythonWheelTask struct {
Parameters []string `json:"parameters,omitempty"`
}
type ResourceJobTaskRunJobTaskPipelineParams struct {
FullRefresh bool `json:"full_refresh,omitempty"`
}
type ResourceJobTaskRunJobTask struct {
JobId int `json:"job_id"`
JobParameters map[string]string `json:"job_parameters,omitempty"`
DbtCommands []string `json:"dbt_commands,omitempty"`
JarParams []string `json:"jar_params,omitempty"`
JobId int `json:"job_id"`
JobParameters map[string]string `json:"job_parameters,omitempty"`
NotebookParams map[string]string `json:"notebook_params,omitempty"`
PythonNamedParams map[string]string `json:"python_named_params,omitempty"`
PythonParams []string `json:"python_params,omitempty"`
SparkSubmitParams []string `json:"spark_submit_params,omitempty"`
SqlParams map[string]string `json:"sql_params,omitempty"`
PipelineParams *ResourceJobTaskRunJobTaskPipelineParams `json:"pipeline_params,omitempty"`
}
type ResourceJobTaskSparkJarTask struct {
@ -1141,7 +1310,7 @@ type ResourceJobTaskSqlTaskQuery struct {
type ResourceJobTaskSqlTask struct {
Parameters map[string]string `json:"parameters,omitempty"`
WarehouseId string `json:"warehouse_id,omitempty"`
WarehouseId string `json:"warehouse_id"`
Alert *ResourceJobTaskSqlTaskAlert `json:"alert,omitempty"`
Dashboard *ResourceJobTaskSqlTaskDashboard `json:"dashboard,omitempty"`
File *ResourceJobTaskSqlTaskFile `json:"file,omitempty"`
@ -1172,34 +1341,35 @@ type ResourceJobTaskWebhookNotifications struct {
}
type ResourceJobTask struct {
Description string `json:"description,omitempty"`
EnvironmentKey string `json:"environment_key,omitempty"`
ExistingClusterId string `json:"existing_cluster_id,omitempty"`
JobClusterKey string `json:"job_cluster_key,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"`
DependsOn []ResourceJobTaskDependsOn `json:"depends_on,omitempty"`
EmailNotifications *ResourceJobTaskEmailNotifications `json:"email_notifications,omitempty"`
ForEachTask *ResourceJobTaskForEachTask `json:"for_each_task,omitempty"`
Health *ResourceJobTaskHealth `json:"health,omitempty"`
Library []ResourceJobTaskLibrary `json:"library,omitempty"`
NewCluster *ResourceJobTaskNewCluster `json:"new_cluster,omitempty"`
NotebookTask *ResourceJobTaskNotebookTask `json:"notebook_task,omitempty"`
NotificationSettings *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"`
PipelineTask *ResourceJobTaskPipelineTask `json:"pipeline_task,omitempty"`
PythonWheelTask *ResourceJobTaskPythonWheelTask `json:"python_wheel_task,omitempty"`
RunJobTask *ResourceJobTaskRunJobTask `json:"run_job_task,omitempty"`
SparkJarTask *ResourceJobTaskSparkJarTask `json:"spark_jar_task,omitempty"`
SparkPythonTask *ResourceJobTaskSparkPythonTask `json:"spark_python_task,omitempty"`
SparkSubmitTask *ResourceJobTaskSparkSubmitTask `json:"spark_submit_task,omitempty"`
SqlTask *ResourceJobTaskSqlTask `json:"sql_task,omitempty"`
WebhookNotifications *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
Description string `json:"description,omitempty"`
DisableAutoOptimization bool `json:"disable_auto_optimization,omitempty"`
EnvironmentKey string `json:"environment_key,omitempty"`
ExistingClusterId string `json:"existing_cluster_id,omitempty"`
JobClusterKey string `json:"job_cluster_key,omitempty"`
MaxRetries int `json:"max_retries,omitempty"`
MinRetryIntervalMillis int `json:"min_retry_interval_millis,omitempty"`
RetryOnTimeout bool `json:"retry_on_timeout,omitempty"`
RunIf string `json:"run_if,omitempty"`
TaskKey string `json:"task_key"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
ConditionTask *ResourceJobTaskConditionTask `json:"condition_task,omitempty"`
DbtTask *ResourceJobTaskDbtTask `json:"dbt_task,omitempty"`
DependsOn []ResourceJobTaskDependsOn `json:"depends_on,omitempty"`
EmailNotifications *ResourceJobTaskEmailNotifications `json:"email_notifications,omitempty"`
ForEachTask *ResourceJobTaskForEachTask `json:"for_each_task,omitempty"`
Health *ResourceJobTaskHealth `json:"health,omitempty"`
Library []ResourceJobTaskLibrary `json:"library,omitempty"`
NewCluster *ResourceJobTaskNewCluster `json:"new_cluster,omitempty"`
NotebookTask *ResourceJobTaskNotebookTask `json:"notebook_task,omitempty"`
NotificationSettings *ResourceJobTaskNotificationSettings `json:"notification_settings,omitempty"`
PipelineTask *ResourceJobTaskPipelineTask `json:"pipeline_task,omitempty"`
PythonWheelTask *ResourceJobTaskPythonWheelTask `json:"python_wheel_task,omitempty"`
RunJobTask *ResourceJobTaskRunJobTask `json:"run_job_task,omitempty"`
SparkJarTask *ResourceJobTaskSparkJarTask `json:"spark_jar_task,omitempty"`
SparkPythonTask *ResourceJobTaskSparkPythonTask `json:"spark_python_task,omitempty"`
SparkSubmitTask *ResourceJobTaskSparkSubmitTask `json:"spark_submit_task,omitempty"`
SqlTask *ResourceJobTaskSqlTask `json:"sql_task,omitempty"`
WebhookNotifications *ResourceJobTaskWebhookNotifications `json:"webhook_notifications,omitempty"`
}
type ResourceJobTriggerFileArrival struct {
@ -1208,6 +1378,13 @@ type ResourceJobTriggerFileArrival struct {
WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"`
}
type ResourceJobTriggerTable struct {
Condition string `json:"condition,omitempty"`
MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"`
TableNames []string `json:"table_names,omitempty"`
WaitAfterLastChangeSeconds int `json:"wait_after_last_change_seconds,omitempty"`
}
type ResourceJobTriggerTableUpdate struct {
Condition string `json:"condition,omitempty"`
MinTimeBetweenTriggersSeconds int `json:"min_time_between_triggers_seconds,omitempty"`
@ -1218,6 +1395,7 @@ type ResourceJobTriggerTableUpdate struct {
type ResourceJobTrigger struct {
PauseStatus string `json:"pause_status,omitempty"`
FileArrival *ResourceJobTriggerFileArrival `json:"file_arrival,omitempty"`
Table *ResourceJobTriggerTable `json:"table,omitempty"`
TableUpdate *ResourceJobTriggerTableUpdate `json:"table_update,omitempty"`
}

17
bundle/internal/tf/schema/resource_library.go
View File

@ -19,12 +19,13 @@ type ResourceLibraryPypi struct {
}
type ResourceLibrary struct {
ClusterId string `json:"cluster_id"`
Egg string `json:"egg,omitempty"`
Id string `json:"id,omitempty"`
Jar string `json:"jar,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceLibraryCran `json:"cran,omitempty"`
Maven *ResourceLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceLibraryPypi `json:"pypi,omitempty"`
ClusterId string `json:"cluster_id"`
Egg string `json:"egg,omitempty"`
Id string `json:"id,omitempty"`
Jar string `json:"jar,omitempty"`
Requirements string `json:"requirements,omitempty"`
Whl string `json:"whl,omitempty"`
Cran *ResourceLibraryCran `json:"cran,omitempty"`
Maven *ResourceLibraryMaven `json:"maven,omitempty"`
Pypi *ResourceLibraryPypi `json:"pypi,omitempty"`
}

16
bundle/internal/tf/schema/resource_model_serving.go
View File

@ -34,12 +34,15 @@ type ResourceModelServingConfigServedEntitiesExternalModelDatabricksModelServing
}
type ResourceModelServingConfigServedEntitiesExternalModelOpenaiConfig struct {
OpenaiApiBase string `json:"openai_api_base,omitempty"`
OpenaiApiKey string `json:"openai_api_key"`
OpenaiApiType string `json:"openai_api_type,omitempty"`
OpenaiApiVersion string `json:"openai_api_version,omitempty"`
OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"`
OpenaiOrganization string `json:"openai_organization,omitempty"`
MicrosoftEntraClientId string `json:"microsoft_entra_client_id,omitempty"`
MicrosoftEntraClientSecret string `json:"microsoft_entra_client_secret,omitempty"`
MicrosoftEntraTenantId string `json:"microsoft_entra_tenant_id,omitempty"`
OpenaiApiBase string `json:"openai_api_base,omitempty"`
OpenaiApiKey string `json:"openai_api_key,omitempty"`
OpenaiApiType string `json:"openai_api_type,omitempty"`
OpenaiApiVersion string `json:"openai_api_version,omitempty"`
OpenaiDeploymentName string `json:"openai_deployment_name,omitempty"`
OpenaiOrganization string `json:"openai_organization,omitempty"`
}
type ResourceModelServingConfigServedEntitiesExternalModelPalmConfig struct {
@ -114,6 +117,7 @@ type ResourceModelServingTags struct {
type ResourceModelServing struct {
Id string `json:"id,omitempty"`
Name string `json:"name"`
RouteOptimized bool `json:"route_optimized,omitempty"`
ServingEndpointId string `json:"serving_endpoint_id,omitempty"`
Config *ResourceModelServingConfig `json:"config,omitempty"`
RateLimits []ResourceModelServingRateLimits `json:"rate_limits,omitempty"`

9
bundle/internal/tf/schema/resource_mws_ncc_binding.go Normal file
View File

@ -0,0 +1,9 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceMwsNccBinding struct {
Id string `json:"id,omitempty"`
NetworkConnectivityConfigId string `json:"network_connectivity_config_id"`
WorkspaceId int `json:"workspace_id"`
}

17
bundle/internal/tf/schema/resource_mws_ncc_private_endpoint_rule.go Normal file
View File

@ -0,0 +1,17 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceMwsNccPrivateEndpointRule struct {
ConnectionState string `json:"connection_state,omitempty"`
CreationTime int `json:"creation_time,omitempty"`
Deactivated bool `json:"deactivated,omitempty"`
DeactivatedAt int `json:"deactivated_at,omitempty"`
EndpointName string `json:"endpoint_name,omitempty"`
GroupId string `json:"group_id"`
Id string `json:"id,omitempty"`
NetworkConnectivityConfigId string `json:"network_connectivity_config_id"`
ResourceId string `json:"resource_id"`
RuleId string `json:"rule_id,omitempty"`
UpdatedTime int `json:"updated_time,omitempty"`
}

51
bundle/internal/tf/schema/resource_mws_network_connectivity_config.go Normal file
View File

@ -0,0 +1,51 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule struct {
CidrBlocks []string `json:"cidr_blocks,omitempty"`
}
type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule struct {
Subnets []string `json:"subnets,omitempty"`
TargetRegion string `json:"target_region,omitempty"`
TargetServices []string `json:"target_services,omitempty"`
}
type ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules struct {
AwsStableIpRule *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAwsStableIpRule `json:"aws_stable_ip_rule,omitempty"`
AzureServiceEndpointRule *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRulesAzureServiceEndpointRule `json:"azure_service_endpoint_rule,omitempty"`
}
type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules struct {
ConnectionState string `json:"connection_state,omitempty"`
CreationTime int `json:"creation_time,omitempty"`
Deactivated bool `json:"deactivated,omitempty"`
DeactivatedAt int `json:"deactivated_at,omitempty"`
EndpointName string `json:"endpoint_name,omitempty"`
GroupId string `json:"group_id,omitempty"`
NetworkConnectivityConfigId string `json:"network_connectivity_config_id,omitempty"`
ResourceId string `json:"resource_id,omitempty"`
RuleId string `json:"rule_id,omitempty"`
UpdatedTime int `json:"updated_time,omitempty"`
}
type ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules struct {
AzurePrivateEndpointRules []ResourceMwsNetworkConnectivityConfigEgressConfigTargetRulesAzurePrivateEndpointRules `json:"azure_private_endpoint_rules,omitempty"`
}
type ResourceMwsNetworkConnectivityConfigEgressConfig struct {
DefaultRules *ResourceMwsNetworkConnectivityConfigEgressConfigDefaultRules `json:"default_rules,omitempty"`
TargetRules *ResourceMwsNetworkConnectivityConfigEgressConfigTargetRules `json:"target_rules,omitempty"`
}
type ResourceMwsNetworkConnectivityConfig struct {
AccountId string `json:"account_id,omitempty"`
CreationTime int `json:"creation_time,omitempty"`
Id string `json:"id,omitempty"`
Name string `json:"name"`
NetworkConnectivityConfigId string `json:"network_connectivity_config_id,omitempty"`
Region string `json:"region"`
UpdatedTime int `json:"updated_time,omitempty"`
EgressConfig *ResourceMwsNetworkConnectivityConfigEgressConfig `json:"egress_config,omitempty"`
}

76
bundle/internal/tf/schema/resource_quality_monitor.go Normal file
View File

@ -0,0 +1,76 @@
// Generated from Databricks Terraform provider schema. DO NOT EDIT.
package schema
type ResourceQualityMonitorCustomMetrics struct {
Definition string `json:"definition"`
InputColumns []string `json:"input_columns"`
Name string `json:"name"`
OutputDataType string `json:"output_data_type"`
Type string `json:"type"`
}
type ResourceQualityMonitorDataClassificationConfig struct {
Enabled bool `json:"enabled,omitempty"`
}
type ResourceQualityMonitorInferenceLog struct {
Granularities []string `json:"granularities"`
LabelCol string `json:"label_col,omitempty"`
ModelIdCol string `json:"model_id_col"`
PredictionCol string `json:"prediction_col"`
PredictionProbaCol string `json:"prediction_proba_col,omitempty"`
ProblemType string `json:"problem_type"`
TimestampCol string `json:"timestamp_col"`
}
type ResourceQualityMonitorNotificationsOnFailure struct {
EmailAddresses []string `json:"email_addresses,omitempty"`
}
type ResourceQualityMonitorNotificationsOnNewClassificationTagDetected struct {
EmailAddresses []string `json:"email_addresses,omitempty"`
}
type ResourceQualityMonitorNotifications struct {
OnFailure *ResourceQualityMonitorNotificationsOnFailure `json:"on_failure,omitempty"`
OnNewClassificationTagDetected *ResourceQualityMonitorNotificationsOnNewClassificationTagDetected `json:"on_new_classification_tag_detected,omitempty"`
}
type ResourceQualityMonitorSchedule struct {
PauseStatus string `json:"pause_status,omitempty"`
QuartzCronExpression string `json:"quartz_cron_expression"`
TimezoneId string `json:"timezone_id"`
}
type ResourceQualityMonitorSnapshot struct {
}
type ResourceQualityMonitorTimeSeries struct {
Granularities []string `json:"granularities"`
TimestampCol string `json:"timestamp_col"`
}
type ResourceQualityMonitor struct {
AssetsDir string `json:"assets_dir"`
BaselineTableName string `json:"baseline_table_name,omitempty"`
DashboardId string `json:"dashboard_id,omitempty"`
DriftMetricsTableName string `json:"drift_metrics_table_name,omitempty"`
Id string `json:"id,omitempty"`
LatestMonitorFailureMsg string `json:"latest_monitor_failure_msg,omitempty"`
MonitorVersion string `json:"monitor_version,omitempty"`
OutputSchemaName string `json:"output_schema_name"`
ProfileMetricsTableName string `json:"profile_metrics_table_name,omitempty"`
SkipBuiltinDashboard bool `json:"skip_builtin_dashboard,omitempty"`
SlicingExprs []string `json:"slicing_exprs,omitempty"`
Status string `json:"status,omitempty"`
TableName string `json:"table_name"`
WarehouseId string `json:"warehouse_id,omitempty"`
CustomMetrics []ResourceQualityMonitorCustomMetrics `json:"custom_metrics,omitempty"`
DataClassificationConfig *ResourceQualityMonitorDataClassificationConfig `json:"data_classification_config,omitempty"`
InferenceLog *ResourceQualityMonitorInferenceLog `json:"inference_log,omitempty"`
Notifications *ResourceQualityMonitorNotifications `json:"notifications,omitempty"`
Schedule *ResourceQualityMonitorSchedule `json:"schedule,omitempty"`
Snapshot *ResourceQualityMonitorSnapshot `json:"snapshot,omitempty"`
TimeSeries *ResourceQualityMonitorTimeSeries `json:"time_series,omitempty"`
}

1
bundle/internal/tf/schema/resource_sql_table.go
View File

@ -18,6 +18,7 @@ type ResourceSqlTable struct {
Id string `json:"id,omitempty"`
Name string `json:"name"`
Options map[string]string `json:"options,omitempty"`
Owner string `json:"owner,omitempty"`
Partitions []string `json:"partitions,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
SchemaName string `json:"schema_name"`

11
bundle/internal/tf/schema/resource_vector_search_index.go
View File

@ -13,11 +13,12 @@ type ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns struct {
}
type ResourceVectorSearchIndexDeltaSyncIndexSpec struct {
PipelineId string `json:"pipeline_id,omitempty"`
PipelineType string `json:"pipeline_type,omitempty"`
SourceTable string `json:"source_table,omitempty"`
EmbeddingSourceColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"`
EmbeddingVectorColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"`
EmbeddingWritebackTable string `json:"embedding_writeback_table,omitempty"`
PipelineId string `json:"pipeline_id,omitempty"`
PipelineType string `json:"pipeline_type,omitempty"`
SourceTable string `json:"source_table,omitempty"`
EmbeddingSourceColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingSourceColumns `json:"embedding_source_columns,omitempty"`
EmbeddingVectorColumns []ResourceVectorSearchIndexDeltaSyncIndexSpecEmbeddingVectorColumns `json:"embedding_vector_columns,omitempty"`
}
type ResourceVectorSearchIndexDirectAccessIndexSpecEmbeddingSourceColumns struct {

218
bundle/internal/tf/schema/resources.go
View File

@ -3,112 +3,122 @@
package schema
type Resources struct {
AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"`
ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"`
AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"`
AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"`
Catalog map[string]any `json:"databricks_catalog,omitempty"`
CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
Cluster map[string]any `json:"databricks_cluster,omitempty"`
ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"`
Connection map[string]any `json:"databricks_connection,omitempty"`
DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"`
DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"`
Directory map[string]any `json:"databricks_directory,omitempty"`
Entitlements map[string]any `json:"databricks_entitlements,omitempty"`
ExternalLocation map[string]any `json:"databricks_external_location,omitempty"`
File map[string]any `json:"databricks_file,omitempty"`
GitCredential map[string]any `json:"databricks_git_credential,omitempty"`
GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"`
Grant map[string]any `json:"databricks_grant,omitempty"`
Grants map[string]any `json:"databricks_grants,omitempty"`
Group map[string]any `json:"databricks_group,omitempty"`
GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"`
GroupMember map[string]any `json:"databricks_group_member,omitempty"`
GroupRole map[string]any `json:"databricks_group_role,omitempty"`
InstancePool map[string]any `json:"databricks_instance_pool,omitempty"`
InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"`
IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"`
Job map[string]any `json:"databricks_job,omitempty"`
LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
Library map[string]any `json:"databricks_library,omitempty"`
Metastore map[string]any `json:"databricks_metastore,omitempty"`
MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"`
MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"`
MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"`
MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"`
MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"`
ModelServing map[string]any `json:"databricks_model_serving,omitempty"`
Mount map[string]any `json:"databricks_mount,omitempty"`
MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"`
MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"`
MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"`
MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"`
Notebook map[string]any `json:"databricks_notebook,omitempty"`
OboToken map[string]any `json:"databricks_obo_token,omitempty"`
OnlineTable map[string]any `json:"databricks_online_table,omitempty"`
PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"`
Permissions map[string]any `json:"databricks_permissions,omitempty"`
Pipeline map[string]any `json:"databricks_pipeline,omitempty"`
Provider map[string]any `json:"databricks_provider,omitempty"`
Recipient map[string]any `json:"databricks_recipient,omitempty"`
RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"`
Repo map[string]any `json:"databricks_repo,omitempty"`
RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
Schema map[string]any `json:"databricks_schema,omitempty"`
Secret map[string]any `json:"databricks_secret,omitempty"`
SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"`
SecretScope map[string]any `json:"databricks_secret_scope,omitempty"`
ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"`
ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"`
ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"`
Share map[string]any `json:"databricks_share,omitempty"`
SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"`
SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"`
SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"`
SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"`
SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"`
SqlQuery map[string]any `json:"databricks_sql_query,omitempty"`
SqlTable map[string]any `json:"databricks_sql_table,omitempty"`
SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"`
SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"`
StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"`
SystemSchema map[string]any `json:"databricks_system_schema,omitempty"`
Table map[string]any `json:"databricks_table,omitempty"`
Token map[string]any `json:"databricks_token,omitempty"`
User map[string]any `json:"databricks_user,omitempty"`
UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"`
UserRole map[string]any `json:"databricks_user_role,omitempty"`
VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"`
Volume map[string]any `json:"databricks_volume,omitempty"`
WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"`
WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"`
AccessControlRuleSet map[string]any `json:"databricks_access_control_rule_set,omitempty"`
ArtifactAllowlist map[string]any `json:"databricks_artifact_allowlist,omitempty"`
AutomaticClusterUpdateWorkspaceSetting map[string]any `json:"databricks_automatic_cluster_update_workspace_setting,omitempty"`
AwsS3Mount map[string]any `json:"databricks_aws_s3_mount,omitempty"`
AzureAdlsGen1Mount map[string]any `json:"databricks_azure_adls_gen1_mount,omitempty"`
AzureAdlsGen2Mount map[string]any `json:"databricks_azure_adls_gen2_mount,omitempty"`
AzureBlobMount map[string]any `json:"databricks_azure_blob_mount,omitempty"`
Catalog map[string]any `json:"databricks_catalog,omitempty"`
CatalogWorkspaceBinding map[string]any `json:"databricks_catalog_workspace_binding,omitempty"`
Cluster map[string]any `json:"databricks_cluster,omitempty"`
ClusterPolicy map[string]any `json:"databricks_cluster_policy,omitempty"`
ComplianceSecurityProfileWorkspaceSetting map[string]any `json:"databricks_compliance_security_profile_workspace_setting,omitempty"`
Connection map[string]any `json:"databricks_connection,omitempty"`
DbfsFile map[string]any `json:"databricks_dbfs_file,omitempty"`
DefaultNamespaceSetting map[string]any `json:"databricks_default_namespace_setting,omitempty"`
Directory map[string]any `json:"databricks_directory,omitempty"`
EnhancedSecurityMonitoringWorkspaceSetting map[string]any `json:"databricks_enhanced_security_monitoring_workspace_setting,omitempty"`
Entitlements map[string]any `json:"databricks_entitlements,omitempty"`
ExternalLocation map[string]any `json:"databricks_external_location,omitempty"`
File map[string]any `json:"databricks_file,omitempty"`
GitCredential map[string]any `json:"databricks_git_credential,omitempty"`
GlobalInitScript map[string]any `json:"databricks_global_init_script,omitempty"`
Grant map[string]any `json:"databricks_grant,omitempty"`
Grants map[string]any `json:"databricks_grants,omitempty"`
Group map[string]any `json:"databricks_group,omitempty"`
GroupInstanceProfile map[string]any `json:"databricks_group_instance_profile,omitempty"`
GroupMember map[string]any `json:"databricks_group_member,omitempty"`
GroupRole map[string]any `json:"databricks_group_role,omitempty"`
InstancePool map[string]any `json:"databricks_instance_pool,omitempty"`
InstanceProfile map[string]any `json:"databricks_instance_profile,omitempty"`
IpAccessList map[string]any `json:"databricks_ip_access_list,omitempty"`
Job map[string]any `json:"databricks_job,omitempty"`
LakehouseMonitor map[string]any `json:"databricks_lakehouse_monitor,omitempty"`
Library map[string]any `json:"databricks_library,omitempty"`
Metastore map[string]any `json:"databricks_metastore,omitempty"`
MetastoreAssignment map[string]any `json:"databricks_metastore_assignment,omitempty"`
MetastoreDataAccess map[string]any `json:"databricks_metastore_data_access,omitempty"`
MlflowExperiment map[string]any `json:"databricks_mlflow_experiment,omitempty"`
MlflowModel map[string]any `json:"databricks_mlflow_model,omitempty"`
MlflowWebhook map[string]any `json:"databricks_mlflow_webhook,omitempty"`
ModelServing map[string]any `json:"databricks_model_serving,omitempty"`
Mount map[string]any `json:"databricks_mount,omitempty"`
MwsCredentials map[string]any `json:"databricks_mws_credentials,omitempty"`
MwsCustomerManagedKeys map[string]any `json:"databricks_mws_customer_managed_keys,omitempty"`
MwsLogDelivery map[string]any `json:"databricks_mws_log_delivery,omitempty"`
MwsNccBinding map[string]any `json:"databricks_mws_ncc_binding,omitempty"`
MwsNccPrivateEndpointRule map[string]any `json:"databricks_mws_ncc_private_endpoint_rule,omitempty"`
MwsNetworkConnectivityConfig map[string]any `json:"databricks_mws_network_connectivity_config,omitempty"`
MwsNetworks map[string]any `json:"databricks_mws_networks,omitempty"`
MwsPermissionAssignment map[string]any `json:"databricks_mws_permission_assignment,omitempty"`
MwsPrivateAccessSettings map[string]any `json:"databricks_mws_private_access_settings,omitempty"`
MwsStorageConfigurations map[string]any `json:"databricks_mws_storage_configurations,omitempty"`
MwsVpcEndpoint map[string]any `json:"databricks_mws_vpc_endpoint,omitempty"`
MwsWorkspaces map[string]any `json:"databricks_mws_workspaces,omitempty"`
Notebook map[string]any `json:"databricks_notebook,omitempty"`
OboToken map[string]any `json:"databricks_obo_token,omitempty"`
OnlineTable map[string]any `json:"databricks_online_table,omitempty"`
PermissionAssignment map[string]any `json:"databricks_permission_assignment,omitempty"`
Permissions map[string]any `json:"databricks_permissions,omitempty"`
Pipeline map[string]any `json:"databricks_pipeline,omitempty"`
Provider map[string]any `json:"databricks_provider,omitempty"`
QualityMonitor map[string]any `json:"databricks_quality_monitor,omitempty"`
Recipient map[string]any `json:"databricks_recipient,omitempty"`
RegisteredModel map[string]any `json:"databricks_registered_model,omitempty"`
Repo map[string]any `json:"databricks_repo,omitempty"`
RestrictWorkspaceAdminsSetting map[string]any `json:"databricks_restrict_workspace_admins_setting,omitempty"`
Schema map[string]any `json:"databricks_schema,omitempty"`
Secret map[string]any `json:"databricks_secret,omitempty"`
SecretAcl map[string]any `json:"databricks_secret_acl,omitempty"`
SecretScope map[string]any `json:"databricks_secret_scope,omitempty"`
ServicePrincipal map[string]any `json:"databricks_service_principal,omitempty"`
ServicePrincipalRole map[string]any `json:"databricks_service_principal_role,omitempty"`
ServicePrincipalSecret map[string]any `json:"databricks_service_principal_secret,omitempty"`
Share map[string]any `json:"databricks_share,omitempty"`
SqlAlert map[string]any `json:"databricks_sql_alert,omitempty"`
SqlDashboard map[string]any `json:"databricks_sql_dashboard,omitempty"`
SqlEndpoint map[string]any `json:"databricks_sql_endpoint,omitempty"`
SqlGlobalConfig map[string]any `json:"databricks_sql_global_config,omitempty"`
SqlPermissions map[string]any `json:"databricks_sql_permissions,omitempty"`
SqlQuery map[string]any `json:"databricks_sql_query,omitempty"`
SqlTable map[string]any `json:"databricks_sql_table,omitempty"`
SqlVisualization map[string]any `json:"databricks_sql_visualization,omitempty"`
SqlWidget map[string]any `json:"databricks_sql_widget,omitempty"`
StorageCredential map[string]any `json:"databricks_storage_credential,omitempty"`
SystemSchema map[string]any `json:"databricks_system_schema,omitempty"`
Table map[string]any `json:"databricks_table,omitempty"`
Token map[string]any `json:"databricks_token,omitempty"`
User map[string]any `json:"databricks_user,omitempty"`
UserInstanceProfile map[string]any `json:"databricks_user_instance_profile,omitempty"`
UserRole map[string]any `json:"databricks_user_role,omitempty"`
VectorSearchEndpoint map[string]any `json:"databricks_vector_search_endpoint,omitempty"`
VectorSearchIndex map[string]any `json:"databricks_vector_search_index,omitempty"`
Volume map[string]any `json:"databricks_volume,omitempty"`
WorkspaceConf map[string]any `json:"databricks_workspace_conf,omitempty"`
WorkspaceFile map[string]any `json:"databricks_workspace_file,omitempty"`
}
func NewResources() *Resources {
return &Resources{
AccessControlRuleSet: make(map[string]any),
ArtifactAllowlist: make(map[string]any),
AwsS3Mount: make(map[string]any),
AzureAdlsGen1Mount: make(map[string]any),
AzureAdlsGen2Mount: make(map[string]any),
AzureBlobMount: make(map[string]any),
Catalog: make(map[string]any),
CatalogWorkspaceBinding: make(map[string]any),
Cluster: make(map[string]any),
ClusterPolicy: make(map[string]any),
Connection: make(map[string]any),
DbfsFile: make(map[string]any),
DefaultNamespaceSetting: make(map[string]any),
Directory: make(map[string]any),
AccessControlRuleSet: make(map[string]any),
ArtifactAllowlist: make(map[string]any),
AutomaticClusterUpdateWorkspaceSetting: make(map[string]any),
AwsS3Mount: make(map[string]any),
AzureAdlsGen1Mount: make(map[string]any),
AzureAdlsGen2Mount: make(map[string]any),
AzureBlobMount: make(map[string]any),
Catalog: make(map[string]any),
CatalogWorkspaceBinding: make(map[string]any),
Cluster: make(map[string]any),
ClusterPolicy: make(map[string]any),
ComplianceSecurityProfileWorkspaceSetting: make(map[string]any),
Connection: make(map[string]any),
DbfsFile: make(map[string]any),
DefaultNamespaceSetting: make(map[string]any),
Directory: make(map[string]any),
EnhancedSecurityMonitoringWorkspaceSetting: make(map[string]any),
Entitlements: make(map[string]any),
ExternalLocation: make(map[string]any),
File: make(map[string]any),
@ -137,6 +147,9 @@ func NewResources() *Resources {
MwsCredentials: make(map[string]any),
MwsCustomerManagedKeys: make(map[string]any),
MwsLogDelivery: make(map[string]any),
MwsNccBinding: make(map[string]any),
MwsNccPrivateEndpointRule: make(map[string]any),
MwsNetworkConnectivityConfig: make(map[string]any),
MwsNetworks: make(map[string]any),
MwsPermissionAssignment: make(map[string]any),
MwsPrivateAccessSettings: make(map[string]any),
@ -150,6 +163,7 @@ func NewResources() *Resources {
Permissions: make(map[string]any),
Pipeline: make(map[string]any),
Provider: make(map[string]any),
QualityMonitor: make(map[string]any),
Recipient: make(map[string]any),
RegisteredModel: make(map[string]any),
Repo: make(map[string]any),

2
bundle/internal/tf/schema/root.go
View File

@ -21,7 +21,7 @@ type Root struct {
const ProviderHost = "registry.terraform.io"
const ProviderSource = "databricks/databricks"
const ProviderVersion = "1.40.0"
const ProviderVersion = "1.46.0"
func NewRoot() *Root {
return &Root{

4
bundle/libraries/libraries.go
View File

@ -30,6 +30,10 @@ func FindAllEnvironments(b *bundle.Bundle) map[string]([]jobs.JobEnvironment) {
func isEnvsWithLocalLibraries(envs []jobs.JobEnvironment) bool {
for _, e := range envs {
if e.Spec == nil {
continue
}
for _, l := range e.Spec.Dependencies {
if IsEnvironmentDependencyLocal(l) {
return true

4
bundle/libraries/match.go
View File

@ -62,6 +62,10 @@ func validateTaskLibraries(libs []compute.Library, b *bundle.Bundle) error {
func validateEnvironments(envs []jobs.JobEnvironment, b *bundle.Bundle) error {
for _, env := range envs {
if env.Spec == nil {
continue
}
for _, dep := range env.Spec.Dependencies {
matches, err := filepath.Glob(filepath.Join(b.RootPath, dep))
if err != nil {

27
bundle/parallel_test.go
View File

@ -2,6 +2,7 @@ package bundle
import (
"context"
"sync"
"testing"
"github.com/databricks/cli/bundle/config"
@ -10,9 +11,14 @@ import (
)
type addToContainer struct {
t *testing.T
container *[]int
value int
err bool
// mu is a mutex that protects container. It is used to ensure that the
// container slice is only modified by one goroutine at a time.
mu *sync.Mutex
}
func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagnostics {
@ -20,9 +26,10 @@ func (m *addToContainer) Apply(ctx context.Context, b ReadOnlyBundle) diag.Diagn
return diag.Errorf("error")
}
c := *m.container
c = append(c, m.value)
*m.container = c
m.mu.Lock()
*m.container = append(*m.container, m.value)
m.mu.Unlock()
return nil
}
@ -36,9 +43,10 @@ func TestParallelMutatorWork(t *testing.T) {
}
container := []int{}
m1 := &addToContainer{container: &container, value: 1}
m2 := &addToContainer{container: &container, value: 2}
m3 := &addToContainer{container: &container, value: 3}
var mu sync.Mutex
m1 := &addToContainer{t: t, container: &container, value: 1, mu: &mu}
m2 := &addToContainer{t: t, container: &container, value: 2, mu: &mu}
m3 := &addToContainer{t: t, container: &container, value: 3, mu: &mu}
m := Parallel(m1, m2, m3)
@ -57,9 +65,10 @@ func TestParallelMutatorWorkWithErrors(t *testing.T) {
}
container := []int{}
m1 := &addToContainer{container: &container, value: 1}
m2 := &addToContainer{container: &container, err: true, value: 2}
m3 := &addToContainer{container: &container, value: 3}
var mu sync.Mutex
m1 := &addToContainer{container: &container, value: 1, mu: &mu}
m2 := &addToContainer{container: &container, err: true, value: 2, mu: &mu}
m3 := &addToContainer{container: &container, value: 3, mu: &mu}
m := Parallel(m1, m2, m3)

7
bundle/permissions/filter_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/assert"
)
@ -45,9 +46,15 @@ func testFixture(userName string) *bundle.Bundle {
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
JobSettings: &jobs.JobSettings{
Name: "job1",
},
Permissions: p,
},
"job2": {
JobSettings: &jobs.JobSettings{
Name: "job2",
},
Permissions: p,
},
},

19
bundle/permissions/mutator_test.go
View File

@ -7,6 +7,7 @@ import (
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/require"
)
@ -23,8 +24,16 @@ func TestApplyBundlePermissions(t *testing.T) {
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job_1": {},
"job_2": {},
"job_1": {
JobSettings: &jobs.JobSettings{
Name: "job_1",
},
},
"job_2": {
JobSettings: &jobs.JobSettings{
Name: "job_2",
},
},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline_1": {},
@ -109,11 +118,17 @@ func TestWarningOnOverlapPermission(t *testing.T) {
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job_1": {
JobSettings: &jobs.JobSettings{
Name: "job_1",
},
Permissions: []resources.Permission{
{Level: CAN_VIEW, UserName: "TestUser"},
},
},
"job_2": {
JobSettings: &jobs.JobSettings{
Name: "job_2",
},
Permissions: []resources.Permission{
{Level: CAN_VIEW, UserName: "TestUser2"},
},

4
bundle/permissions/workspace_root_test.go
View File

@ -30,8 +30,8 @@ func TestApplyWorkspaceRootPermissions(t *testing.T) {
},
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job_1": {JobSettings: &jobs.JobSettings{}},
"job_2": {JobSettings: &jobs.JobSettings{}},
"job_1": {JobSettings: &jobs.JobSettings{Name: "job_1"}},
"job_2": {JobSettings: &jobs.JobSettings{Name: "job_2"}},
},
Pipelines: map[string]*resources.Pipeline{
"pipeline_1": {PipelineSpec: &pipelines.PipelineSpec{}},

2
bundle/phases/deploy.go
View File

@ -36,7 +36,7 @@ func Deploy() bundle.Mutator {
permissions.ApplyWorkspaceRootPermissions(),
terraform.Interpolate(),
terraform.Write(),
deploy.CheckRunningResource(),
terraform.CheckRunningResource(),
bundle.Defer(
terraform.Apply(),
bundle.Seq(

1
bundle/phases/initialize.go
View File

@ -46,6 +46,7 @@ func Initialize() bundle.Mutator {
permissions.ApplyBundlePermissions(),
permissions.FilterCurrentUser(),
metadata.AnnotateJobs(),
metadata.AnnotatePipelines(),
terraform.Initialize(),
scripts.Execute(config.ScriptPostInit),
},

2
bundle/schema/docs.go
View File

@ -70,7 +70,7 @@ func UpdateBundleDescriptions(openapiSpecPath string) (*Docs, error) {
}
openapiReader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
// Generate descriptions for the "resources" field

445
bundle/schema/docs/bundle_descriptions.json generated
View File

@ -46,6 +46,17 @@
"properties": {
"fail_on_active_runs": {
"description": ""
},
"lock": {
"description": "",
"properties": {
"enabled": {
"description": ""
},
"force": {
"description": ""
}
}
}
}
},
@ -76,6 +87,9 @@
"additionalproperties": {
"description": ""
}
},
"use_legacy_run_as": {
"description": ""
}
}
},
@ -242,7 +256,7 @@
"description": "",
"properties": {
"client": {
"description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version"
"description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version."
},
"dependencies": {
"description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]",
@ -334,7 +348,7 @@
"description": "If new_cluster, a description of a cluster that is created for each task.",
"properties": {
"apply_policy_default_values": {
"description": ""
"description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
},
"autoscale": {
"description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@ -410,14 +424,6 @@
}
}
},
"clone_from": {
"description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
"properties": {
"source_cluster_id": {
"description": "The cluster that is being cloned."
}
}
},
"cluster_log_conf": {
"description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
"properties": {
@ -460,9 +466,6 @@
"cluster_name": {
"description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
},
"cluster_source": {
"description": ""
},
"custom_tags": {
"description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
"additionalproperties": {
@ -742,7 +745,7 @@
"description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.",
"properties": {
"pause_status": {
"description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
"description": "Indicate whether this schedule is paused or not."
},
"quartz_cron_expression": {
"description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required."
@ -799,7 +802,7 @@
"description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider."
},
"warehouse_id": {
"description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument."
@ -909,10 +912,10 @@
}
},
"egg": {
"description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
},
"jar": {
"description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
},
"maven": {
"description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`",
@ -942,8 +945,11 @@
}
}
},
"requirements": {
"description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`"
},
"whl": {
"description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
}
}
}
@ -955,10 +961,10 @@
"description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried."
},
"new_cluster": {
"description": "If new_cluster, a description of a cluster that is created for each task.",
"description": "If new_cluster, a description of a new cluster that is created for each run.",
"properties": {
"apply_policy_default_values": {
"description": ""
"description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
},
"autoscale": {
"description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@ -1034,14 +1040,6 @@
}
}
},
"clone_from": {
"description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
"properties": {
"source_cluster_id": {
"description": "The cluster that is being cloned."
}
}
},
"cluster_log_conf": {
"description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
"properties": {
@ -1084,9 +1082,6 @@
"cluster_name": {
"description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
},
"cluster_source": {
"description": ""
},
"custom_tags": {
"description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
"additionalproperties": {
@ -1303,6 +1298,9 @@
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
},
"warehouse_id": {
"description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail."
}
}
},
@ -1399,7 +1397,7 @@
}
},
"python_named_params": {
"description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.",
"description": "",
"additionalproperties": {
"description": ""
}
@ -1454,7 +1452,7 @@
"description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository."
}
}
},
@ -1526,13 +1524,13 @@
}
},
"file": {
"description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.",
"description": "If file, indicates that this job runs a SQL file in a remote Git repository.",
"properties": {
"path": {
"description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider."
}
}
},
@ -1634,10 +1632,10 @@
}
},
"pause_status": {
"description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
"description": "Whether this trigger is paused or not."
},
"table": {
"description": "",
"description": "Old table trigger settings name. Deprecated in favor of `table_update`.",
"properties": {
"condition": {
"description": "The table(s) condition based on which to trigger a job run."
@ -1679,7 +1677,7 @@
}
},
"webhook_notifications": {
"description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
"description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
"properties": {
"on_duration_warning_threshold_exceeded": {
"description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@ -1833,6 +1831,15 @@
"openai_config": {
"description": "OpenAI Config. Only required if the provider is 'openai'.",
"properties": {
"microsoft_entra_client_id": {
"description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n"
},
"microsoft_entra_client_secret": {
"description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n"
},
"microsoft_entra_tenant_id": {
"description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n"
},
"openai_api_base": {
"description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n"
},
@ -1989,6 +1996,9 @@
}
}
},
"route_optimized": {
"description": "Enable route optimization for the serving endpoint."
},
"tags": {
"description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.",
"items": {
@ -2415,6 +2425,17 @@
"continuous": {
"description": "Whether the pipeline is continuous or triggered. This replaces `trigger`."
},
"deployment": {
"description": "Deployment type of this pipeline.",
"properties": {
"kind": {
"description": "The deployment method that manages the pipeline."
},
"metadata_file_path": {
"description": "The path to the file containing metadata about the deployment."
}
}
},
"development": {
"description": "Whether the pipeline is in Development mode. Defaults to false."
},
@ -2438,9 +2459,136 @@
}
}
},
"gateway_definition": {
"description": "The definition of a gateway pipeline to support CDC.",
"properties": {
"connection_id": {
"description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source."
},
"gateway_storage_catalog": {
"description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location."
},
"gateway_storage_name": {
"description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n"
},
"gateway_storage_schema": {
"description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location."
}
}
},
"id": {
"description": "Unique identifier for this pipeline."
},
"ingestion_definition": {
"description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.",
"properties": {
"connection_name": {
"description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name."
},
"ingestion_gateway_id": {
"description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name."
},
"objects": {
"description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.",
"items": {
"description": "",
"properties": {
"schema": {
"description": "Select tables from a specific source schema.",
"properties": {
"destination_catalog": {
"description": "Required. Destination catalog to store tables."
},
"destination_schema": {
"description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists."
},
"source_catalog": {
"description": "The source catalog name. Might be optional depending on the type of source."
},
"source_schema": {
"description": "Required. Schema name in the source database."
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
},
"table": {
"description": "Select tables from a specific source table.",
"properties": {
"destination_catalog": {
"description": "Required. Destination catalog to store table."
},
"destination_schema": {
"description": "Required. Destination schema to store table."
},
"destination_table": {
"description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used."
},
"source_catalog": {
"description": "Source catalog name. Might be optional depending on the type of source."
},
"source_schema": {
"description": "Schema name in the source database. Might be optional depending on the type of source."
},
"source_table": {
"description": "Required. Table name in the source database."
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
}
}
}
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
},
"libraries": {
"description": "Libraries or code needed by this deployment.",
"items": {
@ -2682,6 +2830,17 @@
"properties": {
"fail_on_active_runs": {
"description": ""
},
"lock": {
"description": "",
"properties": {
"enabled": {
"description": ""
},
"force": {
"description": ""
}
}
}
}
},
@ -2878,7 +3037,7 @@
"description": "",
"properties": {
"client": {
"description": "*\nUser-friendly name for the client version: “client”: “1”\nThe version is a string, consisting of the major client version"
"description": "Client version used by the environment\nThe client is the user-facing environment of the runtime.\nEach client comes with a specific set of pre-installed libraries.\nThe version is a string, consisting of the major client version."
},
"dependencies": {
"description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e\nE.g. dependencies: [\"foo==0.0.1\", \"-r /Workspace/test/requirements.txt\"]",
@ -2970,7 +3129,7 @@
"description": "If new_cluster, a description of a cluster that is created for each task.",
"properties": {
"apply_policy_default_values": {
"description": ""
"description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
},
"autoscale": {
"description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@ -3046,14 +3205,6 @@
}
}
},
"clone_from": {
"description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
"properties": {
"source_cluster_id": {
"description": "The cluster that is being cloned."
}
}
},
"cluster_log_conf": {
"description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
"properties": {
@ -3096,9 +3247,6 @@
"cluster_name": {
"description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
},
"cluster_source": {
"description": ""
},
"custom_tags": {
"description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
"additionalproperties": {
@ -3378,7 +3526,7 @@
"description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.",
"properties": {
"pause_status": {
"description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
"description": "Indicate whether this schedule is paused or not."
},
"quartz_cron_expression": {
"description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required."
@ -3435,7 +3583,7 @@
"description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider."
},
"warehouse_id": {
"description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument."
@ -3545,10 +3693,10 @@
}
},
"egg": {
"description": "URI of the egg to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"egg\": \"dbfs:/my/egg\" }` or\n`{ \"egg\": \"s3://my-bucket/egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the egg library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"egg\": \"/Workspace/path/to/library.egg\" }`, `{ \"egg\" : \"/Volumes/path/to/library.egg\" }` or\n`{ \"egg\": \"s3://my-bucket/library.egg\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
},
"jar": {
"description": "URI of the jar to be installed. Currently only DBFS and S3 URIs are supported.\nFor example: `{ \"jar\": \"dbfs:/mnt/databricks/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
},
"maven": {
"description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`",
@ -3578,8 +3726,11 @@
}
}
},
"requirements": {
"description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`"
},
"whl": {
"description": "URI of the wheel to be installed.\nFor example: `{ \"whl\": \"dbfs:/my/whl\" }` or `{ \"whl\": \"s3://my-bucket/whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
"description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI."
}
}
}
@ -3591,10 +3742,10 @@
"description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried."
},
"new_cluster": {
"description": "If new_cluster, a description of a cluster that is created for each task.",
"description": "If new_cluster, a description of a new cluster that is created for each run.",
"properties": {
"apply_policy_default_values": {
"description": ""
"description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied."
},
"autoscale": {
"description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.",
@ -3670,14 +3821,6 @@
}
}
},
"clone_from": {
"description": "When specified, this clones libraries from a source cluster during the creation of a new cluster.",
"properties": {
"source_cluster_id": {
"description": "The cluster that is being cloned."
}
}
},
"cluster_log_conf": {
"description": "The configuration for delivering spark logs to a long-term storage destination.\nTwo kinds of destinations (dbfs and s3) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.",
"properties": {
@ -3720,9 +3863,6 @@
"cluster_name": {
"description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\n"
},
"cluster_source": {
"description": ""
},
"custom_tags": {
"description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags",
"additionalproperties": {
@ -3939,6 +4079,9 @@
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
},
"warehouse_id": {
"description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail."
}
}
},
@ -4035,7 +4178,7 @@
}
},
"python_named_params": {
"description": "A map from keys to values for jobs with Python wheel task, for example `\"python_named_params\": {\"name\": \"task\", \"data\": \"dbfs:/path/to/data.json\"}`.",
"description": "",
"additionalproperties": {
"description": ""
}
@ -4090,7 +4233,7 @@
"description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository."
}
}
},
@ -4162,13 +4305,13 @@
}
},
"file": {
"description": "If file, indicates that this job runs a SQL file in a remote Git repository. Only one SQL statement is supported in a file. Multiple SQL statements separated by semicolons (;) are not permitted.",
"description": "If file, indicates that this job runs a SQL file in a remote Git repository.",
"properties": {
"path": {
"description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths."
},
"source": {
"description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider."
"description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider."
}
}
},
@ -4270,10 +4413,10 @@
}
},
"pause_status": {
"description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED."
"description": "Whether this trigger is paused or not."
},
"table": {
"description": "",
"description": "Old table trigger settings name. Deprecated in favor of `table_update`.",
"properties": {
"condition": {
"description": "The table(s) condition based on which to trigger a job run."
@ -4315,7 +4458,7 @@
}
},
"webhook_notifications": {
"description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.",
"description": "A collection of system notification IDs to notify when runs of this job begin or complete.",
"properties": {
"on_duration_warning_threshold_exceeded": {
"description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.",
@ -4469,6 +4612,15 @@
"openai_config": {
"description": "OpenAI Config. Only required if the provider is 'openai'.",
"properties": {
"microsoft_entra_client_id": {
"description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Client ID.\n"
},
"microsoft_entra_client_secret": {
"description": "The Databricks secret key reference for the Microsoft Entra Client Secret that is\nonly required for Azure AD OpenAI.\n"
},
"microsoft_entra_tenant_id": {
"description": "This field is only required for Azure AD OpenAI and is the Microsoft Entra Tenant ID.\n"
},
"openai_api_base": {
"description": "This is the base URL for the OpenAI API (default: \"https://api.openai.com/v1\").\nFor Azure OpenAI, this field is required, and is the base URL for the Azure OpenAI API service\nprovided by Azure.\n"
},
@ -4625,6 +4777,9 @@
}
}
},
"route_optimized": {
"description": "Enable route optimization for the serving endpoint."
},
"tags": {
"description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.",
"items": {
@ -5051,6 +5206,17 @@
"continuous": {
"description": "Whether the pipeline is continuous or triggered. This replaces `trigger`."
},
"deployment": {
"description": "Deployment type of this pipeline.",
"properties": {
"kind": {
"description": "The deployment method that manages the pipeline."
},
"metadata_file_path": {
"description": "The path to the file containing metadata about the deployment."
}
}
},
"development": {
"description": "Whether the pipeline is in Development mode. Defaults to false."
},
@ -5074,9 +5240,136 @@
}
}
},
"gateway_definition": {
"description": "The definition of a gateway pipeline to support CDC.",
"properties": {
"connection_id": {
"description": "Immutable. The Unity Catalog connection this gateway pipeline uses to communicate with the source."
},
"gateway_storage_catalog": {
"description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location."
},
"gateway_storage_name": {
"description": "Required. The Unity Catalog-compatible naming for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.\n"
},
"gateway_storage_schema": {
"description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location."
}
}
},
"id": {
"description": "Unique identifier for this pipeline."
},
"ingestion_definition": {
"description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'target' or 'catalog' settings.",
"properties": {
"connection_name": {
"description": "Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the source. Specify either ingestion_gateway_id or connection_name."
},
"ingestion_gateway_id": {
"description": "Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate with the source. Specify either ingestion_gateway_id or connection_name."
},
"objects": {
"description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.",
"items": {
"description": "",
"properties": {
"schema": {
"description": "Select tables from a specific source schema.",
"properties": {
"destination_catalog": {
"description": "Required. Destination catalog to store tables."
},
"destination_schema": {
"description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists."
},
"source_catalog": {
"description": "The source catalog name. Might be optional depending on the type of source."
},
"source_schema": {
"description": "Required. Schema name in the source database."
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the ManagedIngestionPipelineDefinition object.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
},
"table": {
"description": "Select tables from a specific source table.",
"properties": {
"destination_catalog": {
"description": "Required. Destination catalog to store table."
},
"destination_schema": {
"description": "Required. Destination schema to store table."
},
"destination_table": {
"description": "Optional. Destination table name. The pipeline fails If a table with that name already exists. If not set, the source table name is used."
},
"source_catalog": {
"description": "Source catalog name. Might be optional depending on the type of source."
},
"source_schema": {
"description": "Schema name in the source database. Might be optional depending on the type of source."
},
"source_table": {
"description": "Required. Table name in the source database."
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the ManagedIngestionPipelineDefinition object and the SchemaSpec.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
}
}
}
},
"table_configuration": {
"description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.",
"properties": {
"primary_keys": {
"description": "The primary key of the table used to apply changes.",
"items": {
"description": ""
}
},
"salesforce_include_formula_fields": {
"description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector"
},
"scd_type": {
"description": "The SCD type to use to ingest the table."
}
}
}
}
},
"libraries": {
"description": "Libraries or code needed by this deployment.",
"items": {

43
bundle/schema/openapi.go
View File

@ -10,17 +10,21 @@ import (
)
type OpenapiReader struct {
// OpenAPI spec to read schemas from.
OpenapiSpec *openapi.Specification
Memo map[string]*jsonschema.Schema
// In-memory cache of schemas read from the OpenAPI spec.
memo map[string]jsonschema.Schema
}
const SchemaPathPrefix = "#/components/schemas/"
func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema, error) {
// Read a schema directly from the OpenAPI spec.
func (reader *OpenapiReader) readOpenapiSchema(path string) (jsonschema.Schema, error) {
schemaKey := strings.TrimPrefix(path, SchemaPathPrefix)
// return early if we already have a computed schema
memoSchema, ok := reader.Memo[schemaKey]
memoSchema, ok := reader.memo[schemaKey]
if ok {
return memoSchema, nil
}
@ -28,18 +32,18 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema,
// check path is present in openapi spec
openapiSchema, ok := reader.OpenapiSpec.Components.Schemas[schemaKey]
if !ok {
return nil, fmt.Errorf("schema with path %s not found in openapi spec", path)
return jsonschema.Schema{}, fmt.Errorf("schema with path %s not found in openapi spec", path)
}
// convert openapi schema to the native schema struct
bytes, err := json.Marshal(*openapiSchema)
if err != nil {
return nil, err
return jsonschema.Schema{}, err
}
jsonSchema := &jsonschema.Schema{}
err = json.Unmarshal(bytes, jsonSchema)
jsonSchema := jsonschema.Schema{}
err = json.Unmarshal(bytes, &jsonSchema)
if err != nil {
return nil, err
return jsonschema.Schema{}, err
}
// A hack to convert a map[string]interface{} to *Schema
@ -49,23 +53,28 @@ func (reader *OpenapiReader) readOpenapiSchema(path string) (*jsonschema.Schema,
if ok {
b, err := json.Marshal(jsonSchema.AdditionalProperties)
if err != nil {
return nil, err
return jsonschema.Schema{}, err
}
additionalProperties := &jsonschema.Schema{}
err = json.Unmarshal(b, additionalProperties)
if err != nil {
return nil, err
return jsonschema.Schema{}, err
}
jsonSchema.AdditionalProperties = additionalProperties
}
// store read schema into memo
reader.Memo[schemaKey] = jsonSchema
reader.memo[schemaKey] = jsonSchema
return jsonSchema, nil
}
// safe againt loops in refs
// Resolve all nested "$ref" references in the schema. This function unrolls a single
// level of "$ref" in the schema and calls into traverseSchema to resolve nested references.
// Thus this function and traverseSchema are mutually recursive.
//
// This function is safe against reference loops. If a reference loop is detected, an error
// is returned.
func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) {
if root.Reference == nil {
return reader.traverseSchema(root, tracker)
@ -91,12 +100,12 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t
// in the memo
root.Reference = nil
// unroll one level of reference
// unroll one level of reference.
selfRef, err := reader.readOpenapiSchema(ref)
if err != nil {
return nil, err
}
root = selfRef
root = &selfRef
root.Description = description
// traverse again to find new references
@ -108,6 +117,8 @@ func (reader *OpenapiReader) safeResolveRefs(root *jsonschema.Schema, tracker *t
return root, err
}
// Traverse the nested properties of the schema to resolve "$ref" references. This function
// and safeResolveRefs are mutually recursive.
func (reader *OpenapiReader) traverseSchema(root *jsonschema.Schema, tracker *tracker) (*jsonschema.Schema, error) {
// case primitive (or invalid)
if root.Type != jsonschema.ObjectType && root.Type != jsonschema.ArrayType {
@ -154,11 +165,11 @@ func (reader *OpenapiReader) readResolvedSchema(path string) (*jsonschema.Schema
}
tracker := newTracker()
tracker.push(path, path)
root, err = reader.safeResolveRefs(root, tracker)
resolvedRoot, err := reader.safeResolveRefs(&root, tracker)
if err != nil {
return nil, tracker.errWithTrace(err.Error(), "")
}
return root, nil
return resolvedRoot, nil
}
func (reader *OpenapiReader) jobsDocs() (*Docs, error) {

74
bundle/schema/openapi_test.go
View File

@ -48,7 +48,7 @@ func TestReadSchemaForObject(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -106,7 +106,7 @@ func TestReadSchemaForArray(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -152,7 +152,7 @@ func TestReadSchemaForMap(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -201,7 +201,7 @@ func TestRootReferenceIsResolved(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -251,7 +251,7 @@ func TestSelfReferenceLoopErrors(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -285,7 +285,7 @@ func TestCrossReferenceLoopErrors(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -330,7 +330,7 @@ func TestReferenceResolutionForMapInObject(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -400,7 +400,7 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) {
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
Memo: make(map[string]*jsonschema.Schema),
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
@ -434,3 +434,61 @@ func TestReferenceResolutionForArrayInObject(t *testing.T) {
t.Log("[DEBUG] expected: ", expected)
assert.Equal(t, expected, string(fruitsSchemaJson))
}
func TestReferenceResolutionDoesNotOverwriteDescriptions(t *testing.T) {
specString := `{
"components": {
"schemas": {
"foo": {
"type": "number"
},
"fruits": {
"type": "object",
"properties": {
"guava": {
"type": "object",
"description": "Guava is a fruit",
"$ref": "#/components/schemas/foo"
},
"mango": {
"type": "object",
"description": "What is a mango?",
"$ref": "#/components/schemas/foo"
}
}
}
}
}
}`
spec := &openapi.Specification{}
reader := &OpenapiReader{
OpenapiSpec: spec,
memo: make(map[string]jsonschema.Schema),
}
err := json.Unmarshal([]byte(specString), spec)
require.NoError(t, err)
fruitsSchema, err := reader.readResolvedSchema("#/components/schemas/fruits")
require.NoError(t, err)
fruitsSchemaJson, err := json.MarshalIndent(fruitsSchema, " ", " ")
require.NoError(t, err)
expected := `{
"type": "object",
"properties": {
"guava": {
"type": "number",
"description": "Guava is a fruit"
},
"mango": {
"type": "number",
"description": "What is a mango?"
}
}
}`
t.Log("[DEBUG] actual: ", string(fruitsSchemaJson))
t.Log("[DEBUG] expected: ", expected)
assert.Equal(t, expected, string(fruitsSchemaJson))
}

17
bundle/schema/schema.go
View File

@ -6,6 +6,7 @@ import (
"reflect"
"strings"
"github.com/databricks/cli/libs/dyn/dynvar"
"github.com/databricks/cli/libs/jsonschema"
)
@ -167,6 +168,22 @@ func toSchema(golangType reflect.Type, docs *Docs, tracker *tracker) (*jsonschem
}
jsonSchema := &jsonschema.Schema{Type: rootJavascriptType}
// If the type is a non-string primitive, then we allow it to be a string
// provided it's a pure variable reference (ie only a single variable reference).
if rootJavascriptType == jsonschema.BooleanType || rootJavascriptType == jsonschema.NumberType {
jsonSchema = &jsonschema.Schema{
AnyOf: []*jsonschema.Schema{
{
Type: rootJavascriptType,
},
{
Type: jsonschema.StringType,
Pattern: dynvar.VariableRegex,
},
},
}
}
if docs != nil {
jsonSchema.Description = docs.Description
}

490
bundle/schema/schema_test.go
View File

@ -14,7 +14,15 @@ func TestIntSchema(t *testing.T) {
expected :=
`{
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}`
schema, err := New(reflect.TypeOf(elemInt), nil)
@ -33,7 +41,15 @@ func TestBooleanSchema(t *testing.T) {
expected :=
`{
"type": "boolean"
"anyOf": [
{
"type": "boolean"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}`
schema, err := New(reflect.TypeOf(elem), nil)
@ -101,46 +117,150 @@ func TestStructOfPrimitivesSchema(t *testing.T) {
"type": "object",
"properties": {
"bool_val": {
"type": "boolean"
"anyOf": [
{
"type": "boolean"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"float32_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"float64_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"int16_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"int32_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"int64_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"int8_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"int_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"string_val": {
"type": "string"
},
"uint16_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"uint32_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"uint64_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"uint8_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"uint_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -200,7 +320,15 @@ func TestStructOfStructsSchema(t *testing.T) {
"type": "object",
"properties": {
"a": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"b": {
"type": "string"
@ -257,7 +385,15 @@ func TestStructOfMapsSchema(t *testing.T) {
"my_map": {
"type": "object",
"additionalProperties": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
}
},
@ -339,7 +475,15 @@ func TestMapOfPrimitivesSchema(t *testing.T) {
`{
"type": "object",
"additionalProperties": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
}`
@ -368,7 +512,15 @@ func TestMapOfStructSchema(t *testing.T) {
"type": "object",
"properties": {
"my_int": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -398,7 +550,15 @@ func TestMapOfMapSchema(t *testing.T) {
"additionalProperties": {
"type": "object",
"additionalProperties": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
}
}`
@ -495,7 +655,15 @@ func TestSliceOfMapSchema(t *testing.T) {
"items": {
"type": "object",
"additionalProperties": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
}
}`
@ -525,7 +693,15 @@ func TestSliceOfStructSchema(t *testing.T) {
"type": "object",
"properties": {
"my_int": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -575,7 +751,15 @@ func TestEmbeddedStructSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"country": {
"type": "string"
@ -607,7 +791,15 @@ func TestEmbeddedStructSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"home": {
"type": "object",
@ -694,7 +886,15 @@ func TestNonAnnotatedFieldsAreSkipped(t *testing.T) {
"type": "object",
"properties": {
"bar": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -728,7 +928,15 @@ func TestDashFieldsAreSkipped(t *testing.T) {
"type": "object",
"properties": {
"bar": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -773,7 +981,15 @@ func TestPointerInStructSchema(t *testing.T) {
"type": "object",
"properties": {
"ptr_val2": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -782,13 +998,29 @@ func TestPointerInStructSchema(t *testing.T) {
]
},
"float_val": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"ptr_bar": {
"type": "object",
"properties": {
"ptr_val2": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -797,7 +1029,15 @@ func TestPointerInStructSchema(t *testing.T) {
]
},
"ptr_int": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"ptr_string": {
"type": "string"
@ -860,7 +1100,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -875,7 +1123,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -895,7 +1151,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -910,7 +1174,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -932,7 +1204,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -950,7 +1230,15 @@ func TestGenericSchema(t *testing.T) {
"type": "object",
"properties": {
"age": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"name": {
"type": "string"
@ -1028,16 +1316,40 @@ func TestFieldsWithoutOmitEmptyAreRequired(t *testing.T) {
"type": "object",
"properties": {
"apple": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"bar": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"papaya": {
"type": "object",
"properties": {
"a": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"b": {
"type": "string"
@ -1111,7 +1423,15 @@ func TestDocIngestionForObject(t *testing.T) {
"description": "docs for a"
},
"b": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -1185,12 +1505,28 @@ func TestDocIngestionForSlice(t *testing.T) {
"type": "object",
"properties": {
"guava": {
"type": "number",
"description": "docs for guava"
"description": "docs for guava",
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"pineapple": {
"type": "number",
"description": "docs for pineapple"
"description": "docs for pineapple",
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -1268,12 +1604,28 @@ func TestDocIngestionForMap(t *testing.T) {
"type": "object",
"properties": {
"apple": {
"type": "number",
"description": "docs for apple"
"description": "docs for apple",
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"mango": {
"type": "number",
"description": "docs for mango"
"description": "docs for mango",
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -1324,8 +1676,16 @@ func TestDocIngestionForTopLevelPrimitive(t *testing.T) {
"description": "docs for root",
"properties": {
"my_val": {
"type": "number",
"description": "docs for my val"
"description": "docs for my val",
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
}
},
"additionalProperties": false,
@ -1395,7 +1755,15 @@ func TestInterfaceGeneratesEmptySchema(t *testing.T) {
"type": "object",
"properties": {
"apple": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"mango": {}
},
@ -1436,7 +1804,15 @@ func TestBundleReadOnlytag(t *testing.T) {
"type": "object",
"properties": {
"apple": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"pokemon": {
"type": "object",
@ -1488,7 +1864,15 @@ func TestBundleInternalTag(t *testing.T) {
"type": "object",
"properties": {
"apple": {
"type": "number"
"anyOf": [
{
"type": "number"
},
{
"type": "string",
"pattern": "\\$\\{([a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*)*)\\}"
}
]
},
"pokemon": {
"type": "object",

11
bundle/tests/enviroment_key_test.go
View File

@ -1,8 +1,11 @@
package config_tests
import (
"context"
"testing"
"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/libraries"
"github.com/stretchr/testify/require"
)
@ -10,3 +13,11 @@ func TestEnvironmentKeySupported(t *testing.T) {
_, diags := loadTargetWithDiags("./python_wheel/environment_key", "default")
require.Empty(t, diags)
}
func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) {
b, diags := loadTargetWithDiags("./environment_key_only", "default")
require.Empty(t, diags)
diags = bundle.Apply(context.Background(), b, libraries.ValidateLocalLibrariesExist())
require.Empty(t, diags)
}

8
bundle/tests/environment_git_test.go
View File

@ -1,6 +1,8 @@
package config_tests
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@ -9,12 +11,14 @@ import (
func TestGitAutoLoadWithEnvironment(t *testing.T) {
b := load(t, "./environments_autoload_git")
assert.True(t, b.Config.Bundle.Git.Inferred)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
}
func TestGitManuallySetBranchWithEnvironment(t *testing.T) {
b := loadTarget(t, "./environments_autoload_git", "production")
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
}

16
bundle/tests/environment_key_only/databricks.yml Normal file
View File

@ -0,0 +1,16 @@
bundle:
name: environment_key_only
resources:
jobs:
test_job:
name: "My Wheel Job"
tasks:
- task_key: TestTask
existing_cluster_id: "0717-132531-5opeqon1"
python_wheel_task:
package_name: "my_test_code"
entry_point: "run"
environment_key: "test_env"
environments:
- environment_key: "test_env"

8
bundle/tests/git_test.go
View File

@ -2,6 +2,8 @@ package config_tests
import (
"context"
"fmt"
"strings"
"testing"
"github.com/databricks/cli/bundle"
@ -13,14 +15,16 @@ import (
func TestGitAutoLoad(t *testing.T) {
b := load(t, "./autoload_git")
assert.True(t, b.Config.Bundle.Git.Inferred)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
}
func TestGitManuallySetBranch(t *testing.T) {
b := loadTarget(t, "./autoload_git", "production")
assert.False(t, b.Config.Bundle.Git.Inferred)
assert.Equal(t, "main", b.Config.Bundle.Git.Branch)
assert.Contains(t, b.Config.Bundle.Git.OriginURL, "/cli")
validUrl := strings.Contains(b.Config.Bundle.Git.OriginURL, "/cli") || strings.Contains(b.Config.Bundle.Git.OriginURL, "/bricks")
assert.True(t, validUrl, fmt.Sprintf("Expected URL to contain '/cli' or '/bricks', got %s", b.Config.Bundle.Git.OriginURL))
}
func TestGitBundleBranchValidation(t *testing.T) {

1
bundle/tests/include_multiple/my_first_job/resource.yml
View File

@ -2,3 +2,4 @@ resources:
jobs:
my_first_job:
id: 1
name: "My First Job"

1
bundle/tests/include_multiple/my_second_job/resource.yml
View File

@ -2,3 +2,4 @@ resources:
jobs:
my_second_job:
id: 2
name: "My Second Job"

1
bundle/tests/include_with_glob/job.yml
View File

@ -2,3 +2,4 @@ resources:
jobs:
my_job:
id: 1
name: "My Job"

40
bundle/tests/quality_monitor/databricks.yml Normal file
View File

@ -0,0 +1,40 @@
resources:
quality_monitors:
my_monitor:
table_name: "main.test.thing1"
assets_dir: "/Shared/provider-test/databricks_monitoring/main.test.thing1"
output_schema_name: "test"
inference_log:
granularities: ["1 day"]
timestamp_col: "timestamp"
prediction_col: "prediction"
model_id_col: "model_id"
problem_type: "PROBLEM_TYPE_REGRESSION"
targets:
development:
mode: development
resources:
quality_monitors:
my_monitor:
table_name: "main.test.dev"
staging:
resources:
quality_monitors:
my_monitor:
table_name: "main.test.staging"
output_schema_name: "staging"
production:
resources:
quality_monitors:
my_monitor:
table_name: "main.test.prod"
output_schema_name: "prod"
inference_log:
granularities: ["1 hour"]
timestamp_col: "timestamp_prod"
prediction_col: "prediction_prod"
model_id_col: "model_id_prod"
problem_type: "PROBLEM_TYPE_REGRESSION"

59
bundle/tests/quality_monitor_test.go Normal file
View File

@ -0,0 +1,59 @@
package config_tests
import (
"testing"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/stretchr/testify/assert"
)
func assertExpectedMonitor(t *testing.T, p *resources.QualityMonitor) {
assert.Equal(t, "timestamp", p.InferenceLog.TimestampCol)
assert.Equal(t, "prediction", p.InferenceLog.PredictionCol)
assert.Equal(t, "model_id", p.InferenceLog.ModelIdCol)
assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
}
func TestMonitorTableNames(t *testing.T) {
b := loadTarget(t, "./quality_monitor", "development")
assert.Len(t, b.Config.Resources.QualityMonitors, 1)
assert.Equal(t, b.Config.Bundle.Mode, config.Development)
p := b.Config.Resources.QualityMonitors["my_monitor"]
assert.Equal(t, "main.test.dev", p.TableName)
assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
assert.Equal(t, "test", p.OutputSchemaName)
assertExpectedMonitor(t, p)
}
func TestMonitorStaging(t *testing.T) {
b := loadTarget(t, "./quality_monitor", "staging")
assert.Len(t, b.Config.Resources.QualityMonitors, 1)
p := b.Config.Resources.QualityMonitors["my_monitor"]
assert.Equal(t, "main.test.staging", p.TableName)
assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
assert.Equal(t, "staging", p.OutputSchemaName)
assertExpectedMonitor(t, p)
}
func TestMonitorProduction(t *testing.T) {
b := loadTarget(t, "./quality_monitor", "production")
assert.Len(t, b.Config.Resources.QualityMonitors, 1)
p := b.Config.Resources.QualityMonitors["my_monitor"]
assert.Equal(t, "main.test.prod", p.TableName)
assert.Equal(t, "/Shared/provider-test/databricks_monitoring/main.test.thing1", p.AssetsDir)
assert.Equal(t, "prod", p.OutputSchemaName)
inferenceLog := p.InferenceLog
assert.Equal(t, []string{"1 day", "1 hour"}, inferenceLog.Granularities)
assert.Equal(t, "timestamp_prod", p.InferenceLog.TimestampCol)
assert.Equal(t, "prediction_prod", p.InferenceLog.PredictionCol)
assert.Equal(t, "model_id_prod", p.InferenceLog.ModelIdCol)
assert.Equal(t, catalog.MonitorInferenceLogProblemType("PROBLEM_TYPE_REGRESSION"), p.InferenceLog.ProblemType)
}

8
bundle/tests/undefined_job/databricks.yml Normal file
View File

@ -0,0 +1,8 @@
bundle:
name: undefined-job
resources:
jobs:
undefined:
test:
name: "Test Job"

12
bundle/tests/undefined_job_test.go Normal file
View File

@ -0,0 +1,12 @@
package config_tests
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestUndefinedJobLoadsWithError(t *testing.T) {
_, diags := loadTargetWithDiags("./undefined_job", "default")
assert.ErrorContains(t, diags.Error(), "job undefined is not defined")
}

41
bundle/tests/variables/variable_overrides_in_target/databricks.yml Normal file
View File

@ -0,0 +1,41 @@
bundle:
name: foobar
resources:
pipelines:
my_pipeline:
name: ${var.foo}
continuous: ${var.baz}
clusters:
- num_workers: ${var.bar}
variables:
foo:
default: "a_string"
description: "A string variable"
bar:
default: 42
description: "An integer variable"
baz:
default: true
description: "A boolean variable"
targets:
use-default-variable-values:
override-string-variable:
variables:
foo: "overridden_string"
override-int-variable:
variables:
bar: 43
override-both-bool-and-string-variables:
variables:
foo: "overridden_string"
baz: false

49
bundle/tests/variables_test.go
View File

@ -120,3 +120,52 @@ func TestVariablesWithTargetLookupOverrides(t *testing.T) {
assert.Equal(t, "cluster: some-test-cluster", b.Config.Variables["d"].Lookup.String())
assert.Equal(t, "instance-pool: some-test-instance-pool", b.Config.Variables["e"].Lookup.String())
}
func TestVariableTargetOverrides(t *testing.T) {
var tcases = []struct {
targetName string
pipelineName string
pipelineContinuous bool
pipelineNumWorkers int
}{
{
"use-default-variable-values",
"a_string",
true,
42,
},
{
"override-string-variable",
"overridden_string",
true,
42,
},
{
"override-int-variable",
"a_string",
true,
43,
},
{
"override-both-bool-and-string-variables",
"overridden_string",
false,
42,
},
}
for _, tcase := range tcases {
t.Run(tcase.targetName, func(t *testing.T) {
b := loadTarget(t, "./variables/variable_overrides_in_target", tcase.targetName)
diags := bundle.Apply(context.Background(), b, bundle.Seq(
mutator.SetVariables(),
mutator.ResolveVariableReferences("variables")),
)
require.NoError(t, diags.Error())
assert.Equal(t, tcase.pipelineName, b.Config.Resources.Pipelines["my_pipeline"].Name)
assert.Equal(t, tcase.pipelineContinuous, b.Config.Resources.Pipelines["my_pipeline"].Continuous)
assert.Equal(t, tcase.pipelineNumWorkers, b.Config.Resources.Pipelines["my_pipeline"].Clusters[0].NumWorkers)
})
}
}

2
cmd/account/csp-enablement-account/csp-enablement-account.go generated
View File

@ -156,4 +156,4 @@ func newUpdate() *cobra.Command {
return cmd
}
// end service CSPEnablementAccount
// end service CspEnablementAccount

2
cmd/account/esm-enablement-account/esm-enablement-account.go generated
View File

@ -154,4 +154,4 @@ func newUpdate() *cobra.Command {
return cmd
}
// end service ESMEnablementAccount
// end service EsmEnablementAccount

4
cmd/auth/env.go
View File

@ -10,7 +10,7 @@ import (
"net/url"
"strings"
"github.com/databricks/cli/libs/databrickscfg"
"github.com/databricks/cli/libs/databrickscfg/profile"
"github.com/databricks/databricks-sdk-go/config"
"github.com/spf13/cobra"
"gopkg.in/ini.v1"
@ -70,7 +70,7 @@ func resolveSection(cfg *config.Config, iniFile *config.File) (*ini.Section, err
}
func loadFromDatabricksCfg(ctx context.Context, cfg *config.Config) error {
iniFile, err := databrickscfg.Get(ctx)
iniFile, err := profile.DefaultProfiler.Get(ctx)
if errors.Is(err, fs.ErrNotExist) {
// it's fine not to have ~/.databrickscfg
return nil

41
cmd/auth/login.go
View File

@ -11,6 +11,7 @@ import (
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/databrickscfg"
"github.com/databricks/cli/libs/databrickscfg/cfgpickers"
"github.com/databricks/cli/libs/databrickscfg/profile"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/config"
"github.com/spf13/cobra"
@ -31,6 +32,7 @@ func configureHost(ctx context.Context, persistentAuth *auth.PersistentAuth, arg
}
const minimalDbConnectVersion = "13.1"
const defaultTimeout = 1 * time.Hour
func newLoginCommand(persistentAuth *auth.PersistentAuth) *cobra.Command {
defaultConfigPath := "~/.databrickscfg"
@ -84,7 +86,7 @@ depends on the existing profiles you have set in your configuration file
var loginTimeout time.Duration
var configureCluster bool
cmd.Flags().DurationVar(&loginTimeout, "timeout", auth.DefaultTimeout,
cmd.Flags().DurationVar(&loginTimeout, "timeout", defaultTimeout,
"Timeout for completing login challenge in the browser")
cmd.Flags().BoolVar(&configureCluster, "configure-cluster", false,
"Prompts to configure cluster")
@ -108,7 +110,7 @@ depends on the existing profiles you have set in your configuration file
profileName = profile
}
err := setHost(ctx, profileName, persistentAuth, args)
err := setHostAndAccountId(ctx, profileName, persistentAuth, args)
if err != nil {
return err
}
@ -117,17 +119,10 @@ depends on the existing profiles you have set in your configuration file
// We need the config without the profile before it's used to initialise new workspace client below.
// Otherwise it will complain about non existing profile because it was not yet saved.
cfg := config.Config{
Host: persistentAuth.Host,
AuthType: "databricks-cli",
Host: persistentAuth.Host,
AccountID: persistentAuth.AccountID,
AuthType: "databricks-cli",
}
if cfg.IsAccountClient() && persistentAuth.AccountID == "" {
accountId, err := promptForAccountID(ctx)
if err != nil {
return err
}
persistentAuth.AccountID = accountId
}
cfg.AccountID = persistentAuth.AccountID
ctx, cancel := context.WithTimeout(ctx, loginTimeout)
defer cancel()
@ -172,15 +167,15 @@ depends on the existing profiles you have set in your configuration file
return cmd
}
func setHost(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error {
func setHostAndAccountId(ctx context.Context, profileName string, persistentAuth *auth.PersistentAuth, args []string) error {
profiler := profile.GetProfiler(ctx)
// If the chosen profile has a hostname and the user hasn't specified a host, infer the host from the profile.
_, profiles, err := databrickscfg.LoadProfiles(ctx, func(p databrickscfg.Profile) bool {
return p.Name == profileName
})
profiles, err := profiler.LoadProfiles(ctx, profile.WithName(profileName))
// Tolerate ErrNoConfiguration here, as we will write out a configuration as part of the login flow.
if err != nil && !errors.Is(err, databrickscfg.ErrNoConfiguration) {
if err != nil && !errors.Is(err, profile.ErrNoConfiguration) {
return err
}
if persistentAuth.Host == "" {
if len(profiles) > 0 && profiles[0].Host != "" {
persistentAuth.Host = profiles[0].Host
@ -188,5 +183,17 @@ func setHost(ctx context.Context, profileName string, persistentAuth *auth.Persi
configureHost(ctx, persistentAuth, args, 0)
}
}
isAccountClient := (&config.Config{Host: persistentAuth.Host}).IsAccountClient()
if isAccountClient && persistentAuth.AccountID == "" {
if len(profiles) > 0 && profiles[0].AccountID != "" {
persistentAuth.AccountID = profiles[0].AccountID
} else {
accountId, err := promptForAccountID(ctx)
if err != nil {
return err
}
persistentAuth.AccountID = accountId
}
}
return nil
}

2
cmd/auth/login_test.go
View File

@ -12,6 +12,6 @@ import (
func TestSetHostDoesNotFailWithNoDatabrickscfg(t *testing.T) {
ctx := context.Background()
ctx = env.Set(ctx, "DATABRICKS_CONFIG_FILE", "./imaginary-file/databrickscfg")
err := setHost(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{})
err := setHostAndAccountId(ctx, "foo", &auth.PersistentAuth{Host: "test"}, []string{})
assert.NoError(t, err)
}

4
cmd/auth/profiles.go
View File

@ -8,7 +8,7 @@ import (
"time"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/databrickscfg"
"github.com/databricks/cli/libs/databrickscfg/profile"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/config"
@ -94,7 +94,7 @@ func newProfilesCommand() *cobra.Command {
cmd.RunE = func(cmd *cobra.Command, args []string) error {
var profiles []*profileMetadata
iniFile, err := databrickscfg.Get(cmd.Context())
iniFile, err := profile.DefaultProfiler.Get(cmd.Context())
if os.IsNotExist(err) {
// return empty list for non-configured machines
iniFile = &config.File{

Some files were not shown because too many files have changed in this diff Show More